@hatchet-dev/typescript-sdk 0.0.3 → 0.1.0

package/dist/clients/admin/admin-client.d.ts

@@ -5,9 +5,7 @@ export declare class AdminClient {
     config: ClientConfig;
     client: WorkflowServiceClient;
     constructor(config: ClientConfig, channel: Channel, factory: ClientFactory);
-    put_workflow(workflow: CreateWorkflowVersionOpts, options?: {
-        autoVersion?: boolean;
-    }): Promise<void>;
+    put_workflow(workflow: CreateWorkflowVersionOpts): Promise<void>;
     schedule_workflow(workflowId: string, options?: {
         schedules?: Date[];
     }): void;

package/dist/clients/admin/admin-client.js

@@ -20,11 +20,8 @@ class AdminClient {
         this.config = config;
         this.client = factory.create(workflows_1.WorkflowServiceDefinition, channel);
     }
-    put_workflow(workflow, options) {
+    put_workflow(workflow) {
         return __awaiter(this, void 0, void 0, function* () {
-            if (workflow.version === '' && !(options === null || options === void 0 ? void 0 : options.autoVersion)) {
-                throw new hatchet_error_1.default('PutWorkflow error: workflow version is required, or use autoVersion');
-            }
             try {
                 yield this.client.putWorkflow({
                     opts: workflow,

package/dist/clients/dispatcher/action-listener.d.ts

@@ -11,6 +11,8 @@ export interface Action {
     actionId: string;
     actionType: number;
     actionPayload: string;
+    workflowRunId: string;
+    getGroupKeyRunId: string;
 }
 export declare class ActionListener {
     config: ClientConfig;

package/dist/clients/dispatcher/dispatcher-client.d.ts

@@ -1,6 +1,7 @@
 import { Channel, ClientFactory } from 'nice-grpc';
-import { DispatcherClient as PbDispatcherClient, ActionEvent } from '../../protoc/dispatcher';
+import { DispatcherClient as PbDispatcherClient, StepActionEvent, GroupKeyActionEvent } from '../../protoc/dispatcher';
 import { ClientConfig } from '../hatchet-client/client-config';
+import { Logger } from '../../util/logger';
 import { ActionListener } from './action-listener';
 interface GetActionListenerOptions {
     workerName: string;
@@ -10,8 +11,10 @@ interface GetActionListenerOptions {
 export declare class DispatcherClient {
     config: ClientConfig;
     client: PbDispatcherClient;
+    logger: Logger;
     constructor(config: ClientConfig, channel: Channel, factory: ClientFactory);
-    get_action_listener(options: GetActionListenerOptions): Promise<ActionListener>;
-    send_action_event(in_: ActionEvent): Promise<import("../../protoc/dispatcher").ActionEventResponse>;
+    getActionListener(options: GetActionListenerOptions): Promise<ActionListener>;
+    sendStepActionEvent(in_: StepActionEvent): Promise<import("../../protoc/dispatcher").ActionEventResponse>;
+    sendGroupKeyActionEvent(in_: GroupKeyActionEvent): Promise<import("../../protoc/dispatcher").ActionEventResponse>;
 }
 export {};

package/dist/clients/dispatcher/dispatcher-client.js

@@ -15,13 +15,15 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DispatcherClient = void 0;
 const dispatcher_1 = require("../../protoc/dispatcher");
 const hatchet_error_1 = __importDefault(require("../../util/errors/hatchet-error"));
+const logger_1 = require("../../util/logger");
 const action_listener_1 = require("./action-listener");
 class DispatcherClient {
     constructor(config, channel, factory) {
         this.config = config;
         this.client = factory.create(dispatcher_1.DispatcherDefinition, channel);
+        this.logger = new logger_1.Logger(`Dispatcher`, config.log_level);
     }
-    get_action_listener(options) {
+    getActionListener(options) {
         return __awaiter(this, void 0, void 0, function* () {
             // Register the worker
             const registration = yield this.client.register(Object.assign({}, options));
@@ -32,10 +34,20 @@ class DispatcherClient {
             return new action_listener_1.ActionListener(this, listener, registration.workerId);
         });
     }
-    send_action_event(in_) {
+    sendStepActionEvent(in_) {
         return __awaiter(this, void 0, void 0, function* () {
             try {
-                return this.client.sendActionEvent(in_);
+                return this.client.sendStepActionEvent(in_);
+            }
+            catch (e) {
+                throw new hatchet_error_1.default(e.message);
+            }
+        });
+    }
+    sendGroupKeyActionEvent(in_) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                return this.client.sendGroupKeyActionEvent(in_);
             }
             catch (e) {
                 throw new hatchet_error_1.default(e.message);

package/dist/clients/event/event-client.d.ts

@@ -1,9 +1,11 @@
 import { Channel, ClientFactory } from 'nice-grpc';
 import { EventsServiceClient } from '../../protoc/events/events';
 import { ClientConfig } from '../hatchet-client/client-config';
+import { Logger } from '../../util/logger';
 export declare class EventClient {
     config: ClientConfig;
     client: EventsServiceClient;
+    logger: Logger;
     constructor(config: ClientConfig, channel: Channel, factory: ClientFactory);
     push<T>(type: string, input: T): Promise<import("../../protoc/events/events").Event>;
 }

package/dist/clients/event/event-client.js

@@ -6,10 +6,12 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.EventClient = void 0;
 const events_1 = require("../../protoc/events/events");
 const hatchet_error_1 = __importDefault(require("../../util/errors/hatchet-error"));
+const logger_1 = require("../../util/logger");
 class EventClient {
     constructor(config, channel, factory) {
         this.config = config;
         this.client = factory.create(events_1.EventsServiceDefinition, channel);
+        this.logger = new logger_1.Logger(`Dispatcher`, config.log_level);
     }
     push(type, input) {
         const req = {
@@ -18,7 +20,9 @@ class EventClient {
             eventTimestamp: new Date(),
         };
         try {
-            return this.client.push(req);
+            const e = this.client.push(req);
+            this.logger.info(`Event pushed: ${type}`);
+            return e;
         }
         catch (e) {
             throw new hatchet_error_1.default(e.message);

package/dist/clients/hatchet-client/hatchet-client.js

@@ -123,7 +123,7 @@ class HatchetClient {
                 name,
             });
             if (typeof workflow !== 'string') {
-                yield worker.register_workflow(workflow);
+                yield worker.registerWorkflow(workflow);
                 return worker;
             }
             return worker;

package/dist/clients/worker/worker.d.ts

@@ -1,9 +1,10 @@
 import { HatchetClient } from '../hatchet-client';
 import { Action, ActionListener } from '../dispatcher/action-listener';
-import { ActionEvent, ActionEventType } from '../../protoc/dispatcher';
+import { StepActionEvent, StepActionEventType, GroupKeyActionEvent, GroupKeyActionEventType } from '../../protoc/dispatcher';
 import HatchetPromise from '../../util/hatchet-promise/hatchet-promise';
 import { Workflow } from '../../workflow';
 import { Logger } from '../../util/logger';
+import { Context } from '../../step';
 export type ActionRegistry = Record<Action['actionId'], Function>;
 export declare class Worker {
     serviceName: string;
@@ -12,20 +13,22 @@ export declare class Worker {
     killing: boolean;
     handle_kill: boolean;
     action_registry: ActionRegistry;
+    concurrency_action_registry: ActionRegistry;
     listener: ActionListener | undefined;
     futures: Record<Action['stepRunId'], HatchetPromise<any>>;
+    contexts: Record<Action['stepRunId'], Context<any>>;
     logger: Logger;
     constructor(client: HatchetClient, options: {
         name: string;
         handleKill?: boolean;
     });
-    register_workflow(workflow: Workflow, options?: {
-        autoVersion?: boolean;
-    }): Promise<void>;
-    handle_start_step_run(action: Action): void;
-    get_action_event(action: Action, eventType: ActionEventType, payload?: any): ActionEvent;
-    handle_cancel_step_run(action: Action): void;
+    registerWorkflow(workflow: Workflow): Promise<void>;
+    handleStartStepRun(action: Action): void;
+    handleStartGroupKeyRun(action: Action): void;
+    getStepActionEvent(action: Action, eventType: StepActionEventType, payload?: any): StepActionEvent;
+    getGroupKeyActionEvent(action: Action, eventType: GroupKeyActionEventType, payload?: any): GroupKeyActionEvent;
+    handleCancelStepRun(action: Action): void;
     stop(): Promise<void>;
-    exit_gracefully(): Promise<void>;
+    exitGracefully(): Promise<void>;
     start(): Promise<void>;
 }

package/dist/clients/worker/worker.js

@@ -23,6 +23,7 @@ exports.Worker = void 0;
 const hatchet_error_1 = __importDefault(require("../../util/errors/hatchet-error"));
 const dispatcher_1 = require("../../protoc/dispatcher");
 const hatchet_promise_1 = __importDefault(require("../../util/hatchet-promise/hatchet-promise"));
+const workflows_1 = require("../../protoc/workflows");
 const logger_1 = require("../../util/logger");
 const sleep_1 = __importDefault(require("../../util/sleep"));
 const step_1 = require("../../step");
@@ -30,44 +31,53 @@ class Worker {
     constructor(client, options) {
         this.serviceName = 'default';
         this.futures = {};
+        this.contexts = {};
         this.client = client;
         this.name = options.name;
         this.action_registry = {};
-        process.on('SIGTERM', () => this.exit_gracefully());
-        process.on('SIGINT', () => this.exit_gracefully());
+        this.concurrency_action_registry = {};
+        process.on('SIGTERM', () => this.exitGracefully());
+        process.on('SIGINT', () => this.exitGracefully());
         this.killing = false;
         this.handle_kill = options.handleKill === undefined ? true : options.handleKill;
         this.logger = new logger_1.Logger(`Worker/${this.name}`, this.client.config.log_level);
     }
-    register_workflow(workflow, options) {
+    registerWorkflow(workflow) {
+        var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
             try {
+                const concurrency = ((_a = workflow.concurrency) === null || _a === void 0 ? void 0 : _a.action)
+                    ? {
+                        action: `${this.serviceName}:${workflow.concurrency.action}`,
+                        maxRuns: workflow.concurrency.maxRuns || 1,
+                        limitStrategy: workflow.concurrency.limitStrategy || workflows_1.ConcurrencyLimitStrategy.CANCEL_IN_PROGRESS,
+                    }
+                    : undefined;
                 yield this.client.admin.put_workflow({
                     name: workflow.id,
                     description: workflow.description,
-                    version: 'v0.55.0', // FIXME  workflow.version,
+                    version: workflow.version || '',
                     eventTriggers: workflow.on.event ? [workflow.on.event] : [],
                     cronTriggers: workflow.on.cron ? [workflow.on.cron] : [],
                     scheduledTriggers: [],
+                    concurrency,
                     jobs: [
                         {
-                            name: 'my-job', // FIXME variable names
-                            timeout: '60s',
-                            description: 'my-job',
+                            name: workflow.id,
+                            timeout: workflow.timeout || '60s',
+                            description: workflow.description,
                             steps: workflow.steps.map((step) => {
                                 var _a;
                                 return ({
                                     readableId: step.name,
                                     action: `${this.serviceName}:${step.name}`,
-                                    timeout: '60s',
+                                    timeout: step.timeout || '60s',
                                     inputs: '{}',
                                     parents: (_a = step.parents) !== null && _a !== void 0 ? _a : [],
                                 });
                             }),
                         },
                     ],
-                }, {
-                    autoVersion: !(options === null || options === void 0 ? void 0 : options.autoVersion),
                 });
             }
             catch (e) {
@@ -77,11 +87,17 @@ class Worker {
                 acc[`${this.serviceName}:${step.name}`] = step.run;
                 return acc;
             }, {});
+            this.concurrency_action_registry = ((_b = workflow.concurrency) === null || _b === void 0 ? void 0 : _b.action)
+                ? {
+                    [`${this.serviceName}:${workflow.concurrency.action}`]: workflow.concurrency.key,
+                }
+                : {};
         });
     }
-    handle_start_step_run(action) {
+    handleStartStepRun(action) {
         const { actionId } = action;
         const context = new step_1.Context(action.actionPayload);
+        this.contexts[action.stepRunId] = context;
         const step = this.action_registry[actionId];
         if (!step) {
             this.logger.error(`Could not find step '${actionId}'`);
@@ -94,8 +110,8 @@ class Worker {
             this.logger.info(`Step run ${action.stepRunId} succeeded`);
             try {
                 // Send the action event to the dispatcher
-                const event = this.get_action_event(action, dispatcher_1.ActionEventType.STEP_EVENT_TYPE_COMPLETED, result);
-                this.client.dispatcher.send_action_event(event);
+                const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_COMPLETED, result);
+                this.client.dispatcher.sendStepActionEvent(event);
                 // delete the run from the futures
                 delete this.futures[action.stepRunId];
             }
@@ -107,8 +123,8 @@ class Worker {
             this.logger.error(`Step run ${action.stepRunId} failed: ${error.message}`);
             try {
                 // Send the action event to the dispatcher
-                const event = this.get_action_event(action, dispatcher_1.ActionEventType.STEP_EVENT_TYPE_FAILED, error);
-                this.client.dispatcher.send_action_event(event);
+                const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, error);
+                this.client.dispatcher.sendStepActionEvent(event);
                 // delete the run from the futures
                 delete this.futures[action.stepRunId];
             }
@@ -120,14 +136,64 @@ class Worker {
         this.futures[action.stepRunId] = future;
         try {
             // Send the action event to the dispatcher
-            const event = this.get_action_event(action, dispatcher_1.ActionEventType.STEP_EVENT_TYPE_STARTED);
-            this.client.dispatcher.send_action_event(event);
+            const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_STARTED);
+            this.client.dispatcher.sendStepActionEvent(event);
         }
         catch (e) {
             this.logger.error(`Could not send action event: ${e.message}`);
         }
     }
-    get_action_event(action, eventType, payload = '') {
+    handleStartGroupKeyRun(action) {
+        const { actionId } = action;
+        const context = new step_1.Context(action.actionPayload);
+        const key = action.getGroupKeyRunId;
+        this.contexts[key] = context;
+        const step = this.action_registry[actionId];
+        if (!step) {
+            this.logger.error(`Could not find step '${actionId}'`);
+            return;
+        }
+        const run = () => __awaiter(this, void 0, void 0, function* () {
+            return step(context);
+        });
+        const success = (result) => {
+            this.logger.info(`Step run ${action.stepRunId} succeeded`);
+            try {
+                // Send the action event to the dispatcher
+                const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_COMPLETED, result);
+                this.client.dispatcher.sendGroupKeyActionEvent(event);
+                // delete the run from the futures
+                delete this.futures[key];
+            }
+            catch (e) {
+                this.logger.error(`Could not send action event: ${e.message}`);
+            }
+        };
+        const failure = (error) => {
+            this.logger.error(`Step run ${key} failed: ${error.message}`);
+            try {
+                // Send the action event to the dispatcher
+                const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_FAILED, error);
+                this.client.dispatcher.sendGroupKeyActionEvent(event);
+                // delete the run from the futures
+                delete this.futures[key];
+            }
+            catch (e) {
+                this.logger.error(`Could not send action event: ${e.message}`);
+            }
+        };
+        const future = new hatchet_promise_1.default(run().then(success).catch(failure));
+        this.futures[action.getGroupKeyRunId] = future;
+        try {
+            // Send the action event to the dispatcher
+            const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_STARTED);
+            this.client.dispatcher.sendStepActionEvent(event);
+        }
+        catch (e) {
+            this.logger.error(`Could not send action event: ${e.message}`);
+        }
+    }
+    getStepActionEvent(action, eventType, payload = '') {
         return {
             workerId: this.name,
             jobId: action.jobId,
@@ -140,9 +206,26 @@ class Worker {
             eventPayload: JSON.stringify(payload),
         };
     }
-    handle_cancel_step_run(action) {
+    getGroupKeyActionEvent(action, eventType, payload = '') {
+        return {
+            workerId: this.name,
+            workflowRunId: action.workflowRunId,
+            getGroupKeyRunId: action.getGroupKeyRunId,
+            actionId: action.actionId,
+            eventTimestamp: new Date(),
+            eventType,
+            eventPayload: JSON.stringify(payload),
+        };
+    }
+    handleCancelStepRun(action) {
         const { stepRunId } = action;
         const future = this.futures[stepRunId];
+        const context = this.contexts[stepRunId];
+        //  TODO send cancel signal to context
+        // if (context && context.cancel) {
+        //   context.cancel();
+        //   delete this.contexts[stepRunId];
+        // }
         if (future) {
             future.cancel();
             delete this.futures[stepRunId];
@@ -150,10 +233,10 @@ class Worker {
     }
     stop() {
         return __awaiter(this, void 0, void 0, function* () {
-            yield this.exit_gracefully();
+            yield this.exitGracefully();
         });
     }
-    exit_gracefully() {
+    exitGracefully() {
         var _a;
         return __awaiter(this, void 0, void 0, function* () {
             this.killing = true;
@@ -179,7 +262,7 @@ class Worker {
             let retries = 0;
             while (retries < 5) {
                 try {
-                    this.listener = yield this.client.dispatcher.get_action_listener({
+                    this.listener = yield this.client.dispatcher.getActionListener({
                         workerName: this.name,
                         services: ['default'],
                         actions: Object.keys(this.action_registry),
@@ -191,12 +274,18 @@ class Worker {
                             _c = generator_1_1.value;
                             _d = false;
                             const action = _c;
-                            this.logger.info(`Worker ${this.name} received action ${action.actionId}`);
+                            this.logger.info(`Worker ${this.name} received action ${action.actionId}:${action.actionType}`);
                             if (action.actionType === dispatcher_1.ActionType.START_STEP_RUN) {
-                                this.handle_start_step_run(action);
+                                this.handleStartStepRun(action);
                             }
                             else if (action.actionType === dispatcher_1.ActionType.CANCEL_STEP_RUN) {
-                                this.handle_cancel_step_run(action);
+                                this.handleCancelStepRun(action);
+                            }
+                            else if (action.actionType === dispatcher_1.ActionType.START_GET_GROUP_KEY) {
+                                this.handleStartGroupKeyRun(action);
+                            }
+                            else {
+                                this.logger.error(`Worker ${this.name} received unknown action type ${action.actionType}`);
                             }
                         }
                     }

package/dist/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@hatchet-dev/typescript-sdk",
+  "version": "0.1.0",
+  "description": "Background task orchestration & visibility for developers",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "!**/*.test.js",
+    "!**/*.test.d.ts"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/hatchet-dev/hatchet.git"
+  },
+  "scripts": {
+    "prepare": "npm run build",
+    "build": "tsc && resolve-tspaths",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "generate": "./generate-protoc.sh",
+    "lint:check": "npm run eslint:check && npm run prettier:check",
+    "lint:fix": "npm run eslint:fix && npm run prettier:fix",
+    "eslint:check": "eslint \"{hatchet,tests}/**/*.{ts,tsx,js}\"",
+    "eslint:fix": "eslint \"{hatchet,tests}/**/*.{ts,tsx,js}\" --fix",
+    "prettier:check": "prettier \"hatchet/**/*.{ts,tsx}\" --list-different",
+    "prettier:fix": "prettier \"hatchet/**/*.{ts,tsx}\" --write",
+    "exec": "npx dotenv -- ts-node -r tsconfig-paths/register --project tsconfig.json",
+    "example:event": "npm run exec -- ./examples/example-event.ts",
+    "worker:simple": "npm run exec -- ./examples/simple-worker.ts",
+    "worker:dag": "npm run exec -- ./examples/dag-worker.ts",
+    "worker:concurrency": "npm run exec -- ./examples/concurrency/concurrency-worker.ts",
+    "event:concurrency": "npm run exec -- ./examples/concurrency/concurrency-event.ts",
+    "prepublish": "cp package.json dist/package.json",
+    "publish": "npm publish dist --access public"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "devDependencies": {
+    "@types/jest": "^29.5.11",
+    "@typescript-eslint/eslint-plugin": "^6.4.0",
+    "autoprefixer": "^10.4.16",
+    "dotenv-cli": "^7.3.0",
+    "eslint": "^8.56.0",
+    "eslint-config-airbnb-typescript": "^17.1.0",
+    "eslint-config-prettier": "^9.1.0",
+    "eslint-config-standard-with-typescript": "^43.0.0",
+    "eslint-import-resolver-typescript": "^3.6.1",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-jest": "^27.6.3",
+    "eslint-plugin-n": "^15.0.0 || ^16.0.0 ",
+    "eslint-plugin-prettier": "^5.0.1",
+    "eslint-plugin-promise": "^6.0.0",
+    "eslint-plugin-unused-imports": "^3.0.0",
+    "grpc-tools": "^1.12.4",
+    "jest": "^29.7.0",
+    "prettier": "^3.1.1",
+    "resolve-tspaths": "^0.8.17",
+    "ts-jest": "^29.1.1",
+    "ts-node": "^10.9.2",
+    "ts-proto": "^1.167.0",
+    "typescript": "^5.3.3"
+  },
+  "dependencies": {
+    "long": "^5.2.3",
+    "nice-grpc": "^2.1.7",
+    "nice-grpc-common": "^2.0.2",
+    "protobufjs": "^7.2.6",
+    "yaml": "^2.3.4",
+    "zod": "^3.22.4"
+  }
+}