@hatchet-dev/typescript-sdk 0.1.1 → 0.1.4

package/clients/worker/worker.d.ts

@@ -0,0 +1,34 @@
+import { HatchetClient } from '../../../../src/clients/hatchet-client';
+import { Action, ActionListener } from '../../../../src/clients/dispatcher/action-listener';
+import { StepActionEvent, StepActionEventType, GroupKeyActionEvent, GroupKeyActionEventType } from '../../../../src/protoc/dispatcher';
+import HatchetPromise from '../../../../src/util/hatchet-promise/hatchet-promise';
+import { Workflow } from '../../../../src/workflow';
+import { Logger } from '../../../../src/util/logger';
+import { Context } from '../../step';
+export type ActionRegistry = Record<Action['actionId'], Function>;
+export declare class Worker {
+    serviceName: string;
+    client: HatchetClient;
+    name: string;
+    killing: boolean;
+    handle_kill: boolean;
+    action_registry: ActionRegistry;
+    concurrency_action_registry: ActionRegistry;
+    listener: ActionListener | undefined;
+    futures: Record<Action['stepRunId'], HatchetPromise<any>>;
+    contexts: Record<Action['stepRunId'], Context<any>>;
+    logger: Logger;
+    constructor(client: HatchetClient, options: {
+        name: string;
+        handleKill?: boolean;
+    });
+    registerWorkflow(workflow: Workflow): Promise<void>;
+    handleStartStepRun(action: Action): void;
+    handleStartGroupKeyRun(action: Action): void;
+    getStepActionEvent(action: Action, eventType: StepActionEventType, payload?: any): StepActionEvent;
+    getGroupKeyActionEvent(action: Action, eventType: GroupKeyActionEventType, payload?: any): GroupKeyActionEvent;
+    handleCancelStepRun(action: Action): void;
+    stop(): Promise<void>;
+    exitGracefully(): Promise<void>;
+    start(): Promise<void>;
+}

package/clients/worker/worker.js

@@ -0,0 +1,317 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Worker = void 0;
+const hatchet_error_1 = __importDefault(require("../../../../src/util/errors/hatchet-error"));
+const dispatcher_1 = require("../../../../src/protoc/dispatcher");
+const hatchet_promise_1 = __importDefault(require("../../../../src/util/hatchet-promise/hatchet-promise"));
+const workflows_1 = require("../../../../src/protoc/workflows");
+const logger_1 = require("../../../../src/util/logger");
+const sleep_1 = __importDefault(require("../../../../src/util/sleep"));
+const step_1 = require("../../step");
+class Worker {
+    constructor(client, options) {
+        this.serviceName = 'default';
+        this.futures = {};
+        this.contexts = {};
+        this.client = client;
+        this.name = options.name;
+        this.action_registry = {};
+        this.concurrency_action_registry = {};
+        process.on('SIGTERM', () => this.exitGracefully());
+        process.on('SIGINT', () => this.exitGracefully());
+        this.killing = false;
+        this.handle_kill = options.handleKill === undefined ? true : options.handleKill;
+        this.logger = new logger_1.Logger(`Worker/${this.name}`, this.client.config.log_level);
+    }
+    registerWorkflow(workflow) {
+        var _a, _b;
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const concurrency = ((_a = workflow.concurrency) === null || _a === void 0 ? void 0 : _a.action)
+                    ? {
+                        action: `${this.serviceName}:${workflow.concurrency.action}`,
+                        maxRuns: workflow.concurrency.maxRuns || 1,
+                        limitStrategy: workflow.concurrency.limitStrategy || workflows_1.ConcurrencyLimitStrategy.CANCEL_IN_PROGRESS,
+                    }
+                    : undefined;
+                yield this.client.admin.put_workflow({
+                    name: workflow.id,
+                    description: workflow.description,
+                    version: workflow.version || '',
+                    eventTriggers: workflow.on.event ? [workflow.on.event] : [],
+                    cronTriggers: workflow.on.cron ? [workflow.on.cron] : [],
+                    scheduledTriggers: [],
+                    concurrency,
+                    jobs: [
+                        {
+                            name: workflow.id,
+                            timeout: workflow.timeout || '60s',
+                            description: workflow.description,
+                            steps: workflow.steps.map((step) => {
+                                var _a;
+                                return ({
+                                    readableId: step.name,
+                                    action: `${this.serviceName}:${step.name}`,
+                                    timeout: step.timeout || '60s',
+                                    inputs: '{}',
+                                    parents: (_a = step.parents) !== null && _a !== void 0 ? _a : [],
+                                });
+                            }),
+                        },
+                    ],
+                });
+            }
+            catch (e) {
+                throw new hatchet_error_1.default(`Could not register workflow: ${e.message}`);
+            }
+            this.action_registry = workflow.steps.reduce((acc, step) => {
+                acc[`${this.serviceName}:${step.name}`] = step.run;
+                return acc;
+            }, {});
+            this.concurrency_action_registry = ((_b = workflow.concurrency) === null || _b === void 0 ? void 0 : _b.action)
+                ? {
+                    [`${this.serviceName}:${workflow.concurrency.action}`]: workflow.concurrency.key,
+                }
+                : {};
+        });
+    }
+    handleStartStepRun(action) {
+        const { actionId } = action;
+        const context = new step_1.Context(action.actionPayload);
+        this.contexts[action.stepRunId] = context;
+        const step = this.action_registry[actionId];
+        if (!step) {
+            this.logger.error(`Could not find step '${actionId}'`);
+            return;
+        }
+        const run = () => __awaiter(this, void 0, void 0, function* () {
+            return step(context);
+        });
+        const success = (result) => {
+            this.logger.info(`Step run ${action.stepRunId} succeeded`);
+            try {
+                // Send the action event to the dispatcher
+                const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_COMPLETED, result);
+                this.client.dispatcher.sendStepActionEvent(event);
+                // delete the run from the futures
+                delete this.futures[action.stepRunId];
+            }
+            catch (e) {
+                this.logger.error(`Could not send action event: ${e.message}`);
+            }
+        };
+        const failure = (error) => {
+            this.logger.error(`Step run ${action.stepRunId} failed: ${error.message}`);
+            try {
+                // Send the action event to the dispatcher
+                const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, error);
+                this.client.dispatcher.sendStepActionEvent(event);
+                // delete the run from the futures
+                delete this.futures[action.stepRunId];
+            }
+            catch (e) {
+                this.logger.error(`Could not send action event: ${e.message}`);
+            }
+        };
+        const future = new hatchet_promise_1.default(run().then(success).catch(failure));
+        this.futures[action.stepRunId] = future;
+        try {
+            // Send the action event to the dispatcher
+            const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_STARTED);
+            this.client.dispatcher.sendStepActionEvent(event);
+        }
+        catch (e) {
+            this.logger.error(`Could not send action event: ${e.message}`);
+        }
+    }
+    handleStartGroupKeyRun(action) {
+        const { actionId } = action;
+        const context = new step_1.Context(action.actionPayload);
+        const key = action.getGroupKeyRunId;
+        this.contexts[key] = context;
+        const step = this.action_registry[actionId];
+        if (!step) {
+            this.logger.error(`Could not find step '${actionId}'`);
+            return;
+        }
+        const run = () => __awaiter(this, void 0, void 0, function* () {
+            return step(context);
+        });
+        const success = (result) => {
+            this.logger.info(`Step run ${action.stepRunId} succeeded`);
+            try {
+                // Send the action event to the dispatcher
+                const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_COMPLETED, result);
+                this.client.dispatcher.sendGroupKeyActionEvent(event);
+                // delete the run from the futures
+                delete this.futures[key];
+            }
+            catch (e) {
+                this.logger.error(`Could not send action event: ${e.message}`);
+            }
+        };
+        const failure = (error) => {
+            this.logger.error(`Step run ${key} failed: ${error.message}`);
+            try {
+                // Send the action event to the dispatcher
+                const event = this.getGroupKeyActionEvent(action, dispatcher_1.GroupKeyActionEventType.GROUP_KEY_EVENT_TYPE_FAILED, error);
+                this.client.dispatcher.sendGroupKeyActionEvent(event);
+                // delete the run from the futures
+                delete this.futures[key];
+            }
+            catch (e) {
+                this.logger.error(`Could not send action event: ${e.message}`);
+            }
+        };
+        const future = new hatchet_promise_1.default(run().then(success).catch(failure));
+        this.futures[action.getGroupKeyRunId] = future;
+        try {
+            // Send the action event to the dispatcher
+            const event = this.getStepActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_STARTED);
+            this.client.dispatcher.sendStepActionEvent(event);
+        }
+        catch (e) {
+            this.logger.error(`Could not send action event: ${e.message}`);
+        }
+    }
+    getStepActionEvent(action, eventType, payload = '') {
+        return {
+            workerId: this.name,
+            jobId: action.jobId,
+            jobRunId: action.jobRunId,
+            stepId: action.stepId,
+            stepRunId: action.stepRunId,
+            actionId: action.actionId,
+            eventTimestamp: new Date(),
+            eventType,
+            eventPayload: JSON.stringify(payload),
+        };
+    }
+    getGroupKeyActionEvent(action, eventType, payload = '') {
+        return {
+            workerId: this.name,
+            workflowRunId: action.workflowRunId,
+            getGroupKeyRunId: action.getGroupKeyRunId,
+            actionId: action.actionId,
+            eventTimestamp: new Date(),
+            eventType,
+            eventPayload: JSON.stringify(payload),
+        };
+    }
+    handleCancelStepRun(action) {
+        const { stepRunId } = action;
+        const future = this.futures[stepRunId];
+        const context = this.contexts[stepRunId];
+        //  TODO send cancel signal to context
+        // if (context && context.cancel) {
+        //   context.cancel();
+        //   delete this.contexts[stepRunId];
+        // }
+        if (future) {
+            future.cancel();
+            delete this.futures[stepRunId];
+        }
+    }
+    stop() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.exitGracefully();
+        });
+    }
+    exitGracefully() {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            this.killing = true;
+            this.logger.info('Starting to exit...');
+            try {
+                (_a = this.listener) === null || _a === void 0 ? void 0 : _a.unregister();
+            }
+            catch (e) {
+                this.logger.error(`Could not unregister listener: ${e.message}`);
+            }
+            this.logger.info('Gracefully exiting hatchet worker, running tasks will attempt to finish...');
+            // attempt to wait for futures to finish
+            yield Promise.all(Object.values(this.futures).map(({ promise }) => promise));
+            if (this.handle_kill) {
+                this.logger.info('Exiting hatchet worker...');
+                process.exit(0);
+            }
+        });
+    }
+    start() {
+        var _a, e_1, _b, _c;
+        return __awaiter(this, void 0, void 0, function* () {
+            let retries = 0;
+            while (retries < 5) {
+                try {
+                    this.listener = yield this.client.dispatcher.getActionListener({
+                        workerName: this.name,
+                        services: ['default'],
+                        actions: Object.keys(this.action_registry),
+                    });
+                    const generator = this.listener.actions();
+                    this.logger.info(`Worker ${this.name} listening for actions`);
+                    try {
+                        for (var _d = true, generator_1 = (e_1 = void 0, __asyncValues(generator)), generator_1_1; generator_1_1 = yield generator_1.next(), _a = generator_1_1.done, !_a; _d = true) {
+                            _c = generator_1_1.value;
+                            _d = false;
+                            const action = _c;
+                            this.logger.info(`Worker ${this.name} received action ${action.actionId}:${action.actionType}`);
+                            if (action.actionType === dispatcher_1.ActionType.START_STEP_RUN) {
+                                this.handleStartStepRun(action);
+                            }
+                            else if (action.actionType === dispatcher_1.ActionType.CANCEL_STEP_RUN) {
+                                this.handleCancelStepRun(action);
+                            }
+                            else if (action.actionType === dispatcher_1.ActionType.START_GET_GROUP_KEY) {
+                                this.handleStartGroupKeyRun(action);
+                            }
+                            else {
+                                this.logger.error(`Worker ${this.name} received unknown action type ${action.actionType}`);
+                            }
+                        }
+                    }
+                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                    finally {
+                        try {
+                            if (!_d && !_a && (_b = generator_1.return)) yield _b.call(generator_1);
+                        }
+                        finally { if (e_1) throw e_1.error; }
+                    }
+                    break;
+                }
+                catch (e) {
+                    this.logger.error(`Could not start worker: ${e.message}`);
+                    retries += 1;
+                    const wait = 500;
+                    this.logger.error(`Could not start worker, retrying in ${500} seconds`);
+                    yield (0, sleep_1.default)(wait);
+                }
+            }
+            if (this.killing)
+                return;
+            if (retries > 5) {
+                throw new hatchet_error_1.default('Could not start worker after 5 retries');
+            }
+        });
+    }
+}
+exports.Worker = Worker;

package/index.d.ts

@@ -0,0 +1,2 @@
+import { HatchetClient as Hatchet } from '../../src/clients/hatchet-client';
+export default Hatchet;

package/index.js

@@ -0,0 +1,4 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const hatchet_client_1 = require("../../src/clients/hatchet-client");
+exports.default = hatchet_client_1.HatchetClient;

package/package.json

@@ -1,11 +1,11 @@
 {
   "name": "@hatchet-dev/typescript-sdk",
-  "version": "0.1.1",
+  "version": "0.1.4",
   "description": "Background task orchestration & visibility for developers",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [
-    "dist",
+    "*",
     "!**/*.test.js",
     "!**/*.test.d.ts"
   ],
@@ -15,6 +15,7 @@
   },
   "scripts": {
     "prepare": "npm run build",
+    "fix-paths": "",
     "build": "tsc && resolve-tspaths",
     "test": "jest",
     "test:watch": "jest --watch",