@hatchet-dev/typescript-sdk 0.0.1

package/hatchet/clients/worker/worker.test.ts

@@ -0,0 +1,347 @@
+import { HatchetClient } from '@clients/hatchet-client';
+import { ActionEventType, ActionType, AssignedAction } from '@protoc/dispatcher';
+import { ActionListener } from '@clients/dispatcher/action-listener';
+import { ServerError, Status } from 'nice-grpc-common';
+import { mockListener } from '@clients/dispatcher/action-listener.test';
+import { never } from 'zod';
+import sleep from '@util/sleep';
+import { ChannelCredentials } from 'nice-grpc';
+import { Worker } from './worker';
+import { Context } from '@hatchet/step';
+
+type AssignActionMock = AssignedAction | Error;
+
+const mockStart: AssignActionMock = {
+  tenantId: 'TENANT_ID',
+  jobId: 'job1',
+  jobName: 'Job One',
+  jobRunId: 'run1',
+  stepId: 'step1',
+  stepRunId: 'runStep1',
+  actionId: 'action1',
+  actionType: ActionType.START_STEP_RUN,
+  actionPayload: JSON.stringify('{"input": {"data": 1}}'),
+};
+
+const mockCancel: AssignActionMock = {
+  ...mockStart,
+  actionType: ActionType.CANCEL_STEP_RUN,
+};
+
+describe('Worker', () => {
+  let hatchet: HatchetClient;
+
+  beforeEach(() => {
+    hatchet = new HatchetClient(
+      {
+        token: 'TOKEN',
+        log_level: 'OFF',
+        host_port: 'HOST_PORT',
+        tls_config: {
+          cert_file: 'TLS_CERT_FILE',
+          key_file: 'TLS_KEY_FILE',
+          ca_file: 'TLS_ROOT_CA_FILE',
+          server_name: 'TLS_SERVER_NAME',
+        },
+      },
+      {
+        credentials: ChannelCredentials.createInsecure(),
+      }
+    );
+  });
+
+  describe('register_workflow', () => {
+    it('should update the registry', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+      const putWorkflowSpy = jest.spyOn(worker.client.admin, 'put_workflow').mockResolvedValue();
+
+      const workflow = {
+        id: 'workflow1',
+        description: 'test',
+        on: {
+          event: 'user:create',
+        },
+        steps: [
+          {
+            name: 'step1',
+            run: (ctx: any) => {
+              return { test: 'test' };
+            },
+          },
+        ],
+      };
+
+      await worker.register_workflow(workflow);
+
+      expect(putWorkflowSpy).toHaveBeenCalledTimes(1);
+
+      expect(worker.action_registry).toEqual({
+        [`default:step1`]: workflow.steps[0].run,
+      });
+    });
+  });
+
+  describe('handle_start_step_run', () => {
+    it('should start a step run', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      const putWorkflowSpy = jest.spyOn(worker.client.admin, 'put_workflow').mockResolvedValue();
+
+      const getActionEventSpy = jest.spyOn(worker, 'get_action_event');
+
+      const sendActionEventSpy = jest
+        .spyOn(worker.client.dispatcher, 'send_action_event')
+        .mockResolvedValue({
+          tenantId: 'TENANT_ID',
+          workerId: 'WORKER_ID',
+        });
+
+      const startSpy = jest.fn().mockReturnValue({ data: 4 });
+
+      worker.action_registry = {
+        [mockStart.actionId]: startSpy,
+      };
+
+      worker.handle_start_step_run(mockStart);
+      await sleep(100);
+
+      expect(startSpy).toHaveBeenCalledTimes(1);
+
+      expect(getActionEventSpy).toHaveBeenNthCalledWith(
+        2,
+        expect.anything(),
+        ActionEventType.STEP_EVENT_TYPE_COMPLETED,
+        { data: 4 }
+      );
+      expect(worker.futures[mockStart.stepRunId]).toBeUndefined();
+      expect(sendActionEventSpy).toHaveBeenCalledTimes(2);
+    });
+
+    it('should fail gracefully', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      const getActionEventSpy = jest.spyOn(worker, 'get_action_event');
+
+      const sendActionEventSpy = jest
+        .spyOn(worker.client.dispatcher, 'send_action_event')
+        .mockResolvedValue({
+          tenantId: 'TENANT_ID',
+          workerId: 'WORKER_ID',
+        });
+
+      const startSpy = jest.fn().mockRejectedValue(new Error('ERROR'));
+
+      worker.action_registry = {
+        [mockStart.actionId]: startSpy,
+      };
+
+      worker.handle_start_step_run(mockStart);
+      await sleep(100);
+
+      expect(startSpy).toHaveBeenCalledTimes(1);
+      expect(getActionEventSpy).toHaveBeenNthCalledWith(
+        2,
+        expect.anything(),
+        ActionEventType.STEP_EVENT_TYPE_FAILED,
+        expect.anything()
+      );
+      expect(worker.futures[mockStart.stepRunId]).toBeUndefined();
+      expect(sendActionEventSpy).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  describe('handle_cancel_step_run', () => {
+    it('should cancel a step run', () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      const cancelSpy = jest.fn().mockReturnValue(undefined);
+
+      worker.futures = {
+        [mockCancel.stepRunId]: {
+          cancel: cancelSpy,
+        } as any,
+      };
+
+      worker.handle_cancel_step_run(mockCancel);
+
+      expect(cancelSpy).toHaveBeenCalledTimes(1);
+      expect(worker.futures[mockCancel.stepRunId]).toBeUndefined();
+    });
+  });
+
+  describe('exit_gracefully', () => {
+    xit('should call exit_gracefully on SIGTERM', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      // the spy is not working and the test is killing the test process
+      const exitSpy = jest.spyOn(worker, 'exit_gracefully').mockImplementationOnce(() => {
+        throw new Error('Simulated error');
+      });
+
+      process.emit('SIGTERM', 'SIGTERM');
+      expect(exitSpy).toHaveBeenCalledTimes(1);
+    });
+
+    xit('should call exit_gracefully on SIGINT', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      // This is killing the process (as it should) fix the spy at some point
+      const exitSpy = jest.spyOn(worker, 'exit_gracefully').mockResolvedValue();
+
+      process.emit('SIGINT', 'SIGINT');
+      expect(exitSpy).toHaveBeenCalledTimes(1);
+    });
+
+    xit('should unregister the listener and exit', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      jest.spyOn(process, 'exit').mockImplementation((number) => {
+        throw new Error(`EXIT ${number}`);
+      }); // This is killing the process (as it should) fix the spy at some point
+
+      const mockActionListener = new ActionListener(
+        hatchet.dispatcher,
+        mockListener([mockStart, mockStart, new ServerError(Status.CANCELLED, 'CANCELLED')]),
+        'WORKER_ID'
+      );
+
+      mockActionListener.unregister = jest.fn().mockResolvedValue(never());
+      worker.listener = mockActionListener;
+
+      expect(async () => {
+        await worker.exit_gracefully();
+      }).toThrow('EXIT 0');
+      expect(mockActionListener.unregister).toHaveBeenCalledTimes(1);
+    });
+
+    it('should exit the process if handle_kill is true', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+      const exitSpy = jest.spyOn(process, 'exit').mockReturnValue(undefined as never);
+      await worker.exit_gracefully();
+      expect(exitSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('start', () => {
+    xit('should get actions and start runs', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      const startSpy = jest.spyOn(worker, 'handle_start_step_run').mockReturnValue();
+      const cancelSpy = jest.spyOn(worker, 'handle_cancel_step_run').mockReturnValue();
+
+      const mockActionListener = new ActionListener(
+        hatchet.dispatcher,
+        mockListener([mockStart, mockStart, new ServerError(Status.CANCELLED, 'CANCELLED')]),
+        'WORKER_ID'
+      );
+
+      const getActionListenerSpy = jest
+        .spyOn(worker.client.dispatcher, 'get_action_listener')
+        .mockResolvedValue(mockActionListener);
+
+      await worker.start();
+
+      expect(getActionListenerSpy).toHaveBeenCalledTimes(1);
+      expect(startSpy).toHaveBeenCalledTimes(2);
+      expect(cancelSpy).toHaveBeenCalledTimes(0);
+    });
+
+    it('should get actions and cancel runs', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      const startSpy = jest.spyOn(worker, 'handle_start_step_run').mockReturnValue();
+      const cancelSpy = jest.spyOn(worker, 'handle_cancel_step_run').mockReturnValue();
+
+      const mockActionListner = new ActionListener(
+        hatchet.dispatcher,
+        mockListener([mockStart, mockCancel, new ServerError(Status.CANCELLED, 'CANCELLED')]),
+        'WORKER_ID'
+      );
+
+      const getActionListenerSpy = jest
+        .spyOn(worker.client.dispatcher, 'get_action_listener')
+        .mockResolvedValue(mockActionListner);
+
+      await worker.start();
+
+      expect(getActionListenerSpy).toHaveBeenCalledTimes(1);
+      expect(startSpy).toHaveBeenCalledTimes(1);
+      expect(cancelSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it('should retry 5 times to start a worker then throw an error', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      const startSpy = jest.spyOn(worker, 'handle_start_step_run').mockReturnValue();
+      const cancelSpy = jest.spyOn(worker, 'handle_cancel_step_run').mockReturnValue();
+
+      const mockActionListner = new ActionListener(
+        hatchet.dispatcher,
+        mockListener([mockStart, mockCancel, new ServerError(Status.CANCELLED, 'CANCELLED')]),
+        'WORKER_ID'
+      );
+
+      const getActionListenerSpy = jest
+        .spyOn(worker.client.dispatcher, 'get_action_listener')
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        });
+
+      await worker.start();
+
+      expect(getActionListenerSpy).toHaveBeenCalledTimes(5);
+      expect(startSpy).toHaveBeenCalledTimes(0);
+      expect(cancelSpy).toHaveBeenCalledTimes(0);
+    });
+
+    it('should successfully run after retrying < 5 times', async () => {
+      const worker = new Worker(hatchet, { name: 'WORKER_NAME' });
+
+      const startSpy = jest.spyOn(worker, 'handle_start_step_run').mockReturnValue();
+      const cancelSpy = jest.spyOn(worker, 'handle_cancel_step_run').mockReturnValue();
+
+      const mockActionLister = new ActionListener(
+        hatchet.dispatcher,
+        mockListener([mockStart, mockCancel, new ServerError(Status.CANCELLED, 'CANCELLED')]),
+        'WORKER_ID'
+      );
+
+      const getActionListenerSpy = jest
+        .spyOn(worker.client.dispatcher, 'get_action_listener')
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(() => {
+          throw new Error('Simulated error');
+        })
+        .mockImplementationOnce(async () => mockActionLister);
+
+      await worker.start();
+
+      expect(getActionListenerSpy).toHaveBeenCalledTimes(5);
+      expect(startSpy).toHaveBeenCalledTimes(1);
+      expect(cancelSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+});

package/hatchet/clients/worker/worker.ts

@@ -0,0 +1,229 @@
+import { HatchetClient } from '@clients/hatchet-client';
+import HatchetError from '@util/errors/hatchet-error';
+import { Action, ActionListener } from '@clients/dispatcher/action-listener';
+import { ActionEvent, ActionEventType, ActionType } from '@protoc/dispatcher';
+import HatchetPromise from '@util/hatchet-promise/hatchet-promise';
+import { Workflow } from '@hatchet/workflow';
+import { CreateWorkflowStepOpts } from '@protoc/workflows';
+import { Logger } from '@hatchet/util/logger';
+import sleep from '@hatchet/util/sleep';
+import { Context } from '../../step';
+
+export type ActionRegistry = Record<Action['actionId'], Function>;
+
+export class Worker {
+  serviceName = 'default';
+  client: HatchetClient;
+  name: string;
+  killing: boolean;
+  handle_kill: boolean;
+
+  action_registry: ActionRegistry;
+  listener: ActionListener | undefined;
+  futures: Record<Action['stepRunId'], HatchetPromise<any>> = {};
+
+  logger: Logger;
+
+  constructor(client: HatchetClient, options: { name: string; handleKill?: boolean }) {
+    this.client = client;
+    this.name = options.name;
+    this.action_registry = {};
+
+    process.on('SIGTERM', () => this.exit_gracefully());
+    process.on('SIGINT', () => this.exit_gracefully());
+
+    this.killing = false;
+    this.handle_kill = options.handleKill === undefined ? true : options.handleKill;
+
+    this.logger = new Logger(`Worker/${this.name}`, this.client.config.log_level);
+  }
+
+  async register_workflow(workflow: Workflow, options?: { autoVersion?: boolean }) {
+    try {
+      await this.client.admin.put_workflow(
+        {
+          name: workflow.id,
+          description: workflow.description,
+          version: 'v0.55.0', // FIXME  workflow.version,
+          eventTriggers: workflow.on.event ? [workflow.on.event] : [],
+          cronTriggers: workflow.on.cron ? [workflow.on.cron] : [],
+          scheduledTriggers: [],
+          jobs: [
+            {
+              name: 'my-job', // FIXME variable names
+              timeout: '60s',
+              description: 'my-job',
+              steps: workflow.steps.map<CreateWorkflowStepOpts>((step) => ({
+                readableId: step.name,
+                action: `${this.serviceName}:${step.name}`,
+                timeout: '60s',
+                inputs: '{}',
+                parents: step.parents ?? [],
+              })),
+            },
+          ],
+        },
+        {
+          autoVersion: !options?.autoVersion,
+        }
+      );
+    } catch (e: any) {
+      throw new HatchetError(`Could not register workflow: ${e.message}`);
+    }
+
+    this.action_registry = workflow.steps.reduce<ActionRegistry>((acc, step) => {
+      acc[`${this.serviceName}:${step.name}`] = step.run;
+      return acc;
+    }, {});
+  }
+
+  handle_start_step_run(action: Action) {
+    const { actionId } = action;
+    const context = new Context(action.actionPayload);
+
+    const step = this.action_registry[actionId];
+    if (!step) {
+      this.logger.error(`Could not find step '${actionId}'`);
+      return;
+    }
+
+    const run = async () => {
+      return step(context);
+    };
+
+    const success = (result: any) => {
+      this.logger.info(`Step run ${action.stepRunId} succeeded`);
+
+      try {
+        // Send the action event to the dispatcher
+        const event = this.get_action_event(
+          action,
+          ActionEventType.STEP_EVENT_TYPE_COMPLETED,
+          result
+        );
+        this.client.dispatcher.send_action_event(event);
+
+        // delete the run from the futures
+        delete this.futures[action.stepRunId];
+      } catch (e: any) {
+        this.logger.error(`Could not send action event: ${e.message}`);
+      }
+    };
+
+    const failure = (error: any) => {
+      this.logger.error(`Step run ${action.stepRunId} failed: ${error.message}`);
+
+      try {
+        // Send the action event to the dispatcher
+        const event = this.get_action_event(action, ActionEventType.STEP_EVENT_TYPE_FAILED, error);
+        this.client.dispatcher.send_action_event(event);
+        // delete the run from the futures
+        delete this.futures[action.stepRunId];
+      } catch (e: any) {
+        this.logger.error(`Could not send action event: ${e.message}`);
+      }
+    };
+
+    const future = new HatchetPromise(run().then(success).catch(failure));
+    this.futures[action.stepRunId] = future;
+
+    try {
+      // Send the action event to the dispatcher
+      const event = this.get_action_event(action, ActionEventType.STEP_EVENT_TYPE_STARTED);
+      this.client.dispatcher.send_action_event(event);
+    } catch (e: any) {
+      this.logger.error(`Could not send action event: ${e.message}`);
+    }
+  }
+
+  get_action_event(action: Action, eventType: ActionEventType, payload: any = ''): ActionEvent {
+    return {
+      workerId: this.name,
+      jobId: action.jobId,
+      jobRunId: action.jobRunId,
+      stepId: action.stepId,
+      stepRunId: action.stepRunId,
+      actionId: action.actionId,
+      eventTimestamp: new Date(),
+      eventType,
+      eventPayload: JSON.stringify(payload),
+    };
+  }
+
+  handle_cancel_step_run(action: Action) {
+    const { stepRunId } = action;
+    const future = this.futures[stepRunId];
+    if (future) {
+      future.cancel();
+      delete this.futures[stepRunId];
+    }
+  }
+
+  async stop() {
+    await this.exit_gracefully();
+  }
+
+  async exit_gracefully() {
+    this.killing = true;
+
+    this.logger.info('Starting to exit...');
+
+    try {
+      this.listener?.unregister();
+    } catch (e: any) {
+      this.logger.error(`Could not unregister listener: ${e.message}`);
+    }
+
+    this.logger.info('Gracefully exiting hatchet worker, running tasks will attempt to finish...');
+
+    // attempt to wait for futures to finish
+    await Promise.all(Object.values(this.futures).map(({ promise }) => promise));
+
+    if (this.handle_kill) {
+      this.logger.info('Exiting hatchet worker...');
+      process.exit(0);
+    }
+  }
+
+  async start() {
+    let retries = 0;
+
+    while (retries < 5) {
+      try {
+        this.listener = await this.client.dispatcher.get_action_listener({
+          workerName: this.name,
+          services: ['default'],
+          actions: Object.keys(this.action_registry),
+        });
+
+        const generator = this.listener.actions();
+
+        this.logger.info(`Worker ${this.name} listening for actions`);
+
+        for await (const action of generator) {
+          this.logger.info(`Worker ${this.name} received action ${action.actionId}`);
+
+          if (action.actionType === ActionType.START_STEP_RUN) {
+            this.handle_start_step_run(action);
+          } else if (action.actionType === ActionType.CANCEL_STEP_RUN) {
+            this.handle_cancel_step_run(action);
+          }
+        }
+
+        break;
+      } catch (e: any) {
+        this.logger.error(`Could not start worker: ${e.message}`);
+        retries += 1;
+        const wait = 500;
+        this.logger.error(`Could not start worker, retrying in ${500} seconds`);
+        await sleep(wait);
+      }
+    }
+
+    if (this.killing) return;
+
+    if (retries > 5) {
+      throw new HatchetError('Could not start worker after 5 retries');
+    }
+  }
+}

package/hatchet/index.ts

@@ -0,0 +1,3 @@
+import { HatchetClient as Hatchet } from '@clients/hatchet-client';
+
+export default Hatchet;

package/hatchet/sdk.ts

@@ -0,0 +1,3 @@
+import { HatchetClient as Hatchet } from '@clients/hatchet-client';
+
+export default Hatchet;

package/hatchet/step.ts

@@ -0,0 +1,48 @@
+import HatchetError from '@util/errors/hatchet-error';
+import * as z from 'zod';
+
+export const CreateStepSchema = z.object({
+  name: z.string(),
+  parents: z.array(z.string()).optional(),
+});
+
+export type NextStep = { [key: string]: string };
+
+interface ContextData<T = unknown> {
+  input: T;
+  parents: Record<string, any>;
+  triggered_by_event: string;
+}
+
+export class Context<T = unknown> {
+  data: ContextData<T>;
+  constructor(payload: string) {
+    try {
+      this.data = JSON.parse(JSON.parse(payload));
+    } catch (e: any) {
+      throw new HatchetError(`Could not parse payload: ${e.message}`);
+    }
+  }
+
+  stepOutput(step: string): string {
+    if (!this.data.parents) {
+      throw new HatchetError('Step output not found');
+    }
+    if (!this.data.parents[step]) {
+      throw new HatchetError(`Step output for '${step}' not found`);
+    }
+    return this.data.parents[step];
+  }
+
+  triggeredByEvent(): boolean {
+    return this.data?.triggered_by_event === 'event';
+  }
+
+  workflowInput(): any {
+    return this.data?.input || {};
+  }
+}
+
+export interface CreateStep<T> extends z.infer<typeof CreateStepSchema> {
+  run: (ctx: Context) => Promise<NextStep> | NextStep | void;
+}