@hatchet-dev/typescript-sdk 1.34.0-alpha.2 → 1.34.1

package/v1/client/client.d.ts

@@ -19,7 +19,7 @@ import { BatchTaskFn } from '../task';
 import type { LegacyWorkflow } from '../../legacy/legacy-transformer';
 import { IHatchetClient } from './client.interface';
 import { CreateWorkerOpts, Worker } from './worker/worker';
-import { CELClient, CronClient, FiltersClient, LogsClient, MetricsClient, RatelimitsClient, RunsClient, ScheduleClient, TenantClient, WebhooksClient, WorkersClient, WorkflowsClient } from './features';
+import { CELClient, CronClient, FiltersClient, LogsClient, MetricsClient, RatelimitsClient, RunsClient, ScheduleClient, StreamsClient, TenantClient, WebhooksClient, WorkersClient, WorkflowsClient } from './features';
 import { InputType, OutputType, UnknownInputType, StrictWorkflowOutputType, Resolved } from '../types';
 import { AdminClient } from './admin';
 import { DurableContext } from './worker/context';
@@ -294,6 +294,13 @@ export declare class HatchetClient<GlobalInput extends Record<string, any> = {},
      * @returns A runs client instance
      */
     get runs(): RunsClient;
+    private _streams;
+    /**
+     * Get the streams client for publishing to and reading from durable,
+     * topic-based streams
+     * @returns A streams client instance
+     */
+    get streams(): StreamsClient;
     private _workflows;
     /**
      * Get the workflows client for creating and managing workflows

package/v1/client/client.js

@@ -405,6 +405,17 @@ class HatchetClient {
         }
         return this._runs;
     }
+    /**
+     * Get the streams client for publishing to and reading from durable,
+     * topic-based streams
+     * @returns A streams client instance
+     */
+    get streams() {
+        if (!this._streams) {
+            this._streams = new features_1.StreamsClient(this);
+        }
+        return this._streams;
+    }
     /**
      * Get the workflows client for creating and managing workflows
      * @returns A workflows client instance

package/v1/client/features/index.d.ts

@@ -10,3 +10,4 @@ export * from './filters';
 export * from './tenant';
 export * from './webhooks';
 export * from './cel';
+export * from './streams';

package/v1/client/features/index.js

@@ -26,3 +26,4 @@ __exportStar(require("./filters"), exports);
 __exportStar(require("./tenant"), exports);
 __exportStar(require("./webhooks"), exports);
 __exportStar(require("./cel"), exports);
+__exportStar(require("./streams"), exports);

package/v1/client/features/streams.d.ts

@@ -0,0 +1,47 @@
+import { HatchetClient } from '../client';
+export type StreamEvent = {
+    /** the message payload */
+    payload: Uint8Array;
+    /** resumes events() right after this event */
+    cursor: string;
+    /** when the message was stored */
+    createdAt?: Date;
+};
+export type StreamCallOptions = {
+    namespace?: string;
+    cursor?: string;
+    /** ends an events() iteration cleanly, without throwing */
+    signal?: AbortSignal;
+};
+/**
+ * StreamsClient publishes to and reads from durable topics. Unlike
+ * `runs.subscribeToStream`, topics are stored, independent of any run, and
+ * readers can resume from a cursor.
+ */
+export declare class StreamsClient {
+    private _config;
+    private _grpc;
+    private producers;
+    private publishChains;
+    constructor(client: HatchetClient);
+    private get grpc();
+    private publishOrdered;
+    /**
+     * Resolves once the message is stored. A client's messages to a topic are
+     * delivered in the order publish() was called.
+     * @param topic - the topic to publish to
+     * @param message - the message payload
+     * @param options - optional namespace override
+     */
+    publish(topic: string, message: Uint8Array | string, options?: StreamCallOptions): Promise<void>;
+    /**
+     * Yields topic's messages after options.cursor, or from the oldest retained.
+     * If the connection drops, it resubscribes from the last message yielded, so
+     * nothing is skipped or repeated. Ends when the server hangs up an idle
+     * subscription, options.signal aborts, or the caller stops iterating; throws
+     * on errors a retry can't fix, such as an expired cursor.
+     * @param topic - the topic to read from
+     * @param options - optional namespace override, resume cursor, and abort signal
+     */
+    events(topic: string, options?: StreamCallOptions): AsyncIterable<StreamEvent>;
+}

package/v1/client/features/streams.js

@@ -0,0 +1,210 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
+var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var g = generator.apply(thisArg, _arguments || []), i, q = [];
+    return i = Object.create((typeof AsyncIterator === "function" ? AsyncIterator : Object).prototype), verb("next"), verb("throw"), verb("return", awaitReturn), i[Symbol.asyncIterator] = function () { return this; }, i;
+    function awaitReturn(f) { return function (v) { return Promise.resolve(v).then(f, reject); }; }
+    function verb(n, f) { if (g[n]) { i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; if (f) i[n] = f(i[n]); } }
+    function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
+    function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
+    function fulfill(value) { resume("next", value); }
+    function reject(value) { resume("throw", value); }
+    function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamsClient = void 0;
+const crypto_1 = require("crypto");
+const nice_grpc_1 = require("nice-grpc");
+const grpc_helpers_1 = require("../../../util/grpc-helpers");
+const grpc_error_1 = require("../../../util/grpc-error");
+const sleep_1 = __importDefault(require("../../../util/sleep"));
+const streams_1 = require("../../../protoc/v1/streams");
+// the server stored nothing, so the seq is unused
+function publishRejectedBeforeStoring(err) {
+    const code = (0, grpc_error_1.getGrpcErrorCode)(err);
+    return (code === nice_grpc_1.Status.INVALID_ARGUMENT ||
+        code === nice_grpc_1.Status.RESOURCE_EXHAUSTED ||
+        code === nice_grpc_1.Status.UNAUTHENTICATED ||
+        code === nice_grpc_1.Status.PERMISSION_DENIED);
+}
+// retrying a subscribe that failed with these can't succeed
+const permanentSubscribeErrors = new Set([
+    nice_grpc_1.Status.INVALID_ARGUMENT,
+    nice_grpc_1.Status.PERMISSION_DENIED,
+    nice_grpc_1.Status.UNAUTHENTICATED,
+    nice_grpc_1.Status.OUT_OF_RANGE,
+    nice_grpc_1.Status.FAILED_PRECONDITION,
+    nice_grpc_1.Status.NOT_FOUND,
+]);
+const resubscribeMinDelayMs = 250;
+const resubscribeMaxDelayMs = 10000;
+function shouldResubscribe(err, attempt) {
+    const code = (0, grpc_error_1.getGrpcErrorCode)(err);
+    if (code === undefined || permanentSubscribeErrors.has(code)) {
+        return false;
+    }
+    // proxies answer 404 while an engine restarts, but on a first attempt it means no streams support
+    if (code === nice_grpc_1.Status.UNIMPLEMENTED) {
+        return attempt > 0;
+    }
+    return true;
+}
+/**
+ * StreamsClient publishes to and reads from durable topics. Unlike
+ * `runs.subscribeToStream`, topics are stored, independent of any run, and
+ * readers can resume from a cursor.
+ */
+class StreamsClient {
+    constructor(client) {
+        // per (namespace, topic); seq only advances after a publish succeeds
+        this.producers = new Map();
+        // serializes publishes per (namespace, topic) so two never share a seq; the
+        // stored promise always resolves, so one failure doesn't block later calls
+        this.publishChains = new Map();
+        this._config = client.config;
+    }
+    get grpc() {
+        if (!this._grpc) {
+            const { client } = (0, grpc_helpers_1.createGrpcClient)(this._config, streams_1.V1StreamsDefinition);
+            this._grpc = client;
+        }
+        return this._grpc;
+    }
+    publishOrdered(key_1, namespace_1, topic_1, payload_1) {
+        return __awaiter(this, arguments, void 0, function* (key, namespace, topic, payload, retryGap = true) {
+            let producer = this.producers.get(key);
+            if (!producer) {
+                producer = { producerId: (0, crypto_1.randomUUID)(), seq: 0 };
+                this.producers.set(key, producer);
+            }
+            try {
+                yield this.grpc.publish({
+                    namespace,
+                    topic,
+                    payload,
+                    producerId: producer.producerId,
+                    producerSeq: producer.seq,
+                });
+            }
+            catch (err) {
+                // it may still land, and a different payload under its seq would be dropped as a duplicate
+                if (!publishRejectedBeforeStoring(err)) {
+                    this.producers.set(key, { producerId: (0, crypto_1.randomUUID)(), seq: 0 });
+                }
+                // a gap stored nothing (e.g. the watermark passed cursor retention), so resend as the new producer
+                if (retryGap && (0, grpc_error_1.getGrpcErrorCode)(err) === nice_grpc_1.Status.FAILED_PRECONDITION) {
+                    return this.publishOrdered(key, namespace, topic, payload, false);
+                }
+                throw err;
+            }
+            producer.seq += 1;
+        });
+    }
+    /**
+     * Resolves once the message is stored. A client's messages to a topic are
+     * delivered in the order publish() was called.
+     * @param topic - the topic to publish to
+     * @param message - the message payload
+     * @param options - optional namespace override
+     */
+    publish(topic, message, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            const payload = typeof message === 'string' ? new TextEncoder().encode(message) : message;
+            const namespace = (_a = options === null || options === void 0 ? void 0 : options.namespace) !== null && _a !== void 0 ? _a : '';
+            const key = `${namespace}\u0000${topic}`;
+            const previous = (_b = this.publishChains.get(key)) !== null && _b !== void 0 ? _b : Promise.resolve();
+            const publishPromise = previous.then(() => this.publishOrdered(key, namespace, topic, payload));
+            this.publishChains.set(key, publishPromise.catch(() => { }));
+            return publishPromise;
+        });
+    }
+    /**
+     * Yields topic's messages after options.cursor, or from the oldest retained.
+     * If the connection drops, it resubscribes from the last message yielded, so
+     * nothing is skipped or repeated. Ends when the server hangs up an idle
+     * subscription, options.signal aborts, or the caller stops iterating; throws
+     * on errors a retry can't fix, such as an expired cursor.
+     * @param topic - the topic to read from
+     * @param options - optional namespace override, resume cursor, and abort signal
+     */
+    events(topic, options) {
+        return __asyncGenerator(this, arguments, function* events_1() {
+            var _a, e_1, _b, _c;
+            var _d;
+            const namespace = (_d = options === null || options === void 0 ? void 0 : options.namespace) !== null && _d !== void 0 ? _d : '';
+            const signal = options === null || options === void 0 ? void 0 : options.signal;
+            let { cursor } = options !== null && options !== void 0 ? options : {};
+            let failures = 0;
+            for (let attempt = 0;; attempt += 1) {
+                try {
+                    const stream = this.grpc.subscribe({ namespace, topic, cursor }, { signal });
+                    try {
+                        for (var _e = true, stream_1 = (e_1 = void 0, __asyncValues(stream)), stream_1_1; stream_1_1 = yield __await(stream_1.next()), _a = stream_1_1.done, !_a; _e = true) {
+                            _c = stream_1_1.value;
+                            _e = false;
+                            const msg = _c;
+                            if (msg.hangup) {
+                                return yield __await(void 0);
+                            }
+                            failures = 0;
+                            for (const entry of msg.entries) {
+                                ({ cursor } = entry);
+                                yield yield __await({
+                                    payload: entry.payload,
+                                    cursor: entry.cursor,
+                                    createdAt: entry.createdAt,
+                                });
+                            }
+                        }
+                    }
+                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                    finally {
+                        try {
+                            if (!_e && !_a && (_b = stream_1.return)) yield __await(_b.call(stream_1));
+                        }
+                        finally { if (e_1) throw e_1.error; }
+                    }
+                    // ending without a hangup means the engine went away, e.g. it is shutting down
+                }
+                catch (err) {
+                    if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
+                        return yield __await(void 0);
+                    }
+                    if (!shouldResubscribe(err, attempt)) {
+                        throw err;
+                    }
+                }
+                const delayMs = Math.min(resubscribeMinDelayMs * 2 ** failures, resubscribeMaxDelayMs);
+                failures += 1;
+                try {
+                    yield __await((0, sleep_1.default)(delayMs * (0.5 + Math.random() / 2), signal));
+                }
+                catch (_f) {
+                    return yield __await(void 0);
+                }
+            }
+        });
+    }
+}
+exports.StreamsClient = StreamsClient;

package/v1/client/worker/worker-internal.d.ts

@@ -45,6 +45,7 @@ export declare class InternalWorker {
     enableHealthServer: boolean;
     private healthServer;
     private status;
+    private shutdownPromise;
     constructor(client: HatchetClient, options: {
         name: string;
         handleKill?: boolean;
@@ -96,6 +97,7 @@ export declare class InternalWorker {
     handleCancelStepRun(action: Action): Promise<void>;
     stop(): Promise<void>;
     exitGracefully(handleKill: boolean): Promise<void>;
+    private shutdown;
     /**
      * Creates an action listener by registering the worker with the dispatcher.
      * Override in subclasses to change registration behavior (e.g. legacy engines).

package/v1/client/worker/worker-internal.js

@@ -628,6 +628,18 @@ class InternalWorker {
     exitGracefully(handleKill) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
+            // Signals and an explicit stop must await the same eviction and listener teardown.
+            (_a = this.shutdownPromise) !== null && _a !== void 0 ? _a : (this.shutdownPromise = this.shutdown());
+            yield this.shutdownPromise;
+            if (handleKill) {
+                this.logger.info('Exiting hatchet worker...');
+                process.exit(0);
+            }
+        });
+    }
+    shutdown() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
             this.killing = true;
             this.setStatus(health_server_1.workerStatus.UNHEALTHY);
             this.logger.info('Starting to exit...');
@@ -658,16 +670,19 @@ class InternalWorker {
             catch (e) {
                 this.logger.error(`Could not stop durable listener: ${e.message}`);
             }
+            (_a = this.listener) === null || _a === void 0 ? void 0 : _a.stopStream();
+            this.logger.info('Gracefully exiting hatchet worker, running tasks will attempt to finish...');
+            // attempt to wait for futures to finish
+            yield Promise.all(Object.values(this.futures).map(({ promise }) => promise));
+            this.logger.info('Successfully finished pending tasks.');
+            // Unregistering stops the heartbeat, so it has to wait until the running tasks are done:
+            // otherwise the engine sees a dead worker and reassigns tasks that are about to complete.
             try {
-                yield ((_a = this.listener) === null || _a === void 0 ? void 0 : _a.unregister());
+                yield ((_b = this.listener) === null || _b === void 0 ? void 0 : _b.unregister());
             }
             catch (e) {
                 this.logger.error(`Could not unregister listener: ${e.message}`);
             }
-            this.logger.info('Gracefully exiting hatchet worker, running tasks will attempt to finish...');
-            // attempt to wait for futures to finish
-            yield Promise.all(Object.values(this.futures).map(({ promise }) => promise));
-            this.logger.info('Successfully finished pending tasks.');
             if (this.healthServer) {
                 try {
                     yield this.healthServer.stop();
@@ -676,10 +691,6 @@ class InternalWorker {
                     this.logger.error(`Could not stop health server: ${e.message}`);
                 }
             }
-            if (handleKill) {
-                this.logger.info('Exiting hatchet worker...');
-                process.exit(0);
-            }
         });
     }
     /**
@@ -700,10 +711,18 @@ class InternalWorker {
     start() {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, e_1, _b, _c;
+            if (this.killing) {
+                return;
+            }
             this.setStatus(health_server_1.workerStatus.STARTING);
             if (this.healthServer) {
                 try {
                     yield this.healthServer.start();
+                    if (this.killing) {
+                        yield this.shutdownPromise;
+                        yield this.healthServer.stop();
+                        return;
+                    }
                 }
                 catch (e) {
                     this.logger.error(`Could not start health server: ${e.message}`);
@@ -713,11 +732,17 @@ class InternalWorker {
             }
             // ensure all workflows are registered
             yield Promise.all(this.registeredWorkflowPromises);
-            if (Object.keys(this.action_registry).length === 0) {
+            if (this.killing || Object.keys(this.action_registry).length === 0) {
                 return;
             }
             try {
-                this.listener = yield this.createListener();
+                const listener = yield this.createListener();
+                if (this.killing) {
+                    // Shutdown cannot clean up a listener that registration has not returned yet.
+                    yield listener.unregister();
+                    return;
+                }
+                this.listener = listener;
                 this.workerId = this.listener.workerId;
                 this.setStatus(health_server_1.workerStatus.HEALTHY);
                 const generator = this.listener.actions();

package/v1/examples/durable-streams/publish-latency.d.ts

@@ -0,0 +1,15 @@
+import { HatchetClient } from '../..';
+export type PublishLatencyStats = {
+    count: number;
+    avgMs: number;
+    p50Ms: number;
+    p99Ms: number;
+    maxMs: number;
+    throughputPerSec: number;
+};
+/**
+ * Times `count` sequential publishes, after `warmupCount` unmeasured ones
+ * absorb connection setup and topic creation.
+ */
+export declare function measurePublishLatency(hatchet: HatchetClient, topic: string, payload: Uint8Array | string, count: number, warmupCount?: number): Promise<PublishLatencyStats>;
+export declare function formatPublishLatency(stats: PublishLatencyStats): string;

package/v1/examples/durable-streams/publish-latency.js

@@ -0,0 +1,50 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.measurePublishLatency = measurePublishLatency;
+exports.formatPublishLatency = formatPublishLatency;
+const perf_hooks_1 = require("perf_hooks");
+function percentile(sorted, p) {
+    return sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))];
+}
+/**
+ * Times `count` sequential publishes, after `warmupCount` unmeasured ones
+ * absorb connection setup and topic creation.
+ */
+function measurePublishLatency(hatchet_1, topic_1, payload_1, count_1) {
+    return __awaiter(this, arguments, void 0, function* (hatchet, topic, payload, count, warmupCount = 5) {
+        for (let i = 0; i < warmupCount; i += 1) {
+            yield hatchet.streams.publish(topic, payload);
+        }
+        const durationsMs = [];
+        const start = perf_hooks_1.performance.now();
+        for (let i = 0; i < count; i += 1) {
+            const before = perf_hooks_1.performance.now();
+            yield hatchet.streams.publish(topic, payload);
+            durationsMs.push(perf_hooks_1.performance.now() - before);
+        }
+        const totalMs = perf_hooks_1.performance.now() - start;
+        const sorted = [...durationsMs].sort((a, b) => a - b);
+        return {
+            count,
+            avgMs: durationsMs.reduce((sum, d) => sum + d, 0) / count,
+            p50Ms: percentile(sorted, 0.5),
+            p99Ms: percentile(sorted, 0.99),
+            maxMs: sorted[sorted.length - 1],
+            throughputPerSec: (count / totalMs) * 1000,
+        };
+    });
+}
+function formatPublishLatency(stats) {
+    const fmt = (ms) => ms.toFixed(2);
+    return (`avg=${fmt(stats.avgMs)}ms p50=${fmt(stats.p50Ms)}ms p99=${fmt(stats.p99Ms)}ms ` +
+        `max=${fmt(stats.maxMs)}ms throughput=${fmt(stats.throughputPerSec)}/s`);
+}

package/v1/examples/durable_callback_ordering/workflow.js

@@ -69,11 +69,15 @@ exports.callbackOrderingMid = hatchet_client_1.hatchet.durableTask({
         };
     }),
 });
+// The root is never evicted. The ordering property under test lives in the
+// mids, and the worker evicts one waiting run per server round trip, oldest
+// wait first: evictable roots register before every mid and would consume the
+// sweep for as long as it takes to ack all of them before any mid is reached.
 exports.callbackOrderingRoot = hatchet_client_1.hatchet.durableTask({
     name: `${WORKFLOW_PREFIX}-root`,
     executionTimeout: '10m',
     retries: 0,
-    evictionPolicy: EVICTION_POLICY,
+    evictionPolicy: { ttl: undefined, allowCapacityEviction: false, priority: 0 },
     fn: (input, ctx) => __awaiter(void 0, void 0, void 0, function* () {
         var _a;
         const durables = (_a = input.durables) !== null && _a !== void 0 ? _a : exports.ROOT_DEFAULTS.durables;

package/version.d.ts

@@ -1 +1 @@
-export declare const HATCHET_VERSION = "1.34.0-alpha.2";
+export declare const HATCHET_VERSION = "1.34.1";

package/version.js

@@ -1,4 +1,4 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.HATCHET_VERSION = void 0;
-exports.HATCHET_VERSION = '1.34.0-alpha.2';
+exports.HATCHET_VERSION = '1.34.1';