@hatchet-dev/typescript-sdk 1.16.0 → 1.17.0

package/v1/client/worker/eviction/eviction-cache.js

@@ -0,0 +1,139 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DurableEvictionCache = exports.EvictionCause = void 0;
+exports.buildEvictionReason = buildEvictionReason;
+const duration_1 = require("../../duration");
+var EvictionCause;
+(function (EvictionCause) {
+    EvictionCause["TTL_EXCEEDED"] = "ttl_exceeded";
+    EvictionCause["CAPACITY_PRESSURE"] = "capacity_pressure";
+    EvictionCause["WORKER_SHUTDOWN"] = "worker_shutdown";
+})(EvictionCause || (exports.EvictionCause = EvictionCause = {}));
+class DurableEvictionCache {
+    constructor() {
+        this._runs = new Map();
+    }
+    registerRun(key, taskRunExternalId, invocationCount, now, evictionPolicy) {
+        this._runs.set(key, {
+            key,
+            taskRunExternalId,
+            invocationCount,
+            evictionPolicy,
+            registeredAt: now,
+            waitingSince: undefined,
+            waitKind: undefined,
+            waitResourceId: undefined,
+            _waitCount: 0,
+            evictionReason: undefined,
+        });
+    }
+    unregisterRun(key) {
+        this._runs.delete(key);
+    }
+    get(key) {
+        return this._runs.get(key);
+    }
+    getAllWaiting() {
+        return [...this._runs.values()].filter((r) => r._waitCount > 0);
+    }
+    findKeyByTaskRunExternalId(taskRunExternalId) {
+        for (const [key, rec] of this._runs) {
+            if (rec.taskRunExternalId === taskRunExternalId)
+                return key;
+        }
+        return undefined;
+    }
+    markWaiting(key, now, waitKind, resourceId) {
+        const rec = this._runs.get(key);
+        if (!rec)
+            return;
+        rec._waitCount += 1;
+        if (rec._waitCount === 1) {
+            rec.waitingSince = now;
+        }
+        rec.waitKind = waitKind;
+        rec.waitResourceId = resourceId;
+    }
+    markActive(key) {
+        const rec = this._runs.get(key);
+        if (!rec)
+            return;
+        rec._waitCount = Math.max(0, rec._waitCount - 1);
+        if (rec._waitCount === 0) {
+            rec.waitingSince = undefined;
+            rec.waitKind = undefined;
+            rec.waitResourceId = undefined;
+        }
+    }
+    selectEvictionCandidate(now, durableSlots, reserveSlots, minWaitForCapacityEvictionMs) {
+        const waiting = [...this._runs.values()].filter((r) => r._waitCount > 0 && r.evictionPolicy !== undefined);
+        if (waiting.length === 0)
+            return undefined;
+        const ttlEligible = waiting.filter((r) => {
+            var _a;
+            const ttl = (_a = r.evictionPolicy) === null || _a === void 0 ? void 0 : _a.ttl;
+            if (!ttl || !r.waitingSince)
+                return false;
+            return now - r.waitingSince >= (0, duration_1.durationToMs)(ttl);
+        });
+        if (ttlEligible.length > 0) {
+            ttlEligible.sort((a, b) => {
+                var _a, _b, _c, _d, _e, _f;
+                return ((_b = (_a = a.evictionPolicy) === null || _a === void 0 ? void 0 : _a.priority) !== null && _b !== void 0 ? _b : 0) - ((_d = (_c = b.evictionPolicy) === null || _c === void 0 ? void 0 : _c.priority) !== null && _d !== void 0 ? _d : 0) ||
+                    ((_e = a.waitingSince) !== null && _e !== void 0 ? _e : now) - ((_f = b.waitingSince) !== null && _f !== void 0 ? _f : now);
+            });
+            const [chosen] = ttlEligible;
+            chosen.evictionReason = buildEvictionReason(EvictionCause.TTL_EXCEEDED, chosen);
+            return chosen.key;
+        }
+        if (!this._hasCapacityPressure(durableSlots, reserveSlots, waiting.length)) {
+            return undefined;
+        }
+        const capacityCandidates = waiting.filter((r) => {
+            var _a;
+            return ((_a = r.evictionPolicy) === null || _a === void 0 ? void 0 : _a.allowCapacityEviction) !== false &&
+                r.waitingSince !== undefined &&
+                now - r.waitingSince >= minWaitForCapacityEvictionMs;
+        });
+        if (capacityCandidates.length === 0)
+            return undefined;
+        capacityCandidates.sort((a, b) => {
+            var _a, _b, _c, _d, _e, _f;
+            return ((_b = (_a = a.evictionPolicy) === null || _a === void 0 ? void 0 : _a.priority) !== null && _b !== void 0 ? _b : 0) - ((_d = (_c = b.evictionPolicy) === null || _c === void 0 ? void 0 : _c.priority) !== null && _d !== void 0 ? _d : 0) ||
+                ((_e = a.waitingSince) !== null && _e !== void 0 ? _e : now) - ((_f = b.waitingSince) !== null && _f !== void 0 ? _f : now);
+        });
+        const [chosen] = capacityCandidates;
+        chosen.evictionReason = buildEvictionReason(EvictionCause.CAPACITY_PRESSURE, chosen);
+        return chosen.key;
+    }
+    _hasCapacityPressure(durableSlots, reserveSlots, waitingCount) {
+        if (durableSlots <= 0)
+            return false;
+        const maxWaiting = durableSlots - reserveSlots;
+        if (maxWaiting <= 0)
+            return false;
+        return waitingCount >= maxWaiting;
+    }
+}
+exports.DurableEvictionCache = DurableEvictionCache;
+function buildEvictionReason(cause, rec) {
+    var _a;
+    let waitDesc = rec.waitKind || 'unknown';
+    if (rec.waitResourceId) {
+        waitDesc = `${waitDesc}(${rec.waitResourceId})`;
+    }
+    switch (cause) {
+        case EvictionCause.TTL_EXCEEDED: {
+            const ttlStr = ((_a = rec.evictionPolicy) === null || _a === void 0 ? void 0 : _a.ttl) ? ` (${rec.evictionPolicy.ttl})` : '';
+            return `Wait TTL${ttlStr} exceeded while waiting on ${waitDesc}`;
+        }
+        case EvictionCause.CAPACITY_PRESSURE:
+            return `Worker at capacity while waiting on ${waitDesc}`;
+        case EvictionCause.WORKER_SHUTDOWN:
+            return `Worker shutdown while waiting on ${waitDesc}`;
+        default: {
+            const _exhaustive = cause;
+            throw new Error(`Unknown eviction cause: ${_exhaustive}`);
+        }
+    }
+}

package/v1/client/worker/eviction/eviction-manager.d.ts

@@ -0,0 +1,42 @@
+import { Logger } from '../../../../util/logger';
+import { EvictionPolicy } from './eviction-policy';
+import { ActionKey, DurableEvictionCache, DurableRunRecord } from './eviction-cache';
+export interface DurableEvictionConfig {
+    /** How often we try selecting an eviction candidate. Default: 1000ms */
+    checkIntervalMs?: number;
+    /** How many slots to reserve from capacity-based eviction decisions. Default: 0 */
+    reserveSlots?: number;
+    /** Avoid immediately evicting runs that just entered a wait. Default: 10000ms */
+    minWaitForCapacityEvictionMs?: number;
+}
+export declare const DEFAULT_DURABLE_EVICTION_CONFIG: Required<DurableEvictionConfig>;
+export declare class DurableEvictionManager {
+    private _durableSlots;
+    private _cancelLocal;
+    private _requestEvictionWithAck;
+    private _config;
+    private _cache;
+    private _logger;
+    private _timer;
+    private _ticking;
+    constructor(opts: {
+        durableSlots: number;
+        cancelLocal: (key: ActionKey) => void;
+        requestEvictionWithAck: (key: ActionKey, rec: DurableRunRecord) => Promise<void>;
+        config?: DurableEvictionConfig;
+        cache?: DurableEvictionCache;
+        logger: Logger;
+    });
+    get cache(): DurableEvictionCache;
+    start(): void;
+    stop(): void;
+    registerRun(key: ActionKey, taskRunExternalId: string, invocationCount: number, evictionPolicy: EvictionPolicy | undefined): void;
+    unregisterRun(key: ActionKey): void;
+    markWaiting(key: ActionKey, waitKind: string, resourceId: string): void;
+    markActive(key: ActionKey): void;
+    private _evictRun;
+    private _tickSafe;
+    private _tick;
+    handleServerEviction(taskRunExternalId: string, invocationCount: number): void;
+    evictAllWaiting(): Promise<number>;
+}

package/v1/client/worker/eviction/eviction-manager.js

@@ -0,0 +1,132 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DurableEvictionManager = exports.DEFAULT_DURABLE_EVICTION_CONFIG = void 0;
+const eviction_cache_1 = require("./eviction-cache");
+const hatchet_error_1 = require("../../../../util/errors/hatchet-error");
+exports.DEFAULT_DURABLE_EVICTION_CONFIG = {
+    checkIntervalMs: 1000,
+    reserveSlots: 0,
+    minWaitForCapacityEvictionMs: 10000,
+};
+class DurableEvictionManager {
+    constructor(opts) {
+        this._ticking = false;
+        this._durableSlots = opts.durableSlots;
+        this._cancelLocal = opts.cancelLocal;
+        this._requestEvictionWithAck = opts.requestEvictionWithAck;
+        this._config = Object.assign(Object.assign({}, exports.DEFAULT_DURABLE_EVICTION_CONFIG), opts.config);
+        this._cache = opts.cache || new eviction_cache_1.DurableEvictionCache();
+        this._logger = opts.logger;
+    }
+    get cache() {
+        return this._cache;
+    }
+    start() {
+        if (this._timer)
+            return;
+        this._timer = setInterval(() => this._tickSafe(), this._config.checkIntervalMs);
+    }
+    stop() {
+        if (this._timer) {
+            clearInterval(this._timer);
+            this._timer = undefined;
+        }
+    }
+    registerRun(key, taskRunExternalId, invocationCount, evictionPolicy) {
+        this._cache.registerRun(key, taskRunExternalId, invocationCount, Date.now(), evictionPolicy);
+    }
+    unregisterRun(key) {
+        this._cache.unregisterRun(key);
+    }
+    markWaiting(key, waitKind, resourceId) {
+        this._cache.markWaiting(key, Date.now(), waitKind, resourceId);
+    }
+    markActive(key) {
+        this._cache.markActive(key);
+    }
+    _evictRun(key) {
+        this._cancelLocal(key);
+        this.unregisterRun(key);
+    }
+    _tickSafe() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this._ticking)
+                return;
+            this._ticking = true;
+            try {
+                yield this._tick();
+            }
+            catch (err) {
+                this._logger.error(`DurableEvictionManager: error in eviction loop: ${(0, hatchet_error_1.getErrorMessage)(err)}`);
+            }
+            finally {
+                this._ticking = false;
+            }
+        });
+    }
+    _tick() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const evictedThisTick = new Set();
+            while (true) {
+                const key = this._cache.selectEvictionCandidate(Date.now(), this._durableSlots, this._config.reserveSlots, this._config.minWaitForCapacityEvictionMs);
+                if (!key)
+                    return;
+                if (evictedThisTick.has(key))
+                    return;
+                evictedThisTick.add(key);
+                const rec = this._cache.get(key);
+                if (!rec || !rec.evictionPolicy)
+                    continue;
+                this._logger.debug(`DurableEvictionManager: evicting task_run_external_id=${rec.taskRunExternalId} ` +
+                    `wait_kind=${rec.waitKind} resource_id=${rec.waitResourceId}`);
+                yield this._requestEvictionWithAck(key, rec);
+                this._evictRun(key);
+            }
+        });
+    }
+    handleServerEviction(taskRunExternalId, invocationCount) {
+        const key = this._cache.findKeyByTaskRunExternalId(taskRunExternalId);
+        if (!key)
+            return;
+        const rec = this._cache.get(key);
+        if (rec && rec.invocationCount !== invocationCount)
+            return;
+        this._logger.info(`DurableEvictionManager: server-initiated eviction for task_run_external_id=${taskRunExternalId} invocation_count=${invocationCount}`);
+        this._evictRun(key);
+    }
+    evictAllWaiting() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.stop();
+            const waiting = this._cache.getAllWaiting();
+            let evicted = 0;
+            for (const rec of waiting) {
+                rec.evictionReason = (0, eviction_cache_1.buildEvictionReason)(eviction_cache_1.EvictionCause.WORKER_SHUTDOWN, rec);
+                this._logger.debug(`DurableEvictionManager: shutdown-evicting task_run_external_id=${rec.taskRunExternalId} ` +
+                    `wait_kind=${rec.waitKind}`);
+                try {
+                    yield this._requestEvictionWithAck(rec.key, rec);
+                }
+                catch (err) {
+                    this._logger.error(`DurableEvictionManager: failed to send eviction for ` +
+                        `task_run_external_id=${rec.taskRunExternalId}: ${(0, hatchet_error_1.getErrorMessage)(err)}`);
+                }
+                // Always cancel locally even if the server ACK failed, so the
+                // future settles and exitGracefully doesn't hang.
+                // This will get resolved by the reassignment of the task.
+                this._evictRun(rec.key);
+                evicted++;
+            }
+            return evicted;
+        });
+    }
+}
+exports.DurableEvictionManager = DurableEvictionManager;

package/v1/client/worker/eviction/eviction-policy.d.ts

@@ -0,0 +1,19 @@
+import { Duration } from '../../duration';
+export type EvictionPolicy = {
+    /**
+     * Maximum continuous waiting duration before TTL-eligible eviction.
+     * `undefined` means no TTL-based eviction.
+     */
+    ttl?: Duration;
+    /**
+     * Whether this task may be evicted under durable-slot pressure.
+     * @default true
+     */
+    allowCapacityEviction?: boolean;
+    /**
+     * Lower values are evicted first when multiple candidates exist.
+     * @default 0
+     */
+    priority?: number;
+};
+export declare const DEFAULT_DURABLE_TASK_EVICTION_POLICY: EvictionPolicy;

package/v1/client/worker/eviction/eviction-policy.js

@@ -0,0 +1,8 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_DURABLE_TASK_EVICTION_POLICY = void 0;
+exports.DEFAULT_DURABLE_TASK_EVICTION_POLICY = {
+    ttl: '15m',
+    allowCapacityEviction: true,
+    priority: 0,
+};

package/v1/client/worker/eviction/index.d.ts

@@ -0,0 +1,3 @@
+export { EvictionPolicy, DEFAULT_DURABLE_TASK_EVICTION_POLICY } from './eviction-policy';
+export { DurableEvictionCache, ActionKey, DurableRunRecord, EvictionCause } from './eviction-cache';
+export { DurableEvictionManager, DurableEvictionConfig, DEFAULT_DURABLE_EVICTION_CONFIG, } from './eviction-manager';

package/v1/client/worker/eviction/index.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_DURABLE_EVICTION_CONFIG = exports.DurableEvictionManager = exports.EvictionCause = exports.DurableEvictionCache = exports.DEFAULT_DURABLE_TASK_EVICTION_POLICY = void 0;
+var eviction_policy_1 = require("./eviction-policy");
+Object.defineProperty(exports, "DEFAULT_DURABLE_TASK_EVICTION_POLICY", { enumerable: true, get: function () { return eviction_policy_1.DEFAULT_DURABLE_TASK_EVICTION_POLICY; } });
+var eviction_cache_1 = require("./eviction-cache");
+Object.defineProperty(exports, "DurableEvictionCache", { enumerable: true, get: function () { return eviction_cache_1.DurableEvictionCache; } });
+Object.defineProperty(exports, "EvictionCause", { enumerable: true, get: function () { return eviction_cache_1.EvictionCause; } });
+var eviction_manager_1 = require("./eviction-manager");
+Object.defineProperty(exports, "DurableEvictionManager", { enumerable: true, get: function () { return eviction_manager_1.DurableEvictionManager; } });
+Object.defineProperty(exports, "DEFAULT_DURABLE_EVICTION_CONFIG", { enumerable: true, get: function () { return eviction_manager_1.DEFAULT_DURABLE_EVICTION_CONFIG; } });

package/v1/client/worker/worker-internal.d.ts

@@ -1,4 +1,4 @@
-import { Action, ActionListener } from '../../../clients/dispatcher/action-listener';
+import { Action, ActionKey, ActionListener } from '../../../clients/dispatcher/action-listener';
 import { StepActionEvent, StepActionEventType, GroupKeyActionEvent, GroupKeyActionEventType } from '../../../protoc/dispatcher';
 import HatchetPromise from '../../../util/hatchet-promise/hatchet-promise';
 import { CreateStepRateLimit } from '../../../protoc/workflows';
@@ -6,8 +6,11 @@ import { Logger } from '../../../util/logger';
 import { BaseWorkflowDeclaration, WorkflowDefinition, HatchetClient } from '../..';
 import { CreateWorkflowTaskOpts } from '../../task';
 import { WorkerLabels } from '../../../clients/dispatcher/dispatcher-client';
+import { Duration } from '../duration';
 import { Context } from './context';
 import { SlotConfig } from '../../slot-types';
+import { DurableEvictionManager } from './eviction/eviction-manager';
+import { EvictionPolicy } from './eviction/eviction-policy';
 export type ActionRegistry = Record<Action['actionId'], Function>;
 export interface WorkerOpts {
     name: string;
@@ -25,13 +28,17 @@ export declare class InternalWorker {
     killing: boolean;
     handle_kill: boolean;
     action_registry: ActionRegistry;
+    durable_action_set: Set<string>;
+    eviction_policies: Map<string, EvictionPolicy | undefined>;
+    evictionManager: DurableEvictionManager | undefined;
     workflow_registry: Array<WorkflowDefinition>;
     listener: ActionListener | undefined;
-    futures: Record<Action['taskRunExternalId'], HatchetPromise<any>>;
-    contexts: Record<Action['taskRunExternalId'], Context<any, any>>;
+    futures: Record<ActionKey, HatchetPromise<any>>;
+    contexts: Record<ActionKey, Context<any, any>>;
     slots?: number;
     durableSlots?: number;
     slotConfig: SlotConfig;
+    engineVersion: string | undefined;
     logger: Logger;
     registeredWorkflowPromises: Array<Promise<any>>;
     labels: WorkerLabels;
@@ -55,8 +62,9 @@ export declare class InternalWorker {
     registerDurableActions(workflow: WorkflowDefinition): void;
     private registerActions;
     registerWorkflow(initWorkflow: BaseWorkflowDeclaration<any, any>, durable?: boolean): Promise<void>;
+    private ensureEvictionManager;
+    private cleanupRun;
     handleStartStepRun(action: Action): Promise<void>;
-    handleStartGroupKeyRun(action: Action): Promise<void>;
     getStepActionEvent(action: Action, eventType: StepActionEventType, shouldNotRetry: boolean, payload?: any, retryCount?: number): StepActionEvent;
     getGroupKeyActionEvent(action: Action, eventType: GroupKeyActionEventType, payload?: any): GroupKeyActionEvent;
     handleCancelStepRun(action: Action): Promise<void>;
@@ -72,3 +80,14 @@ export declare class InternalWorker {
     upsertLabels(labels: WorkerLabels): Promise<WorkerLabels>;
 }
 export declare function mapRateLimitPb(limits: CreateWorkflowTaskOpts<any, any>['rateLimits']): CreateStepRateLimit[];
+export declare function resolveExecutionTimeout(task: {
+    executionTimeout?: Duration;
+    timeout?: Duration;
+}, workflowDefaults?: {
+    executionTimeout?: Duration;
+}): string;
+export declare function resolveScheduleTimeout(task: {
+    scheduleTimeout?: Duration;
+}, workflowDefaults?: {
+    scheduleTimeout?: Duration;
+}): string | undefined;