@hatchet-dev/typescript-sdk 1.27.0 → 1.28.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/clients/dispatcher/action-listener.js +11 -2
- package/clients/dispatcher/dispatcher-client.d.ts +6 -1
- package/clients/dispatcher/dispatcher-client.js +14 -0
- package/clients/dispatcher/heartbeat/heartbeat-controller.d.ts +13 -4
- package/clients/dispatcher/heartbeat/heartbeat-controller.js +11 -1
- package/clients/dispatcher/heartbeat/heartbeat-severity.d.ts +3 -0
- package/clients/dispatcher/heartbeat/heartbeat-severity.js +11 -0
- package/clients/dispatcher/heartbeat/heartbeat-worker.js +11 -4
- package/clients/dispatcher/listener-severity.d.ts +3 -0
- package/clients/dispatcher/listener-severity.js +11 -0
- package/clients/listeners/durable-listener/durable-listener-client.d.ts +1 -0
- package/clients/listeners/durable-listener/durable-listener-client.js +20 -2
- package/clients/rest/generated/Api.d.ts +2 -0
- package/clients/rest/generated/data-contracts.d.ts +2 -0
- package/clients/rest/generated/data-contracts.js +2 -0
- package/package.json +3 -3
- package/protoc/dispatcher/dispatcher.d.ts +67 -0
- package/protoc/dispatcher/dispatcher.js +495 -3
- package/protoc/events/events.js +1 -1
- package/protoc/google/protobuf/any.js +1 -1
- package/protoc/google/protobuf/timestamp.js +1 -1
- package/protoc/google/rpc/status.js +1 -1
- package/protoc/v1/dispatcher.js +1 -1
- package/protoc/v1/shared/condition.js +1 -1
- package/protoc/v1/shared/trigger.js +1 -1
- package/protoc/v1/workflows.d.ts +24 -7
- package/protoc/v1/workflows.js +218 -40
- package/protoc/workflows/workflows.js +1 -1
- package/util/failure-severity.d.ts +2 -0
- package/util/failure-severity.js +15 -0
- package/util/grpc-error.d.ts +6 -0
- package/util/grpc-error.js +11 -0
- package/util/workflow-run-ref.js +23 -9
- package/v1/client/client.d.ts +26 -1
- package/v1/client/client.js +6 -0
- package/v1/client/worker/context.js +23 -3
- package/v1/client/worker/worker-internal.d.ts +22 -1
- package/v1/client/worker/worker-internal.js +167 -2
- package/v1/declaration.d.ts +62 -2
- package/v1/declaration.js +56 -0
- package/v1/examples/batch_assign/workflow.d.ts +62 -0
- package/v1/examples/batch_assign/workflow.js +161 -0
- package/v1/examples/e2e-worker.js +35 -21
- package/v1/task.d.ts +50 -1
- package/version.d.ts +1 -1
- package/version.js +1 -1
|
@@ -29,6 +29,7 @@ const parse_1 = require("../../../util/parse");
|
|
|
29
29
|
const conditions_1 = require("../../conditions");
|
|
30
30
|
const transformer_1 = require("../../conditions/transformer");
|
|
31
31
|
const condition_1 = require("../../../protoc/v1/shared/condition");
|
|
32
|
+
const dispatcher_1 = require("../../../protoc/dispatcher");
|
|
32
33
|
const apply_namespace_1 = require("../../../util/apply-namespace");
|
|
33
34
|
const abort_error_1 = require("../../../util/abort-error");
|
|
34
35
|
const parent_run_context_vars_1 = require("../../parent-run-context-vars");
|
|
@@ -135,9 +136,28 @@ class Context {
|
|
|
135
136
|
}
|
|
136
137
|
cancel() {
|
|
137
138
|
return __awaiter(this, void 0, void 0, function* () {
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
139
|
+
var _a, _b;
|
|
140
|
+
if (this.action.batchId) {
|
|
141
|
+
// Batch tasks share one context across every buffered member, so there is no single
|
|
142
|
+
// task-run id to cancel — cancel every member of the batch instead. `this.data` is
|
|
143
|
+
// the raw batch-items map for a START_BATCH action, keyed by each member's
|
|
144
|
+
// task-run external id.
|
|
145
|
+
const memberIds = Object.keys((_a = this.data) !== null && _a !== void 0 ? _a : {});
|
|
146
|
+
yield this.v1.dispatcher.sendBatchActionEvent({
|
|
147
|
+
workerId: (_b = this.worker.id()) !== null && _b !== void 0 ? _b : '',
|
|
148
|
+
jobId: this.action.jobId,
|
|
149
|
+
actionId: this.action.actionId,
|
|
150
|
+
batchId: this.action.batchId,
|
|
151
|
+
eventTimestamp: new Date(),
|
|
152
|
+
eventType: dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_CANCELLED,
|
|
153
|
+
items: memberIds.map((id) => ({ taskRunExternalId: id, eventPayload: '' })),
|
|
154
|
+
});
|
|
155
|
+
}
|
|
156
|
+
else {
|
|
157
|
+
yield this.v1.runs.cancel({
|
|
158
|
+
ids: [this.action.taskRunExternalId],
|
|
159
|
+
});
|
|
160
|
+
}
|
|
141
161
|
// optimistically abort the run
|
|
142
162
|
this.controller.abort();
|
|
143
163
|
});
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { Action, ActionKey, ActionListener } from '../../../clients/dispatcher/action-listener';
|
|
2
|
-
import { StepActionEvent, StepActionEventType, GroupKeyActionEvent, GroupKeyActionEventType } from '../../../protoc/dispatcher';
|
|
2
|
+
import { StepActionEvent, StepActionEventType, GroupKeyActionEvent, GroupKeyActionEventType, BatchActionEvent } from '../../../protoc/dispatcher';
|
|
3
3
|
import HatchetPromise from '../../../util/hatchet-promise/hatchet-promise';
|
|
4
4
|
import { CreateStepRateLimit } from '../../../protoc/workflows';
|
|
5
5
|
import { Logger } from '../../../util/logger';
|
|
6
6
|
import { BaseWorkflowDeclaration, WorkflowDefinition, HatchetClient } from '../..';
|
|
7
|
+
import { TaskBatchConfig } from '../../../protoc/v1/workflows';
|
|
7
8
|
import { CreateWorkflowTaskOpts } from '../../task';
|
|
8
9
|
import { WorkerLabels } from '../../../clients/dispatcher/dispatcher-client';
|
|
9
10
|
import { Duration } from '../duration';
|
|
@@ -70,6 +71,25 @@ export declare class InternalWorker {
|
|
|
70
71
|
*/
|
|
71
72
|
handleStartStepRun(action: Action): Promise<Error | undefined>;
|
|
72
73
|
getStepActionEvent(action: Action, eventType: StepActionEventType, shouldNotRetry: boolean, payload?: any, retryCount?: number): StepActionEvent;
|
|
74
|
+
/**
|
|
75
|
+
* Handles a START_BATCH action: invokes the registered batch task handler once with
|
|
76
|
+
* every buffered item, then reports completion/failure back per-member (or, for
|
|
77
|
+
* broadcast batch tasks, the same result for every member) via sendBatchActionEvent.
|
|
78
|
+
*/
|
|
79
|
+
handleStartBatch(action: Action): Promise<Error | undefined>;
|
|
80
|
+
/**
|
|
81
|
+
* Reports the result of a successful batch handler invocation. result is expected to be
|
|
82
|
+
* a Record<string, any> keyed by batch member id; each member's output is serialized
|
|
83
|
+
* independently so that one member's serialization failure does not fail the whole
|
|
84
|
+
* batch.
|
|
85
|
+
*/
|
|
86
|
+
private sendBatchCompleted;
|
|
87
|
+
/** Fails every member of the batch uniformly, e.g. when the handler itself throws. */
|
|
88
|
+
private sendBatchFailureForAll;
|
|
89
|
+
getBatchActionEvent(action: Action, eventType: StepActionEventType, items: {
|
|
90
|
+
taskRunExternalId: string;
|
|
91
|
+
eventPayload: string;
|
|
92
|
+
}[]): BatchActionEvent;
|
|
73
93
|
getGroupKeyActionEvent(action: Action, eventType: GroupKeyActionEventType, payload?: any): GroupKeyActionEvent;
|
|
74
94
|
/**
|
|
75
95
|
* @important This method is instrumented by HatchetInstrumentor._patchHandleCancelStepRun.
|
|
@@ -93,6 +113,7 @@ export declare function mapSlotRequestsPb(task: {
|
|
|
93
113
|
slotCost?: number;
|
|
94
114
|
}, isDurable: boolean): Record<string, number>;
|
|
95
115
|
export declare function mapRateLimitPb(limits: CreateWorkflowTaskOpts<any, any>['rateLimits']): CreateStepRateLimit[];
|
|
116
|
+
export declare function mapBatchConfigPb(batch: CreateWorkflowTaskOpts<any, any>['batch']): TaskBatchConfig | undefined;
|
|
96
117
|
export declare function resolveExecutionTimeout(task: {
|
|
97
118
|
executionTimeout?: Duration;
|
|
98
119
|
timeout?: Duration;
|
|
@@ -55,6 +55,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
55
55
|
exports.InternalWorker = void 0;
|
|
56
56
|
exports.mapSlotRequestsPb = mapSlotRequestsPb;
|
|
57
57
|
exports.mapRateLimitPb = mapRateLimitPb;
|
|
58
|
+
exports.mapBatchConfigPb = mapBatchConfigPb;
|
|
58
59
|
exports.resolveExecutionTimeout = resolveExecutionTimeout;
|
|
59
60
|
exports.resolveScheduleTimeout = resolveScheduleTimeout;
|
|
60
61
|
const hatchet_error_1 = __importDefault(require("../../../util/errors/hatchet-error"));
|
|
@@ -153,7 +154,13 @@ class InternalWorker {
|
|
|
153
154
|
const newActions = workflow._tasks
|
|
154
155
|
.filter((task) => !!task.fn)
|
|
155
156
|
.reduce((acc, task) => {
|
|
156
|
-
|
|
157
|
+
const actionId = `${workflow.name}:${task.name.toLowerCase()}`;
|
|
158
|
+
if (task.batch) {
|
|
159
|
+
acc[actionId] = (ctx) => task.fn(decodeBatchItems(ctx.action.actionPayload), ctx);
|
|
160
|
+
}
|
|
161
|
+
else {
|
|
162
|
+
acc[actionId] = (ctx) => task.fn(ctx.input, ctx);
|
|
163
|
+
}
|
|
157
164
|
return acc;
|
|
158
165
|
}, {});
|
|
159
166
|
const onFailureFn = workflow.onFailure
|
|
@@ -311,7 +318,9 @@ class InternalWorker {
|
|
|
311
318
|
inputs: '{}',
|
|
312
319
|
parents: (_b = (_a = task.parents) === null || _a === void 0 ? void 0 : _a.map((p) => p.name)) !== null && _b !== void 0 ? _b : [],
|
|
313
320
|
userData: '{}',
|
|
314
|
-
|
|
321
|
+
// Batch tasks buffer many concurrent runs into a single execution; per-item retry
|
|
322
|
+
// semantics don't apply, so retries is always forced to 0.
|
|
323
|
+
retries: batchOf(task) ? 0 : task.retries || ((_c = workflow.taskDefaults) === null || _c === void 0 ? void 0 : _c.retries) || 0,
|
|
315
324
|
rateLimits: mapRateLimitPb(task.rateLimits || ((_d = workflow.taskDefaults) === null || _d === void 0 ? void 0 : _d.rateLimits)),
|
|
316
325
|
workerLabels: mapWorkerLabelPb(task.desiredWorkerLabels || ((_e = workflow.taskDefaults) === null || _e === void 0 ? void 0 : _e.workerLabels)),
|
|
317
326
|
backoffFactor: ((_f = task.backoff) === null || _f === void 0 ? void 0 : _f.factor) || ((_h = (_g = workflow.taskDefaults) === null || _g === void 0 ? void 0 : _g.backoff) === null || _h === void 0 ? void 0 : _h.factor),
|
|
@@ -319,6 +328,7 @@ class InternalWorker {
|
|
|
319
328
|
conditions: (0, transformer_1.taskConditionsToPb)(task, this.client.config.namespace),
|
|
320
329
|
isDurable: durableTaskSet.has(task),
|
|
321
330
|
slotRequests: mapSlotRequestsPb(task, durableTaskSet.has(task)),
|
|
331
|
+
batch: mapBatchConfigPb(batchOf(task)),
|
|
322
332
|
concurrency: task.concurrency
|
|
323
333
|
? Array.isArray(task.concurrency)
|
|
324
334
|
? task.concurrency
|
|
@@ -588,6 +598,106 @@ class InternalWorker {
|
|
|
588
598
|
retryCount,
|
|
589
599
|
};
|
|
590
600
|
}
|
|
601
|
+
/**
|
|
602
|
+
* Handles a START_BATCH action: invokes the registered batch task handler once with
|
|
603
|
+
* every buffered item, then reports completion/failure back per-member (or, for
|
|
604
|
+
* broadcast batch tasks, the same result for every member) via sendBatchActionEvent.
|
|
605
|
+
*/
|
|
606
|
+
handleStartBatch(action) {
|
|
607
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
608
|
+
const { actionId, taskName } = action;
|
|
609
|
+
const memberIds = batchMemberIds(action.actionPayload);
|
|
610
|
+
this.client.dispatcher
|
|
611
|
+
.sendBatchActionEvent(this.getBatchActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_STARTED, memberIds.map((id) => ({ taskRunExternalId: id, eventPayload: '' }))))
|
|
612
|
+
.catch((e) => {
|
|
613
|
+
this.logger.error(`Could not send batch started event: ${e.message}`);
|
|
614
|
+
});
|
|
615
|
+
const step = this.action_registry[actionId];
|
|
616
|
+
if (!step) {
|
|
617
|
+
this.logger.error(`Registered actions: '${Object.keys(this.action_registry).join(', ')}'`);
|
|
618
|
+
this.logger.error(`Could not find step '${actionId}'`);
|
|
619
|
+
return;
|
|
620
|
+
}
|
|
621
|
+
try {
|
|
622
|
+
const context = new context_1.Context(action, this.client, this);
|
|
623
|
+
const result = yield step(context);
|
|
624
|
+
// If the handler cancelled the batch (ctx.cancel()), a CANCELLED batch event
|
|
625
|
+
// covering every member was already sent from within cancel() itself. Don't also
|
|
626
|
+
// send a COMPLETED event for the same members afterward — matches the Python SDK's
|
|
627
|
+
// `if context.is_cancelled: return` guard in its batch runner.
|
|
628
|
+
if (context.cancelled) {
|
|
629
|
+
return undefined;
|
|
630
|
+
}
|
|
631
|
+
yield this.sendBatchCompleted(action, result);
|
|
632
|
+
return undefined;
|
|
633
|
+
}
|
|
634
|
+
catch (e) {
|
|
635
|
+
this.logger.error((0, logger_1.taskRunLog)(taskName, actionId, `batch failed: ${e.message}`));
|
|
636
|
+
if (e.stack) {
|
|
637
|
+
this.logger.error(e.stack);
|
|
638
|
+
}
|
|
639
|
+
yield this.sendBatchFailureForAll(action, memberIds, e);
|
|
640
|
+
return e instanceof Error ? e : new Error(String(e));
|
|
641
|
+
}
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
/**
|
|
645
|
+
* Reports the result of a successful batch handler invocation. result is expected to be
|
|
646
|
+
* a Record<string, any> keyed by batch member id; each member's output is serialized
|
|
647
|
+
* independently so that one member's serialization failure does not fail the whole
|
|
648
|
+
* batch.
|
|
649
|
+
*/
|
|
650
|
+
sendBatchCompleted(action, result) {
|
|
651
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
652
|
+
if (typeof result !== 'object' || result === null || Array.isArray(result)) {
|
|
653
|
+
yield this.sendBatchFailureForAll(action, Object.keys(result !== null && result !== void 0 ? result : {}), new Error('batch task handler did not return a valid per-member result map'));
|
|
654
|
+
return;
|
|
655
|
+
}
|
|
656
|
+
const completedItems = [];
|
|
657
|
+
const failedItems = [];
|
|
658
|
+
Object.entries(result).forEach(([id, output]) => {
|
|
659
|
+
try {
|
|
660
|
+
completedItems.push({ taskRunExternalId: id, eventPayload: JSON.stringify(output) });
|
|
661
|
+
}
|
|
662
|
+
catch (e) {
|
|
663
|
+
failedItems.push({
|
|
664
|
+
taskRunExternalId: id,
|
|
665
|
+
eventPayload: JSON.stringify({ message: e.message }),
|
|
666
|
+
});
|
|
667
|
+
}
|
|
668
|
+
});
|
|
669
|
+
if (completedItems.length > 0) {
|
|
670
|
+
yield this.client.dispatcher.sendBatchActionEvent(this.getBatchActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_COMPLETED, completedItems));
|
|
671
|
+
}
|
|
672
|
+
if (failedItems.length > 0) {
|
|
673
|
+
yield this.client.dispatcher.sendBatchActionEvent(this.getBatchActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, failedItems));
|
|
674
|
+
}
|
|
675
|
+
});
|
|
676
|
+
}
|
|
677
|
+
/** Fails every member of the batch uniformly, e.g. when the handler itself throws. */
|
|
678
|
+
sendBatchFailureForAll(action, memberIds, error) {
|
|
679
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
680
|
+
const payload = JSON.stringify({ message: error === null || error === void 0 ? void 0 : error.message, stack: error === null || error === void 0 ? void 0 : error.stack });
|
|
681
|
+
const items = memberIds.map((id) => ({ taskRunExternalId: id, eventPayload: payload }));
|
|
682
|
+
try {
|
|
683
|
+
yield this.client.dispatcher.sendBatchActionEvent(this.getBatchActionEvent(action, dispatcher_1.StepActionEventType.STEP_EVENT_TYPE_FAILED, items));
|
|
684
|
+
}
|
|
685
|
+
catch (e) {
|
|
686
|
+
this.logger.error(`Could not send batch failed event: ${e.message}`);
|
|
687
|
+
}
|
|
688
|
+
});
|
|
689
|
+
}
|
|
690
|
+
getBatchActionEvent(action, eventType, items) {
|
|
691
|
+
return {
|
|
692
|
+
workerId: this.name,
|
|
693
|
+
jobId: action.jobId,
|
|
694
|
+
actionId: action.actionId,
|
|
695
|
+
batchId: action.batchId,
|
|
696
|
+
eventTimestamp: new Date(),
|
|
697
|
+
eventType,
|
|
698
|
+
items,
|
|
699
|
+
};
|
|
700
|
+
}
|
|
591
701
|
getGroupKeyActionEvent(action, eventType, payload = '') {
|
|
592
702
|
if (!action.getGroupKeyRunId) {
|
|
593
703
|
throw new hatchet_error_1.default('No group key run id provided');
|
|
@@ -807,6 +917,8 @@ class InternalWorker {
|
|
|
807
917
|
switch (type) {
|
|
808
918
|
case dispatcher_1.ActionType.START_STEP_RUN:
|
|
809
919
|
return this.handleStartStepRun(action);
|
|
920
|
+
case dispatcher_1.ActionType.START_BATCH:
|
|
921
|
+
return this.handleStartBatch(action);
|
|
810
922
|
case dispatcher_1.ActionType.CANCEL_STEP_RUN:
|
|
811
923
|
return this.handleCancelStepRun(action);
|
|
812
924
|
case dispatcher_1.ActionType.START_GET_GROUP_KEY:
|
|
@@ -954,6 +1066,59 @@ function mapRateLimitPb(limits) {
|
|
|
954
1066
|
};
|
|
955
1067
|
});
|
|
956
1068
|
}
|
|
1069
|
+
/**
|
|
1070
|
+
* Decodes the buffered items of a batch task's START_BATCH action into a Record keyed by
|
|
1071
|
+
* each buffered item's task-run external id, mapping to that item's input. The wire shape
|
|
1072
|
+
* of actionPayload for a START_BATCH action is
|
|
1073
|
+
* `{ "<taskRunExternalId>": { "payload": { "input": {...}, ... }, "workflow_run_id": "..." }, ... }`,
|
|
1074
|
+
* distinct from the flat single-input shape used by START_STEP_RUN actions.
|
|
1075
|
+
*/
|
|
1076
|
+
function decodeBatchItems(actionPayload) {
|
|
1077
|
+
const parsed = parseBatchPayload(actionPayload);
|
|
1078
|
+
return Object.fromEntries(Object.entries(parsed).map(([id, item]) => { var _a, _b; return [id, (_b = (_a = item === null || item === void 0 ? void 0 : item.payload) === null || _a === void 0 ? void 0 : _a.input) !== null && _b !== void 0 ? _b : {}]; }));
|
|
1079
|
+
}
|
|
1080
|
+
function batchMemberIds(actionPayload) {
|
|
1081
|
+
return Object.keys(parseBatchPayload(actionPayload));
|
|
1082
|
+
}
|
|
1083
|
+
function parseBatchPayload(actionPayload) {
|
|
1084
|
+
if (!actionPayload) {
|
|
1085
|
+
return {};
|
|
1086
|
+
}
|
|
1087
|
+
try {
|
|
1088
|
+
const parsed = JSON.parse(actionPayload);
|
|
1089
|
+
return typeof parsed === 'object' && parsed !== null ? parsed : {};
|
|
1090
|
+
}
|
|
1091
|
+
catch (_a) {
|
|
1092
|
+
return {};
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
/** Batch tasks are only available on non-durable tasks; durable tasks never carry `batch`. */
|
|
1096
|
+
function batchOf(task) {
|
|
1097
|
+
return 'batch' in task ? task.batch : undefined;
|
|
1098
|
+
}
|
|
1099
|
+
function mapBatchConfigPb(batch) {
|
|
1100
|
+
if (!batch) {
|
|
1101
|
+
return undefined;
|
|
1102
|
+
}
|
|
1103
|
+
if (!Number.isInteger(batch.maxSize) || batch.maxSize <= 0) {
|
|
1104
|
+
throw new Error(`batch.maxSize must be a positive integer, got: ${batch.maxSize}`);
|
|
1105
|
+
}
|
|
1106
|
+
const batchMaxIntervalMs = batch.maxInterval !== undefined ? (0, duration_1.durationToMs)(batch.maxInterval) : undefined;
|
|
1107
|
+
if (batchMaxIntervalMs !== undefined && batchMaxIntervalMs <= 0) {
|
|
1108
|
+
throw new Error('batch.maxInterval must be positive when provided');
|
|
1109
|
+
}
|
|
1110
|
+
if (batch.groupMaxRuns !== undefined &&
|
|
1111
|
+
(!Number.isInteger(batch.groupMaxRuns) || batch.groupMaxRuns <= 0)) {
|
|
1112
|
+
throw new Error(`batch.groupMaxRuns must be a positive integer when provided, got: ${batch.groupMaxRuns}`);
|
|
1113
|
+
}
|
|
1114
|
+
return {
|
|
1115
|
+
batchMaxSize: batch.maxSize,
|
|
1116
|
+
batchMaxIntervalMs,
|
|
1117
|
+
batchGroupKey: batch.groupKey,
|
|
1118
|
+
batchGroupMaxRuns: batch.groupMaxRuns,
|
|
1119
|
+
broadcastOutput: batch.broadcastOutput,
|
|
1120
|
+
};
|
|
1121
|
+
}
|
|
957
1122
|
// Helper function to validate CEL expressions
|
|
958
1123
|
function validateCelExpression(_expr) {
|
|
959
1124
|
// FIXME: this is a placeholder. In a real implementation, you'd need to use a CEL parser or validator.
|
package/v1/declaration.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ import WorkflowRunRef from '../util/workflow-run-ref';
|
|
|
9
9
|
import { CronWorkflows, ScheduledWorkflows, V1CreateFilterRequest } from '../clients/rest/generated/data-contracts';
|
|
10
10
|
import * as z from 'zod/v4';
|
|
11
11
|
import { IHatchetClient } from './client/client.interface';
|
|
12
|
-
import { CreateWorkflowTaskOpts, CreateOnFailureTaskOpts, TaskFn, CreateWorkflowDurableTaskOpts, CreateBaseTaskOpts, CreateOnSuccessTaskOpts, Concurrency, DurableTaskFn, WorkerLabelComparator, IdempotencyConfig } from './task';
|
|
12
|
+
import { CreateWorkflowTaskOpts, CreateOnFailureTaskOpts, TaskFn, CreateWorkflowDurableTaskOpts, CreateBaseTaskOpts, CreateOnSuccessTaskOpts, Concurrency, DurableTaskFn, WorkerLabelComparator, IdempotencyConfig, BatchTaskConfig, BatchTaskFn } from './task';
|
|
13
13
|
import { Duration } from './client/duration';
|
|
14
14
|
import { MetricsClient } from './client/features/metrics';
|
|
15
15
|
import { InputType, OutputType, UnknownInputType, JsonObject, Resolved } from './types';
|
|
@@ -159,6 +159,16 @@ export type CreateBaseWorkflowOpts = {
|
|
|
159
159
|
idempotency?: IdempotencyConfig;
|
|
160
160
|
};
|
|
161
161
|
export type CreateTaskWorkflowOpts<I extends InputType = UnknownInputType, O extends OutputType = void> = CreateBaseWorkflowOpts & CreateBaseTaskOpts<I, O, TaskFn<I, O>>;
|
|
162
|
+
/**
|
|
163
|
+
* Options for creating a batch task workflow. Batch tasks buffer concurrent runs and
|
|
164
|
+
* dispatch them together as a single execution; fn receives a Record keyed by each
|
|
165
|
+
* buffered run's task-run external id.
|
|
166
|
+
*
|
|
167
|
+
* Preview: batch tasks are in beta and may change in future releases.
|
|
168
|
+
*/
|
|
169
|
+
export type CreateBatchTaskWorkflowOpts<I extends InputType = UnknownInputType, O extends OutputType = void> = CreateBaseWorkflowOpts & CreateBaseTaskOpts<I, O, BatchTaskFn<I, O>> & {
|
|
170
|
+
batch: BatchTaskConfig;
|
|
171
|
+
};
|
|
162
172
|
export type CreateDurableTaskWorkflowOpts<I extends InputType = UnknownInputType, O extends OutputType = void> = CreateBaseWorkflowOpts & CreateBaseTaskOpts<I, O, DurableTaskFn<I, O>> & {
|
|
163
173
|
evictionPolicy?: EvictionPolicy;
|
|
164
174
|
};
|
|
@@ -434,6 +444,24 @@ export declare class WorkflowDeclaration<I extends InputType = UnknownInputType,
|
|
|
434
444
|
name: Name;
|
|
435
445
|
fn?: Fn;
|
|
436
446
|
}) | TaskWorkflowDeclaration<I, TO>): CreateWorkflowTaskOpts<I, TO>;
|
|
447
|
+
/**
|
|
448
|
+
* Adds a batch task to the workflow. Batch tasks buffer concurrent runs until Hatchet
|
|
449
|
+
* flushes the batch (size reached or flush interval), then invoke the handler once with
|
|
450
|
+
* all buffered inputs keyed by each run's task-run external id. The handler must return
|
|
451
|
+
* a Record mapping each id to its output, or set `batch.broadcastOutput` to return the
|
|
452
|
+
* same result to all callers. retries is always forced to 0 for batch tasks.
|
|
453
|
+
*
|
|
454
|
+
* Preview: batch tasks are in beta and may change in future releases.
|
|
455
|
+
* @template Fn The type of the batch task function.
|
|
456
|
+
* @param options The batch task configuration options.
|
|
457
|
+
* @returns The task options that were added.
|
|
458
|
+
*/
|
|
459
|
+
batchTask<Fn extends BatchTaskFn<TI, TO>, TI extends InputType = Parameters<Fn>[0] extends Record<string, infer II> ? II extends InputType ? II : UnknownInputType : UnknownInputType, TO extends OutputType = ReturnType<Fn> extends Promise<infer P> ? P extends OutputType ? P : void : ReturnType<Fn> extends OutputType ? ReturnType<Fn> : void>(options: {
|
|
460
|
+
fn: Fn;
|
|
461
|
+
batch: BatchTaskConfig;
|
|
462
|
+
} & Omit<CreateBatchTaskWorkflowOpts<TI, TO>, 'fn' | 'batch' | 'name'> & {
|
|
463
|
+
name: string;
|
|
464
|
+
}): CreateWorkflowTaskOpts<TI, TO>;
|
|
437
465
|
/**
|
|
438
466
|
* Adds an onFailure task to the workflow.
|
|
439
467
|
* This will only run if any task in the workflow fails.
|
|
@@ -465,7 +493,7 @@ export declare class WorkflowDeclaration<I extends InputType = UnknownInputType,
|
|
|
465
493
|
* @param options The task configuration options.
|
|
466
494
|
* @returns The task options that were added.
|
|
467
495
|
*/
|
|
468
|
-
durableTask<Name extends string, Fn extends (Name extends keyof O ? (input: I & MiddlewareBefore, ctx: DurableContext<I & MiddlewareBefore>) => O[Name] extends OutputType ? O[Name] | Promise<O[Name]> : void : (input: I & MiddlewareBefore, ctx: DurableContext<I & MiddlewareBefore>) => void), FnReturn = ReturnType<Fn> extends Promise<infer P> ? P : ReturnType<Fn>, TO extends OutputType = Name extends keyof O ? O[Name] extends OutputType ? O[Name] : never : FnReturn extends OutputType ? FnReturn : never>(options: Omit<CreateWorkflowTaskOpts<I, TO>, 'fn' | 'slotCost'> & {
|
|
496
|
+
durableTask<Name extends string, Fn extends (Name extends keyof O ? (input: I & MiddlewareBefore, ctx: DurableContext<I & MiddlewareBefore>) => O[Name] extends OutputType ? O[Name] | Promise<O[Name]> : void : (input: I & MiddlewareBefore, ctx: DurableContext<I & MiddlewareBefore>) => void), FnReturn = ReturnType<Fn> extends Promise<infer P> ? P : ReturnType<Fn>, TO extends OutputType = Name extends keyof O ? O[Name] extends OutputType ? O[Name] : never : FnReturn extends OutputType ? FnReturn : never>(options: Omit<CreateWorkflowTaskOpts<I, TO>, 'fn' | 'slotCost' | 'batch'> & {
|
|
469
497
|
name: Name;
|
|
470
498
|
fn: Fn;
|
|
471
499
|
}): CreateWorkflowDurableTaskOpts<I, TO>;
|
|
@@ -573,6 +601,38 @@ export declare class TaskWorkflowDeclaration<I extends InputType = UnknownInputT
|
|
|
573
601
|
export declare function CreateTaskWorkflow<Fn extends (input: I, ctx?: any) => O | Promise<O>, I extends InputType = Parameters<Fn>[0], O extends OutputType = ReturnType<Fn> extends Promise<infer P> ? P extends OutputType ? P : void : ReturnType<Fn> extends OutputType ? ReturnType<Fn> : void>(options: {
|
|
574
602
|
fn: Fn;
|
|
575
603
|
} & Omit<CreateTaskWorkflowOpts<I, O>, 'fn'>, client?: IHatchetClient): TaskWorkflowDeclaration<I, O>;
|
|
604
|
+
/**
|
|
605
|
+
* Creates a new batch task workflow declaration, with the handler's single return value
|
|
606
|
+
* broadcast to every member of the batch.
|
|
607
|
+
* @template Fn The type of the batch task function
|
|
608
|
+
* @param options The batch task configuration options.
|
|
609
|
+
* @param client Optional Hatchet client instance.
|
|
610
|
+
* @returns A new TaskWorkflowDeclaration with inferred types.
|
|
611
|
+
*
|
|
612
|
+
* Preview: batch tasks are in beta and may change in future releases.
|
|
613
|
+
*/
|
|
614
|
+
export declare function CreateBatchTaskWorkflow<Fn extends (input: Record<string, I>, ctx: Context<Record<string, I>>) => O | Promise<O>, I extends InputType = Parameters<Fn>[0] extends Record<string, infer II> ? II extends InputType ? II : UnknownInputType : UnknownInputType, O extends OutputType = ReturnType<Fn> extends Promise<infer P> ? P extends OutputType ? P : void : ReturnType<Fn> extends OutputType ? ReturnType<Fn> : void>(options: {
|
|
615
|
+
fn: Fn;
|
|
616
|
+
batch: BatchTaskConfig & {
|
|
617
|
+
broadcastOutput: true;
|
|
618
|
+
};
|
|
619
|
+
} & Omit<CreateBatchTaskWorkflowOpts<I, O>, 'fn' | 'batch'>, client?: IHatchetClient): TaskWorkflowDeclaration<I, O>;
|
|
620
|
+
/**
|
|
621
|
+
* Creates a new batch task workflow declaration. The handler receives a Record keyed by
|
|
622
|
+
* batch member id and must return a Record with the exact same key set.
|
|
623
|
+
* @template Fn The type of the batch task function
|
|
624
|
+
* @param options The batch task configuration options.
|
|
625
|
+
* @param client Optional Hatchet client instance.
|
|
626
|
+
* @returns A new TaskWorkflowDeclaration with inferred types.
|
|
627
|
+
*
|
|
628
|
+
* Preview: batch tasks are in beta and may change in future releases.
|
|
629
|
+
*/
|
|
630
|
+
export declare function CreateBatchTaskWorkflow<Fn extends (input: Record<string, I>, ctx: Context<Record<string, I>>) => Record<string, O> | Promise<Record<string, O>>, I extends InputType = Parameters<Fn>[0] extends Record<string, infer II> ? II extends InputType ? II : UnknownInputType : UnknownInputType, O extends OutputType = ReturnType<Fn> extends Promise<infer P> ? P extends Record<string, infer OO> ? OO extends OutputType ? OO : void : void : ReturnType<Fn> extends Record<string, infer OO> ? OO extends OutputType ? OO : void : void>(options: {
|
|
631
|
+
fn: Fn;
|
|
632
|
+
batch: BatchTaskConfig & {
|
|
633
|
+
broadcastOutput?: false;
|
|
634
|
+
};
|
|
635
|
+
} & Omit<CreateBatchTaskWorkflowOpts<I, O>, 'fn' | 'batch'>, client?: IHatchetClient): TaskWorkflowDeclaration<I, O>;
|
|
576
636
|
/**
|
|
577
637
|
* Creates a new workflow instance.
|
|
578
638
|
* @template I The input type for the workflow.
|
package/v1/declaration.js
CHANGED
|
@@ -26,11 +26,16 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
26
26
|
}
|
|
27
27
|
return t;
|
|
28
28
|
};
|
|
29
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
30
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
31
|
+
};
|
|
29
32
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
30
33
|
exports.TaskWorkflowDeclaration = exports.WorkflowDeclaration = exports.BaseWorkflowDeclaration = exports.StickyStrategy = exports.Priority = void 0;
|
|
31
34
|
exports.CreateTaskWorkflow = CreateTaskWorkflow;
|
|
35
|
+
exports.CreateBatchTaskWorkflow = CreateBatchTaskWorkflow;
|
|
32
36
|
exports.CreateWorkflow = CreateWorkflow;
|
|
33
37
|
exports.CreateDurableTaskWorkflow = CreateDurableTaskWorkflow;
|
|
38
|
+
const hatchet_error_1 = __importDefault(require("../util/errors/hatchet-error"));
|
|
34
39
|
const abort_error_1 = require("../util/abort-error");
|
|
35
40
|
const parent_run_context_vars_1 = require("./parent-run-context-vars");
|
|
36
41
|
const UNBOUND_ERR = new Error('workflow unbound to hatchet client, hint: use client.run instead');
|
|
@@ -481,6 +486,25 @@ class WorkflowDeclaration extends BaseWorkflowDeclaration {
|
|
|
481
486
|
this.definition._tasks.push(typedOptions);
|
|
482
487
|
return typedOptions;
|
|
483
488
|
}
|
|
489
|
+
/**
|
|
490
|
+
* Adds a batch task to the workflow. Batch tasks buffer concurrent runs until Hatchet
|
|
491
|
+
* flushes the batch (size reached or flush interval), then invoke the handler once with
|
|
492
|
+
* all buffered inputs keyed by each run's task-run external id. The handler must return
|
|
493
|
+
* a Record mapping each id to its output, or set `batch.broadcastOutput` to return the
|
|
494
|
+
* same result to all callers. retries is always forced to 0 for batch tasks.
|
|
495
|
+
*
|
|
496
|
+
* Preview: batch tasks are in beta and may change in future releases.
|
|
497
|
+
* @template Fn The type of the batch task function.
|
|
498
|
+
* @param options The batch task configuration options.
|
|
499
|
+
* @returns The task options that were added.
|
|
500
|
+
*/
|
|
501
|
+
batchTask(options) {
|
|
502
|
+
const { fn, batch } = options, rest = __rest(options, ["fn", "batch"]);
|
|
503
|
+
const wrappedFn = wrapBatchFn(fn, !!batch.broadcastOutput);
|
|
504
|
+
const typedOptions = Object.assign(Object.assign({}, rest), { fn: wrappedFn, batch });
|
|
505
|
+
this.definition._tasks.push(typedOptions);
|
|
506
|
+
return typedOptions;
|
|
507
|
+
}
|
|
484
508
|
/**
|
|
485
509
|
* Adds an onFailure task to the workflow.
|
|
486
510
|
* This will only run if any task in the workflow fails.
|
|
@@ -692,6 +716,38 @@ exports.TaskWorkflowDeclaration = TaskWorkflowDeclaration;
|
|
|
692
716
|
function CreateTaskWorkflow(options, client) {
|
|
693
717
|
return new TaskWorkflowDeclaration(options, client);
|
|
694
718
|
}
|
|
719
|
+
/**
|
|
720
|
+
* Wraps a user-provided batch task handler so it always resolves to a Record keyed by
|
|
721
|
+
* batch member id, regardless of whether the handler itself uses broadcastOutput. When
|
|
722
|
+
* broadcastOutput is set, the handler's single return value is copied to every member id
|
|
723
|
+
* present in the input. Otherwise, the handler's returned Record must have exactly the
|
|
724
|
+
* same key set as the input; a mismatch throws, which fails every member of the batch
|
|
725
|
+
* uniformly (matching the behavior of an uncaught error from the handler itself).
|
|
726
|
+
*/
|
|
727
|
+
function wrapBatchFn(fn, broadcastOutput) {
|
|
728
|
+
return (input, ctx) => __awaiter(this, void 0, void 0, function* () {
|
|
729
|
+
const result = yield fn(input, ctx);
|
|
730
|
+
if (broadcastOutput) {
|
|
731
|
+
return Object.fromEntries(Object.keys(input).map((id) => [id, result]));
|
|
732
|
+
}
|
|
733
|
+
if (typeof result !== 'object' || result === null || Array.isArray(result)) {
|
|
734
|
+
throw new hatchet_error_1.default('batch task handler must return an object keyed by batch member id when broadcastOutput is false');
|
|
735
|
+
}
|
|
736
|
+
const inputKeys = Object.keys(input);
|
|
737
|
+
const resultKeys = Object.keys(result);
|
|
738
|
+
const missing = inputKeys.filter((k) => !resultKeys.includes(k));
|
|
739
|
+
const extra = resultKeys.filter((k) => !inputKeys.includes(k));
|
|
740
|
+
if (missing.length > 0 || extra.length > 0) {
|
|
741
|
+
throw new hatchet_error_1.default(`batch task handler result keys do not match batch member ids (missing=${missing.join(', ')}, extra=${extra.join(', ')})`);
|
|
742
|
+
}
|
|
743
|
+
return result;
|
|
744
|
+
});
|
|
745
|
+
}
|
|
746
|
+
function CreateBatchTaskWorkflow(options, client) {
|
|
747
|
+
const { fn, batch } = options, rest = __rest(options, ["fn", "batch"]);
|
|
748
|
+
const wrappedFn = wrapBatchFn(fn, !!batch.broadcastOutput);
|
|
749
|
+
return new TaskWorkflowDeclaration(Object.assign(Object.assign({}, rest), { fn: wrappedFn, batch, retries: 0 }), client);
|
|
750
|
+
}
|
|
695
751
|
/**
|
|
696
752
|
* Creates a new workflow instance.
|
|
697
753
|
* @template I The input type for the workflow.
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
type SimpleInput = {
|
|
2
|
+
message: string;
|
|
3
|
+
};
|
|
4
|
+
type SimpleOutput = {
|
|
5
|
+
transformed_message: string;
|
|
6
|
+
};
|
|
7
|
+
type KeyedInput = {
|
|
8
|
+
message: string;
|
|
9
|
+
group: string;
|
|
10
|
+
};
|
|
11
|
+
type KeyedFailableInput = {
|
|
12
|
+
message: string;
|
|
13
|
+
group: string | number;
|
|
14
|
+
};
|
|
15
|
+
type KeyedOutput = {
|
|
16
|
+
batch_key?: string;
|
|
17
|
+
batch_size?: number;
|
|
18
|
+
unique_keys?: number;
|
|
19
|
+
uppercase: string;
|
|
20
|
+
};
|
|
21
|
+
type LargePayloadInput = {
|
|
22
|
+
data: string;
|
|
23
|
+
};
|
|
24
|
+
type LargeOutput = {
|
|
25
|
+
batch_id: string;
|
|
26
|
+
received: boolean;
|
|
27
|
+
batch_size: number;
|
|
28
|
+
data_length: number;
|
|
29
|
+
};
|
|
30
|
+
type SingleOutput = {
|
|
31
|
+
original: string;
|
|
32
|
+
batch_size: number;
|
|
33
|
+
};
|
|
34
|
+
type OrderedInput = {
|
|
35
|
+
index: number;
|
|
36
|
+
};
|
|
37
|
+
type OrderedOutput = {
|
|
38
|
+
index: number;
|
|
39
|
+
};
|
|
40
|
+
type BroadcastOutput = {
|
|
41
|
+
sum: number;
|
|
42
|
+
};
|
|
43
|
+
type ChildOutput = {
|
|
44
|
+
message_len: number;
|
|
45
|
+
};
|
|
46
|
+
type ChildBatchOutput = {
|
|
47
|
+
out: Record<string, SimpleInput>;
|
|
48
|
+
};
|
|
49
|
+
export declare const batchSimple: import("../..").TaskWorkflowDeclaration<SimpleInput, Record<string, SimpleOutput>, {}, {}, {}, {}>;
|
|
50
|
+
export declare const batchKeyed: import("../..").TaskWorkflowDeclaration<KeyedInput, Record<string, KeyedOutput>, {}, {}, {}, {}>;
|
|
51
|
+
export declare const batchKeyedFailable: import("../..").TaskWorkflowDeclaration<KeyedFailableInput, Record<string, KeyedOutput>, {}, {}, {}, {}>;
|
|
52
|
+
export declare const batchKeyedInterval: import("../..").TaskWorkflowDeclaration<KeyedInput, Record<string, KeyedOutput>, {}, {}, {}, {}>;
|
|
53
|
+
export declare const batchLarge: import("../..").TaskWorkflowDeclaration<LargePayloadInput, Record<string, LargeOutput>, {}, {}, {}, {}>;
|
|
54
|
+
export declare const batchSingle: import("../..").TaskWorkflowDeclaration<SimpleInput, Record<string, SingleOutput>, {}, {}, {}, {}>;
|
|
55
|
+
export declare const batchOrdered: import("../..").TaskWorkflowDeclaration<OrderedInput, Record<string, OrderedOutput>, {}, {}, {}, {}>;
|
|
56
|
+
export declare const batchBroadcast: import("../..").TaskWorkflowDeclaration<SimpleInput, BroadcastOutput, {}, {}, {}, {}>;
|
|
57
|
+
export declare const batchCancel: import("../..").TaskWorkflowDeclaration<SimpleInput, {}, {}, {}, {}, {}>;
|
|
58
|
+
export declare const child: import("../..").TaskWorkflowDeclaration<SimpleInput, ChildOutput, {}, {}, {}, {}>;
|
|
59
|
+
export declare const childBatch: import("../..").TaskWorkflowDeclaration<SimpleInput, ChildBatchOutput, {}, {}, {}, {}>;
|
|
60
|
+
export declare const batchChildSpawn: import("../..").TaskWorkflowDeclaration<SimpleInput, Record<string, ChildOutput>, {}, {}, {}, {}>;
|
|
61
|
+
export declare const batchChildBatchSpawn: import("../..").TaskWorkflowDeclaration<SimpleInput, Record<string, ChildBatchOutput>, {}, {}, {}, {}>;
|
|
62
|
+
export {};
|