@workflow/core 4.0.1-beta.9 → 4.1.0-beta.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtins.js +1 -1
- package/dist/class-serialization.d.ts +26 -0
- package/dist/class-serialization.d.ts.map +1 -0
- package/dist/class-serialization.js +66 -0
- package/dist/create-hook.js +1 -1
- package/dist/define-hook.d.ts +40 -25
- package/dist/define-hook.d.ts.map +1 -1
- package/dist/define-hook.js +22 -27
- package/dist/events-consumer.d.ts.map +1 -1
- package/dist/events-consumer.js +5 -1
- package/dist/flushable-stream.d.ts +82 -0
- package/dist/flushable-stream.d.ts.map +1 -0
- package/dist/flushable-stream.js +214 -0
- package/dist/global.d.ts +4 -1
- package/dist/global.d.ts.map +1 -1
- package/dist/global.js +21 -9
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/logger.js +1 -1
- package/dist/observability.d.ts +60 -0
- package/dist/observability.d.ts.map +1 -1
- package/dist/observability.js +265 -32
- package/dist/private.d.ts +10 -1
- package/dist/private.d.ts.map +1 -1
- package/dist/private.js +6 -1
- package/dist/runtime/helpers.d.ts +52 -0
- package/dist/runtime/helpers.d.ts.map +1 -0
- package/dist/runtime/helpers.js +264 -0
- package/dist/runtime/resume-hook.d.ts +17 -12
- package/dist/runtime/resume-hook.d.ts.map +1 -1
- package/dist/runtime/resume-hook.js +79 -64
- package/dist/runtime/run.d.ts +100 -0
- package/dist/runtime/run.d.ts.map +1 -0
- package/dist/runtime/run.js +132 -0
- package/dist/runtime/start.d.ts +15 -1
- package/dist/runtime/start.d.ts.map +1 -1
- package/dist/runtime/start.js +72 -46
- package/dist/runtime/step-handler.d.ts +7 -0
- package/dist/runtime/step-handler.d.ts.map +1 -0
- package/dist/runtime/step-handler.js +337 -0
- package/dist/runtime/suspension-handler.d.ts +25 -0
- package/dist/runtime/suspension-handler.d.ts.map +1 -0
- package/dist/runtime/suspension-handler.js +182 -0
- package/dist/runtime/world.d.ts.map +1 -1
- package/dist/runtime/world.js +20 -21
- package/dist/runtime.d.ts +4 -105
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +97 -531
- package/dist/schemas.d.ts +1 -15
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +2 -15
- package/dist/serialization.d.ts +112 -21
- package/dist/serialization.d.ts.map +1 -1
- package/dist/serialization.js +469 -85
- package/dist/sleep.d.ts +10 -0
- package/dist/sleep.d.ts.map +1 -1
- package/dist/sleep.js +1 -1
- package/dist/source-map.d.ts +10 -0
- package/dist/source-map.d.ts.map +1 -0
- package/dist/source-map.js +56 -0
- package/dist/step/context-storage.d.ts +2 -1
- package/dist/step/context-storage.d.ts.map +1 -1
- package/dist/step/context-storage.js +1 -1
- package/dist/step/get-closure-vars.d.ts +9 -0
- package/dist/step/get-closure-vars.d.ts.map +1 -0
- package/dist/step/get-closure-vars.js +16 -0
- package/dist/step/get-step-metadata.js +1 -1
- package/dist/step/get-workflow-metadata.js +1 -1
- package/dist/step/writable-stream.d.ts +10 -2
- package/dist/step/writable-stream.d.ts.map +1 -1
- package/dist/step/writable-stream.js +6 -5
- package/dist/step.d.ts +1 -1
- package/dist/step.d.ts.map +1 -1
- package/dist/step.js +93 -47
- package/dist/symbols.d.ts +6 -0
- package/dist/symbols.d.ts.map +1 -1
- package/dist/symbols.js +7 -1
- package/dist/telemetry/semantic-conventions.d.ts +66 -38
- package/dist/telemetry/semantic-conventions.d.ts.map +1 -1
- package/dist/telemetry/semantic-conventions.js +16 -3
- package/dist/telemetry.d.ts +8 -4
- package/dist/telemetry.d.ts.map +1 -1
- package/dist/telemetry.js +39 -6
- package/dist/types.js +1 -1
- package/dist/util.d.ts +5 -24
- package/dist/util.d.ts.map +1 -1
- package/dist/util.js +19 -38
- package/dist/version.d.ts +2 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +3 -0
- package/dist/vm/index.js +2 -2
- package/dist/vm/uuid.js +1 -1
- package/dist/workflow/create-hook.js +1 -1
- package/dist/workflow/define-hook.d.ts +3 -3
- package/dist/workflow/define-hook.d.ts.map +1 -1
- package/dist/workflow/define-hook.js +1 -1
- package/dist/workflow/get-workflow-metadata.js +1 -1
- package/dist/workflow/hook.d.ts.map +1 -1
- package/dist/workflow/hook.js +49 -14
- package/dist/workflow/index.d.ts +1 -1
- package/dist/workflow/index.d.ts.map +1 -1
- package/dist/workflow/index.js +2 -2
- package/dist/workflow/sleep.d.ts +1 -1
- package/dist/workflow/sleep.d.ts.map +1 -1
- package/dist/workflow/sleep.js +26 -39
- package/dist/workflow/writable-stream.d.ts +1 -1
- package/dist/workflow/writable-stream.d.ts.map +1 -1
- package/dist/workflow/writable-stream.js +1 -1
- package/dist/workflow.d.ts +1 -1
- package/dist/workflow.d.ts.map +1 -1
- package/dist/workflow.js +72 -9
- package/docs/api-reference/create-hook.mdx +134 -0
- package/docs/api-reference/create-webhook.mdx +226 -0
- package/docs/api-reference/define-hook.mdx +207 -0
- package/docs/api-reference/fatal-error.mdx +38 -0
- package/docs/api-reference/fetch.mdx +140 -0
- package/docs/api-reference/get-step-metadata.mdx +77 -0
- package/docs/api-reference/get-workflow-metadata.mdx +45 -0
- package/docs/api-reference/get-writable.mdx +292 -0
- package/docs/api-reference/index.mdx +56 -0
- package/docs/api-reference/meta.json +3 -0
- package/docs/api-reference/retryable-error.mdx +107 -0
- package/docs/api-reference/sleep.mdx +60 -0
- package/docs/foundations/common-patterns.mdx +254 -0
- package/docs/foundations/errors-and-retries.mdx +191 -0
- package/docs/foundations/hooks.mdx +456 -0
- package/docs/foundations/idempotency.mdx +56 -0
- package/docs/foundations/index.mdx +33 -0
- package/docs/foundations/meta.json +14 -0
- package/docs/foundations/serialization.mdx +158 -0
- package/docs/foundations/starting-workflows.mdx +212 -0
- package/docs/foundations/streaming.mdx +570 -0
- package/docs/foundations/workflows-and-steps.mdx +198 -0
- package/docs/how-it-works/code-transform.mdx +335 -0
- package/docs/how-it-works/event-sourcing.mdx +255 -0
- package/docs/how-it-works/framework-integrations.mdx +438 -0
- package/docs/how-it-works/meta.json +10 -0
- package/docs/how-it-works/understanding-directives.mdx +612 -0
- package/package.json +31 -25
- package/dist/builtins.js.map +0 -1
- package/dist/create-hook.js.map +0 -1
- package/dist/define-hook.js.map +0 -1
- package/dist/events-consumer.js.map +0 -1
- package/dist/global.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/logger.js.map +0 -1
- package/dist/observability.js.map +0 -1
- package/dist/parse-name.d.ts +0 -25
- package/dist/parse-name.d.ts.map +0 -1
- package/dist/parse-name.js +0 -40
- package/dist/parse-name.js.map +0 -1
- package/dist/private.js.map +0 -1
- package/dist/runtime/resume-hook.js.map +0 -1
- package/dist/runtime/start.js.map +0 -1
- package/dist/runtime/world.js.map +0 -1
- package/dist/runtime.js.map +0 -1
- package/dist/schemas.js.map +0 -1
- package/dist/serialization.js.map +0 -1
- package/dist/sleep.js.map +0 -1
- package/dist/step/context-storage.js.map +0 -1
- package/dist/step/get-step-metadata.js.map +0 -1
- package/dist/step/get-workflow-metadata.js.map +0 -1
- package/dist/step/writable-stream.js.map +0 -1
- package/dist/step.js.map +0 -1
- package/dist/symbols.js.map +0 -1
- package/dist/telemetry/semantic-conventions.js.map +0 -1
- package/dist/telemetry.js.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/util.js.map +0 -1
- package/dist/vm/index.js.map +0 -1
- package/dist/vm/uuid.js.map +0 -1
- package/dist/workflow/create-hook.js.map +0 -1
- package/dist/workflow/define-hook.js.map +0 -1
- package/dist/workflow/get-workflow-metadata.js.map +0 -1
- package/dist/workflow/hook.js.map +0 -1
- package/dist/workflow/index.js.map +0 -1
- package/dist/workflow/sleep.js.map +0 -1
- package/dist/workflow/writable-stream.js.map +0 -1
- package/dist/workflow.js.map +0 -1
- package/dist/writable-stream.d.ts +0 -23
- package/dist/writable-stream.d.ts.map +0 -1
- package/dist/writable-stream.js +0 -17
- package/dist/writable-stream.js.map +0 -1
package/dist/runtime.js
CHANGED
|
@@ -1,169 +1,24 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { WorkflowRuntimeError } from '@workflow/errors';
|
|
2
|
+
import { parseWorkflowName } from '@workflow/utils/parse-name';
|
|
3
|
+
import { SPEC_VERSION_CURRENT, WorkflowInvokePayloadSchema, } from '@workflow/world';
|
|
3
4
|
import { WorkflowSuspension } from './global.js';
|
|
4
5
|
import { runtimeLogger } from './logger.js';
|
|
5
|
-
import {
|
|
6
|
+
import { getAllWorkflowRunEvents, getQueueOverhead, handleHealthCheckMessage, parseHealthCheckPayload, withHealthCheck, } from './runtime/helpers.js';
|
|
7
|
+
import { handleSuspension } from './runtime/suspension-handler.js';
|
|
6
8
|
import { getWorld, getWorldHandlers } from './runtime/world.js';
|
|
7
|
-
import {
|
|
8
|
-
import { dehydrateStepArguments, dehydrateStepReturnValue, getExternalRevivers, hydrateStepArguments, hydrateWorkflowReturnValue, } from './serialization.js';
|
|
9
|
-
// TODO: move step handler out to a separate file
|
|
10
|
-
import { contextStorage } from './step/context-storage.js';
|
|
9
|
+
import { remapErrorStack } from './source-map.js';
|
|
11
10
|
import * as Attribute from './telemetry/semantic-conventions.js';
|
|
12
|
-
import {
|
|
11
|
+
import { linkToCurrentContext, trace, withTraceContext } from './telemetry.js';
|
|
13
12
|
import { getErrorName, getErrorStack } from './types.js';
|
|
14
|
-
import { buildWorkflowSuspensionMessage
|
|
13
|
+
import { buildWorkflowSuspensionMessage } from './util.js';
|
|
15
14
|
import { runWorkflow } from './workflow.js';
|
|
16
15
|
export { WorkflowSuspension } from './global.js';
|
|
16
|
+
export { healthCheck, } from './runtime/helpers.js';
|
|
17
17
|
export { getHookByToken, resumeHook, resumeWebhook, } from './runtime/resume-hook.js';
|
|
18
18
|
export { start } from './runtime/start.js';
|
|
19
|
+
export { stepEntrypoint } from './runtime/step-handler.js';
|
|
19
20
|
export { createWorld, getWorld, getWorldHandlers, setWorld, } from './runtime/world.js';
|
|
20
|
-
|
|
21
|
-
* A handler class for a workflow run.
|
|
22
|
-
*/
|
|
23
|
-
export class Run {
|
|
24
|
-
/**
|
|
25
|
-
* The ID of the workflow run.
|
|
26
|
-
*/
|
|
27
|
-
runId;
|
|
28
|
-
/**
|
|
29
|
-
* The world object.
|
|
30
|
-
* @internal
|
|
31
|
-
*/
|
|
32
|
-
world;
|
|
33
|
-
constructor(runId) {
|
|
34
|
-
this.runId = runId;
|
|
35
|
-
this.world = getWorld();
|
|
36
|
-
}
|
|
37
|
-
/**
|
|
38
|
-
* Cancels the workflow run.
|
|
39
|
-
*/
|
|
40
|
-
async cancel() {
|
|
41
|
-
await this.world.runs.cancel(this.runId);
|
|
42
|
-
}
|
|
43
|
-
/**
|
|
44
|
-
* The status of the workflow run.
|
|
45
|
-
*/
|
|
46
|
-
get status() {
|
|
47
|
-
return this.world.runs.get(this.runId).then((run) => run.status);
|
|
48
|
-
}
|
|
49
|
-
/**
|
|
50
|
-
* The return value of the workflow run.
|
|
51
|
-
* Polls the workflow return value until it is completed.
|
|
52
|
-
*/
|
|
53
|
-
get returnValue() {
|
|
54
|
-
return this.pollReturnValue();
|
|
55
|
-
}
|
|
56
|
-
/**
|
|
57
|
-
* The name of the workflow.
|
|
58
|
-
*/
|
|
59
|
-
get workflowName() {
|
|
60
|
-
return this.world.runs.get(this.runId).then((run) => run.workflowName);
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* The timestamp when the workflow run was created.
|
|
64
|
-
*/
|
|
65
|
-
get createdAt() {
|
|
66
|
-
return this.world.runs.get(this.runId).then((run) => run.createdAt);
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* The timestamp when the workflow run started execution.
|
|
70
|
-
* Returns undefined if the workflow has not started yet.
|
|
71
|
-
*/
|
|
72
|
-
get startedAt() {
|
|
73
|
-
return this.world.runs.get(this.runId).then((run) => run.startedAt);
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* The timestamp when the workflow run completed.
|
|
77
|
-
* Returns undefined if the workflow has not completed yet.
|
|
78
|
-
*/
|
|
79
|
-
get completedAt() {
|
|
80
|
-
return this.world.runs.get(this.runId).then((run) => run.completedAt);
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* The readable stream of the workflow run.
|
|
84
|
-
*/
|
|
85
|
-
get readable() {
|
|
86
|
-
return this.getReadable();
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Retrieves the workflow run's default readable stream, which reads chunks
|
|
90
|
-
* written to the corresponding writable stream {@link getWritable}.
|
|
91
|
-
*
|
|
92
|
-
* @param options - The options for the readable stream.
|
|
93
|
-
* @returns The `ReadableStream` for the workflow run.
|
|
94
|
-
*/
|
|
95
|
-
getReadable(options = {}) {
|
|
96
|
-
const { ops = [], global = globalThis, startIndex, namespace } = options;
|
|
97
|
-
const name = getWorkflowRunStreamId(this.runId, namespace);
|
|
98
|
-
return getExternalRevivers(global, ops).ReadableStream({
|
|
99
|
-
name,
|
|
100
|
-
startIndex,
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
|
-
/**
|
|
104
|
-
* Polls the workflow return value every 1 second until it is completed.
|
|
105
|
-
* @internal
|
|
106
|
-
* @returns The workflow return value.
|
|
107
|
-
*/
|
|
108
|
-
async pollReturnValue() {
|
|
109
|
-
while (true) {
|
|
110
|
-
try {
|
|
111
|
-
const run = await this.world.runs.get(this.runId);
|
|
112
|
-
if (run.status === 'completed') {
|
|
113
|
-
return hydrateWorkflowReturnValue(run.output, [], globalThis);
|
|
114
|
-
}
|
|
115
|
-
if (run.status === 'cancelled') {
|
|
116
|
-
throw new WorkflowRunCancelledError(this.runId);
|
|
117
|
-
}
|
|
118
|
-
if (run.status === 'failed') {
|
|
119
|
-
throw new WorkflowRunFailedError(this.runId, run.error ?? 'Unknown error');
|
|
120
|
-
}
|
|
121
|
-
throw new WorkflowRunNotCompletedError(this.runId, run.status);
|
|
122
|
-
}
|
|
123
|
-
catch (error) {
|
|
124
|
-
if (WorkflowRunNotCompletedError.is(error)) {
|
|
125
|
-
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
|
126
|
-
continue;
|
|
127
|
-
}
|
|
128
|
-
throw error;
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
/**
|
|
134
|
-
* Retrieves a `Run` object for a given run ID.
|
|
135
|
-
*
|
|
136
|
-
* @param runId - The workflow run ID obtained from {@link start}.
|
|
137
|
-
* @returns A `Run` object.
|
|
138
|
-
* @throws WorkflowRunNotFoundError if the run ID is not found.
|
|
139
|
-
*/
|
|
140
|
-
export function getRun(runId) {
|
|
141
|
-
return new Run(runId);
|
|
142
|
-
}
|
|
143
|
-
/**
|
|
144
|
-
* Loads all workflow run events by iterating through all pages of paginated results.
|
|
145
|
-
* This ensures that *all* events are loaded into memory before running the workflow.
|
|
146
|
-
* Events must be in chronological order (ascending) for proper workflow replay.
|
|
147
|
-
*/
|
|
148
|
-
async function getAllWorkflowRunEvents(runId) {
|
|
149
|
-
const allEvents = [];
|
|
150
|
-
let cursor = null;
|
|
151
|
-
let hasMore = true;
|
|
152
|
-
const world = getWorld();
|
|
153
|
-
while (hasMore) {
|
|
154
|
-
const response = await world.events.list({
|
|
155
|
-
runId,
|
|
156
|
-
pagination: {
|
|
157
|
-
sortOrder: 'asc', // Required: events must be in chronological order for replay
|
|
158
|
-
cursor: cursor ?? undefined,
|
|
159
|
-
},
|
|
160
|
-
});
|
|
161
|
-
allEvents.push(...response.data);
|
|
162
|
-
hasMore = response.hasMore;
|
|
163
|
-
cursor = response.cursor;
|
|
164
|
-
}
|
|
165
|
-
return allEvents;
|
|
166
|
-
}
|
|
21
|
+
export { getRun, Run, } from './runtime/run.js';
|
|
167
22
|
/**
|
|
168
23
|
* Function that creates a single route which handles any workflow execution
|
|
169
24
|
* request and routes to the appropriate workflow function.
|
|
@@ -173,18 +28,30 @@ async function getAllWorkflowRunEvents(runId) {
|
|
|
173
28
|
* @returns A function that can be used as a Vercel API route.
|
|
174
29
|
*/
|
|
175
30
|
export function workflowEntrypoint(workflowCode) {
|
|
176
|
-
|
|
177
|
-
|
|
31
|
+
const handler = getWorldHandlers().createQueueHandler('__wkf_workflow_', async (message_, metadata) => {
|
|
32
|
+
// Check if this is a health check message
|
|
33
|
+
// NOTE: Health check messages are intentionally unauthenticated for monitoring purposes.
|
|
34
|
+
// They only write a simple status response to a stream and do not expose sensitive data.
|
|
35
|
+
// The stream name includes a unique correlationId that must be known by the caller.
|
|
36
|
+
const healthCheck = parseHealthCheckPayload(message_);
|
|
37
|
+
if (healthCheck) {
|
|
38
|
+
await handleHealthCheckMessage(healthCheck, 'workflow');
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
41
|
+
const { runId, traceCarrier: traceContext, requestedAt, } = WorkflowInvokePayloadSchema.parse(message_);
|
|
178
42
|
// Extract the workflow name from the topic name
|
|
179
43
|
const workflowName = metadata.queueName.slice('__wkf_workflow_'.length);
|
|
44
|
+
const spanLinks = await linkToCurrentContext();
|
|
180
45
|
// Invoke user workflow within the propagated trace context
|
|
181
46
|
return await withTraceContext(traceContext, async () => {
|
|
182
47
|
const world = getWorld();
|
|
183
|
-
return trace(`WORKFLOW ${workflowName}`, async (span) => {
|
|
48
|
+
return trace(`WORKFLOW ${workflowName}`, { links: spanLinks }, async (span) => {
|
|
184
49
|
span?.setAttributes({
|
|
185
50
|
...Attribute.WorkflowName(workflowName),
|
|
186
51
|
...Attribute.WorkflowOperation('execute'),
|
|
187
52
|
...Attribute.QueueName(metadata.queueName),
|
|
53
|
+
...Attribute.QueueMessageId(metadata.messageId),
|
|
54
|
+
...getQueueOverhead({ requestedAt }),
|
|
188
55
|
});
|
|
189
56
|
// TODO: validate `workflowName` exists before consuming message?
|
|
190
57
|
span?.setAttributes({
|
|
@@ -195,15 +62,21 @@ export function workflowEntrypoint(workflowCode) {
|
|
|
195
62
|
try {
|
|
196
63
|
let workflowRun = await world.runs.get(runId);
|
|
197
64
|
if (workflowRun.status === 'pending') {
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
65
|
+
// Transition run to 'running' via event (event-sourced architecture)
|
|
66
|
+
const result = await world.events.create(runId, {
|
|
67
|
+
eventType: 'run_started',
|
|
68
|
+
specVersion: SPEC_VERSION_CURRENT,
|
|
201
69
|
});
|
|
70
|
+
// Use the run entity from the event response (no extra get call needed)
|
|
71
|
+
if (!result.run) {
|
|
72
|
+
throw new WorkflowRuntimeError(`Event creation for 'run_started' did not return the run entity for run \"${runId}\"`);
|
|
73
|
+
}
|
|
74
|
+
workflowRun = result.run;
|
|
202
75
|
}
|
|
203
76
|
// At this point, the workflow is "running" and `startedAt` should
|
|
204
77
|
// definitely be set.
|
|
205
78
|
if (!workflowRun.startedAt) {
|
|
206
|
-
throw new
|
|
79
|
+
throw new WorkflowRuntimeError(`Workflow run "${runId}" has no "startedAt" timestamp`);
|
|
207
80
|
}
|
|
208
81
|
workflowStartedAt = +workflowRun.startedAt;
|
|
209
82
|
span?.setAttributes({
|
|
@@ -224,27 +97,35 @@ export function workflowEntrypoint(workflowCode) {
|
|
|
224
97
|
const events = await getAllWorkflowRunEvents(workflowRun.runId);
|
|
225
98
|
// Check for any elapsed waits and create wait_completed events
|
|
226
99
|
const now = Date.now();
|
|
227
|
-
for (
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
100
|
+
// Pre-compute completed correlation IDs for O(n) lookup instead of O(n²)
|
|
101
|
+
const completedWaitIds = new Set(events
|
|
102
|
+
.filter((e) => e.eventType === 'wait_completed')
|
|
103
|
+
.map((e) => e.correlationId));
|
|
104
|
+
// Collect all waits that need completion
|
|
105
|
+
const waitsToComplete = events
|
|
106
|
+
.filter((e) => e.eventType === 'wait_created' &&
|
|
107
|
+
e.correlationId !== undefined &&
|
|
108
|
+
!completedWaitIds.has(e.correlationId) &&
|
|
109
|
+
now >= e.eventData.resumeAt.getTime())
|
|
110
|
+
.map((e) => ({
|
|
111
|
+
eventType: 'wait_completed',
|
|
112
|
+
specVersion: SPEC_VERSION_CURRENT,
|
|
113
|
+
correlationId: e.correlationId,
|
|
114
|
+
}));
|
|
115
|
+
// Create all wait_completed events
|
|
116
|
+
for (const waitEvent of waitsToComplete) {
|
|
117
|
+
const result = await world.events.create(runId, waitEvent);
|
|
118
|
+
// Add the event to the events array so the workflow can see it
|
|
119
|
+
events.push(result.event);
|
|
242
120
|
}
|
|
243
121
|
const result = await runWorkflow(workflowCode, workflowRun, events);
|
|
244
|
-
//
|
|
245
|
-
await world.
|
|
246
|
-
|
|
247
|
-
|
|
122
|
+
// Complete the workflow run via event (event-sourced architecture)
|
|
123
|
+
await world.events.create(runId, {
|
|
124
|
+
eventType: 'run_completed',
|
|
125
|
+
specVersion: SPEC_VERSION_CURRENT,
|
|
126
|
+
eventData: {
|
|
127
|
+
output: result,
|
|
128
|
+
},
|
|
248
129
|
});
|
|
249
130
|
span?.setAttributes({
|
|
250
131
|
...Attribute.WorkflowRunStatus('completed'),
|
|
@@ -255,128 +136,45 @@ export function workflowEntrypoint(workflowCode) {
|
|
|
255
136
|
if (WorkflowSuspension.is(err)) {
|
|
256
137
|
const suspensionMessage = buildWorkflowSuspensionMessage(runId, err.stepCount, err.hookCount, err.waitCount);
|
|
257
138
|
if (suspensionMessage) {
|
|
258
|
-
|
|
259
|
-
// console.debug(suspensionMessage);
|
|
260
|
-
}
|
|
261
|
-
// Process each operation in the queue (steps and hooks)
|
|
262
|
-
let minTimeoutSeconds = null;
|
|
263
|
-
for (const queueItem of err.steps) {
|
|
264
|
-
if (queueItem.type === 'step') {
|
|
265
|
-
// Handle step operations
|
|
266
|
-
const ops = [];
|
|
267
|
-
const dehydratedArgs = dehydrateStepArguments(queueItem.args, err.globalThis);
|
|
268
|
-
try {
|
|
269
|
-
const step = await world.steps.create(runId, {
|
|
270
|
-
stepId: queueItem.correlationId,
|
|
271
|
-
stepName: queueItem.stepName,
|
|
272
|
-
input: dehydratedArgs,
|
|
273
|
-
});
|
|
274
|
-
waitUntil(Promise.all(ops));
|
|
275
|
-
await world.queue(`__wkf_step_${queueItem.stepName}`, {
|
|
276
|
-
workflowName,
|
|
277
|
-
workflowRunId: runId,
|
|
278
|
-
workflowStartedAt,
|
|
279
|
-
stepId: step.stepId,
|
|
280
|
-
traceCarrier: await serializeTraceCarrier(),
|
|
281
|
-
}, {
|
|
282
|
-
idempotencyKey: queueItem.correlationId,
|
|
283
|
-
});
|
|
284
|
-
}
|
|
285
|
-
catch (err) {
|
|
286
|
-
if (WorkflowAPIError.is(err) && err.status === 409) {
|
|
287
|
-
// Step already exists, so we can skip it
|
|
288
|
-
console.warn(`Step "${queueItem.stepName}" with correlation ID "${queueItem.correlationId}" already exists, skipping: ${err.message}`);
|
|
289
|
-
continue;
|
|
290
|
-
}
|
|
291
|
-
throw err;
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
else if (queueItem.type === 'hook') {
|
|
295
|
-
// Handle hook operations
|
|
296
|
-
try {
|
|
297
|
-
// Create hook in database
|
|
298
|
-
const hookMetadata = typeof queueItem.metadata === 'undefined'
|
|
299
|
-
? undefined
|
|
300
|
-
: dehydrateStepArguments(queueItem.metadata, err.globalThis);
|
|
301
|
-
await world.hooks.create(runId, {
|
|
302
|
-
hookId: queueItem.correlationId,
|
|
303
|
-
token: queueItem.token,
|
|
304
|
-
metadata: hookMetadata,
|
|
305
|
-
});
|
|
306
|
-
// Create hook_created event in event log
|
|
307
|
-
await world.events.create(runId, {
|
|
308
|
-
eventType: 'hook_created',
|
|
309
|
-
correlationId: queueItem.correlationId,
|
|
310
|
-
});
|
|
311
|
-
}
|
|
312
|
-
catch (err) {
|
|
313
|
-
if (WorkflowAPIError.is(err)) {
|
|
314
|
-
if (err.status === 409) {
|
|
315
|
-
// Hook already exists (duplicate hook_id constraint), so we can skip it
|
|
316
|
-
console.warn(`Hook with correlation ID "${queueItem.correlationId}" already exists, skipping: ${err.message}`);
|
|
317
|
-
continue;
|
|
318
|
-
}
|
|
319
|
-
else if (err.status === 410) {
|
|
320
|
-
// Workflow has already completed, so no-op
|
|
321
|
-
console.warn(`Workflow run "${runId}" has already completed, skipping hook "${queueItem.correlationId}": ${err.message}`);
|
|
322
|
-
continue;
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
throw err;
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
else if (queueItem.type === 'wait') {
|
|
329
|
-
// Handle wait operations
|
|
330
|
-
try {
|
|
331
|
-
// Only create wait_created event if it hasn't been created yet
|
|
332
|
-
if (!queueItem.hasCreatedEvent) {
|
|
333
|
-
await world.events.create(runId, {
|
|
334
|
-
eventType: 'wait_created',
|
|
335
|
-
correlationId: queueItem.correlationId,
|
|
336
|
-
eventData: {
|
|
337
|
-
resumeAt: queueItem.resumeAt,
|
|
338
|
-
},
|
|
339
|
-
});
|
|
340
|
-
}
|
|
341
|
-
// Calculate how long to wait before resuming
|
|
342
|
-
const now = Date.now();
|
|
343
|
-
const resumeAtMs = queueItem.resumeAt.getTime();
|
|
344
|
-
const delayMs = Math.max(1000, resumeAtMs - now);
|
|
345
|
-
const timeoutSeconds = Math.ceil(delayMs / 1000);
|
|
346
|
-
// Track the minimum timeout across all waits
|
|
347
|
-
if (minTimeoutSeconds === null ||
|
|
348
|
-
timeoutSeconds < minTimeoutSeconds) {
|
|
349
|
-
minTimeoutSeconds = timeoutSeconds;
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
catch (err) {
|
|
353
|
-
if (WorkflowAPIError.is(err) && err.status === 409) {
|
|
354
|
-
// Wait already exists, so we can skip it
|
|
355
|
-
console.warn(`Wait with correlation ID "${queueItem.correlationId}" already exists, skipping: ${err.message}`);
|
|
356
|
-
continue;
|
|
357
|
-
}
|
|
358
|
-
throw err;
|
|
359
|
-
}
|
|
360
|
-
}
|
|
139
|
+
runtimeLogger.debug(suspensionMessage);
|
|
361
140
|
}
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
141
|
+
const result = await handleSuspension({
|
|
142
|
+
suspension: err,
|
|
143
|
+
world,
|
|
144
|
+
runId,
|
|
145
|
+
workflowName,
|
|
146
|
+
workflowStartedAt,
|
|
147
|
+
span,
|
|
365
148
|
});
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
return { timeoutSeconds: minTimeoutSeconds };
|
|
149
|
+
if (result.timeoutSeconds !== undefined) {
|
|
150
|
+
return { timeoutSeconds: result.timeoutSeconds };
|
|
369
151
|
}
|
|
370
152
|
}
|
|
371
153
|
else {
|
|
154
|
+
// NOTE: this error could be an error thrown in user code, or could also be a WorkflowRuntimeError
|
|
155
|
+
// (for instance when the event log is corrupted, this is thrown by the event consumer). We could
|
|
156
|
+
// specially handle these if needed.
|
|
372
157
|
const errorName = getErrorName(err);
|
|
373
|
-
const
|
|
158
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
159
|
+
let errorStack = getErrorStack(err);
|
|
160
|
+
// Remap error stack using source maps to show original source locations
|
|
161
|
+
if (errorStack) {
|
|
162
|
+
const parsedName = parseWorkflowName(workflowName);
|
|
163
|
+
const filename = parsedName?.path || workflowName;
|
|
164
|
+
errorStack = remapErrorStack(errorStack, filename, workflowCode);
|
|
165
|
+
}
|
|
374
166
|
console.error(`${errorName} while running "${runId}" workflow:\n\n${errorStack}`);
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
167
|
+
// Fail the workflow run via event (event-sourced architecture)
|
|
168
|
+
await world.events.create(runId, {
|
|
169
|
+
eventType: 'run_failed',
|
|
170
|
+
specVersion: SPEC_VERSION_CURRENT,
|
|
171
|
+
eventData: {
|
|
172
|
+
error: {
|
|
173
|
+
message: errorMessage,
|
|
174
|
+
stack: errorStack,
|
|
175
|
+
},
|
|
176
|
+
// TODO: include error codes when we define them
|
|
177
|
+
},
|
|
380
178
|
});
|
|
381
179
|
span?.setAttributes({
|
|
382
180
|
...Attribute.WorkflowRunStatus('failed'),
|
|
@@ -388,241 +186,9 @@ export function workflowEntrypoint(workflowCode) {
|
|
|
388
186
|
}); // End withTraceContext
|
|
389
187
|
});
|
|
390
188
|
});
|
|
189
|
+
return withHealthCheck(handler);
|
|
391
190
|
}
|
|
392
|
-
/**
|
|
393
|
-
* A single route that handles any step execution request and routes to the
|
|
394
|
-
* appropriate step function. We may eventually want to create different bundles
|
|
395
|
-
* for each step, this is temporary.
|
|
396
|
-
*/
|
|
397
|
-
export const stepEntrypoint =
|
|
398
|
-
/* @__PURE__ */ getWorldHandlers().createQueueHandler('__wkf_step_', async (message_, metadata) => {
|
|
399
|
-
const { workflowName, workflowRunId, workflowStartedAt, stepId, traceCarrier: traceContext, } = StepInvokePayloadSchema.parse(message_);
|
|
400
|
-
// Execute step within the propagated trace context
|
|
401
|
-
return await withTraceContext(traceContext, async () => {
|
|
402
|
-
// Extract the step name from the topic name
|
|
403
|
-
const stepName = metadata.queueName.slice('__wkf_step_'.length);
|
|
404
|
-
const world = getWorld();
|
|
405
|
-
return trace(`STEP ${stepName}`, async (span) => {
|
|
406
|
-
span?.setAttributes({
|
|
407
|
-
...Attribute.StepName(stepName),
|
|
408
|
-
...Attribute.StepAttempt(metadata.attempt),
|
|
409
|
-
...Attribute.QueueName(metadata.queueName),
|
|
410
|
-
});
|
|
411
|
-
const stepFn = getStepFunction(stepName);
|
|
412
|
-
if (!stepFn) {
|
|
413
|
-
throw new Error(`Step "${stepName}" not found`);
|
|
414
|
-
}
|
|
415
|
-
if (typeof stepFn !== 'function') {
|
|
416
|
-
throw new Error(`Step "${stepName}" is not a function (got ${typeof stepFn})`);
|
|
417
|
-
}
|
|
418
|
-
span?.setAttributes({
|
|
419
|
-
...Attribute.WorkflowName(workflowName),
|
|
420
|
-
...Attribute.WorkflowRunId(workflowRunId),
|
|
421
|
-
...Attribute.StepId(stepId),
|
|
422
|
-
...Attribute.StepMaxRetries(stepFn.maxRetries ?? 3),
|
|
423
|
-
...Attribute.StepTracePropagated(!!traceContext),
|
|
424
|
-
});
|
|
425
|
-
let step = await world.steps.get(workflowRunId, stepId);
|
|
426
|
-
runtimeLogger.debug('Step execution details', {
|
|
427
|
-
stepName,
|
|
428
|
-
stepId: step.stepId,
|
|
429
|
-
status: step.status,
|
|
430
|
-
attempt: step.attempt,
|
|
431
|
-
});
|
|
432
|
-
span?.setAttributes({
|
|
433
|
-
...Attribute.StepStatus(step.status),
|
|
434
|
-
});
|
|
435
|
-
// Check if the step has a `retryAfter` timestamp that hasn't been reached yet
|
|
436
|
-
const now = Date.now();
|
|
437
|
-
if (step.retryAfter && step.retryAfter.getTime() > now) {
|
|
438
|
-
const timeoutSeconds = Math.ceil((step.retryAfter.getTime() - now) / 1000);
|
|
439
|
-
span?.setAttributes({
|
|
440
|
-
...Attribute.StepRetryTimeoutSeconds(timeoutSeconds),
|
|
441
|
-
});
|
|
442
|
-
runtimeLogger.debug('Step retryAfter timestamp not yet reached', {
|
|
443
|
-
stepName,
|
|
444
|
-
stepId: step.stepId,
|
|
445
|
-
retryAfter: step.retryAfter,
|
|
446
|
-
timeoutSeconds,
|
|
447
|
-
});
|
|
448
|
-
return { timeoutSeconds };
|
|
449
|
-
}
|
|
450
|
-
let result;
|
|
451
|
-
const attempt = step.attempt + 1;
|
|
452
|
-
try {
|
|
453
|
-
if (step.status !== 'pending') {
|
|
454
|
-
// We should only be running the step if it's pending
|
|
455
|
-
// (initial state, or state set on re-try), so the step has been
|
|
456
|
-
// invoked erroneously.
|
|
457
|
-
console.error(`[Workflows] "${workflowRunId}" - Step invoked erroneously, expected status "pending", got "${step.status}" instead, skipping execution`);
|
|
458
|
-
span?.setAttributes({
|
|
459
|
-
...Attribute.StepSkipped(true),
|
|
460
|
-
...Attribute.StepSkipReason(step.status),
|
|
461
|
-
});
|
|
462
|
-
return;
|
|
463
|
-
}
|
|
464
|
-
await world.events.create(workflowRunId, {
|
|
465
|
-
eventType: 'step_started', // TODO: Replace with 'step_retrying'
|
|
466
|
-
correlationId: stepId,
|
|
467
|
-
});
|
|
468
|
-
step = await world.steps.update(workflowRunId, stepId, {
|
|
469
|
-
attempt,
|
|
470
|
-
status: 'running',
|
|
471
|
-
});
|
|
472
|
-
if (!step.startedAt) {
|
|
473
|
-
throw new WorkflowRuntimeError(`Step "${stepId}" has no "startedAt" timestamp`);
|
|
474
|
-
}
|
|
475
|
-
// Hydrate the step input arguments
|
|
476
|
-
const ops = [];
|
|
477
|
-
const args = hydrateStepArguments(step.input, ops);
|
|
478
|
-
span?.setAttributes({
|
|
479
|
-
...Attribute.StepArgumentsCount(args.length),
|
|
480
|
-
});
|
|
481
|
-
result = await contextStorage.run({
|
|
482
|
-
stepMetadata: {
|
|
483
|
-
stepId,
|
|
484
|
-
stepStartedAt: new Date(+step.startedAt),
|
|
485
|
-
attempt,
|
|
486
|
-
},
|
|
487
|
-
workflowMetadata: {
|
|
488
|
-
workflowRunId,
|
|
489
|
-
workflowStartedAt: new Date(+workflowStartedAt),
|
|
490
|
-
// TODO: there should be a getUrl method on the world interface itself. This
|
|
491
|
-
// solution only works for vercel + embedded worlds.
|
|
492
|
-
url: process.env.VERCEL_URL
|
|
493
|
-
? `https://${process.env.VERCEL_URL}`
|
|
494
|
-
: `http://localhost:${process.env.PORT || 3000}`,
|
|
495
|
-
},
|
|
496
|
-
ops,
|
|
497
|
-
}, () => stepFn(...args));
|
|
498
|
-
result = dehydrateStepReturnValue(result, ops);
|
|
499
|
-
waitUntil(Promise.all(ops));
|
|
500
|
-
// Update the event log with the step result
|
|
501
|
-
await world.events.create(workflowRunId, {
|
|
502
|
-
eventType: 'step_completed',
|
|
503
|
-
correlationId: stepId,
|
|
504
|
-
eventData: {
|
|
505
|
-
result: result,
|
|
506
|
-
},
|
|
507
|
-
});
|
|
508
|
-
await world.steps.update(workflowRunId, stepId, {
|
|
509
|
-
status: 'completed',
|
|
510
|
-
output: result,
|
|
511
|
-
});
|
|
512
|
-
span?.setAttributes({
|
|
513
|
-
...Attribute.StepStatus('completed'),
|
|
514
|
-
...Attribute.StepResultType(typeof result),
|
|
515
|
-
});
|
|
516
|
-
}
|
|
517
|
-
catch (err) {
|
|
518
|
-
span?.setAttributes({
|
|
519
|
-
...Attribute.StepErrorName(getErrorName(err)),
|
|
520
|
-
...Attribute.StepErrorMessage(String(err)),
|
|
521
|
-
});
|
|
522
|
-
if (WorkflowAPIError.is(err)) {
|
|
523
|
-
if (err.status === 410) {
|
|
524
|
-
// Workflow has already completed, so no-op
|
|
525
|
-
console.warn(`Workflow run "${workflowRunId}" has already completed, skipping step "${stepId}": ${err.message}`);
|
|
526
|
-
return;
|
|
527
|
-
}
|
|
528
|
-
}
|
|
529
|
-
if (FatalError.is(err)) {
|
|
530
|
-
const stackLines = getErrorStack(err).split('\n').slice(0, 4);
|
|
531
|
-
console.error(`[Workflows] "${workflowRunId}" - Encountered \`FatalError\` while executing step "${stepName}":\n > ${stackLines.join('\n > ')}\n\nBubbling up error to parent workflow`);
|
|
532
|
-
// Fatal error - store the error in the event log and re-invoke the workflow
|
|
533
|
-
await world.events.create(workflowRunId, {
|
|
534
|
-
eventType: 'step_failed',
|
|
535
|
-
correlationId: stepId,
|
|
536
|
-
eventData: {
|
|
537
|
-
error: String(err),
|
|
538
|
-
stack: err.stack,
|
|
539
|
-
fatal: true,
|
|
540
|
-
},
|
|
541
|
-
});
|
|
542
|
-
await world.steps.update(workflowRunId, stepId, {
|
|
543
|
-
status: 'failed',
|
|
544
|
-
error: String(err),
|
|
545
|
-
// TODO: include error codes when we define them
|
|
546
|
-
// TODO: serialize/include the error name and stack?
|
|
547
|
-
});
|
|
548
|
-
span?.setAttributes({
|
|
549
|
-
...Attribute.StepStatus('failed'),
|
|
550
|
-
...Attribute.StepFatalError(true),
|
|
551
|
-
});
|
|
552
|
-
}
|
|
553
|
-
else {
|
|
554
|
-
const maxRetries = stepFn.maxRetries ?? 3;
|
|
555
|
-
span?.setAttributes({
|
|
556
|
-
...Attribute.StepAttempt(attempt),
|
|
557
|
-
...Attribute.StepMaxRetries(maxRetries),
|
|
558
|
-
});
|
|
559
|
-
if (attempt >= maxRetries) {
|
|
560
|
-
// Max retries reached
|
|
561
|
-
const stackLines = getErrorStack(err).split('\n').slice(0, 4);
|
|
562
|
-
console.error(`[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${attempt}):\n > ${stackLines.join('\n > ')}\n\n Max retries reached\n Bubbling error to parent workflow`);
|
|
563
|
-
const errorMessage = `Step "${stepName}" failed after max retries: ${String(err)}`;
|
|
564
|
-
await world.events.create(workflowRunId, {
|
|
565
|
-
eventType: 'step_failed',
|
|
566
|
-
correlationId: stepId,
|
|
567
|
-
eventData: {
|
|
568
|
-
error: errorMessage,
|
|
569
|
-
stack: getErrorStack(err),
|
|
570
|
-
fatal: true,
|
|
571
|
-
},
|
|
572
|
-
});
|
|
573
|
-
await world.steps.update(workflowRunId, stepId, {
|
|
574
|
-
status: 'failed',
|
|
575
|
-
error: errorMessage,
|
|
576
|
-
});
|
|
577
|
-
span?.setAttributes({
|
|
578
|
-
...Attribute.StepStatus('failed'),
|
|
579
|
-
...Attribute.StepRetryExhausted(true),
|
|
580
|
-
});
|
|
581
|
-
}
|
|
582
|
-
else {
|
|
583
|
-
// Not at max retries yet - log as a retryable error
|
|
584
|
-
if (RetryableError.is(err)) {
|
|
585
|
-
console.warn(`[Workflows] "${workflowRunId}" - Encountered \`RetryableError\` while executing step "${stepName}" (attempt ${attempt}):\n > ${String(err.message)}\n\n This step has failed but will be retried`);
|
|
586
|
-
}
|
|
587
|
-
else {
|
|
588
|
-
const stackLines = getErrorStack(err).split('\n').slice(0, 4);
|
|
589
|
-
console.error(`[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${attempt}):\n > ${stackLines.join('\n > ')}\n\n This step has failed but will be retried`);
|
|
590
|
-
}
|
|
591
|
-
await world.events.create(workflowRunId, {
|
|
592
|
-
eventType: 'step_failed',
|
|
593
|
-
correlationId: stepId,
|
|
594
|
-
eventData: {
|
|
595
|
-
error: String(err),
|
|
596
|
-
stack: getErrorStack(err),
|
|
597
|
-
},
|
|
598
|
-
});
|
|
599
|
-
await world.steps.update(workflowRunId, stepId, {
|
|
600
|
-
status: 'pending', // TODO: Should be "retrying" once we have that status
|
|
601
|
-
...(RetryableError.is(err) && {
|
|
602
|
-
retryAfter: err.retryAfter,
|
|
603
|
-
}),
|
|
604
|
-
});
|
|
605
|
-
const timeoutSeconds = Math.max(1, RetryableError.is(err)
|
|
606
|
-
? Math.ceil((+err.retryAfter.getTime() - Date.now()) / 1000)
|
|
607
|
-
: 1);
|
|
608
|
-
span?.setAttributes({
|
|
609
|
-
...Attribute.StepRetryTimeoutSeconds(timeoutSeconds),
|
|
610
|
-
...Attribute.StepRetryWillRetry(true),
|
|
611
|
-
});
|
|
612
|
-
// It's a retryable error - so have the queue keep the message visible
|
|
613
|
-
// so that it gets retried.
|
|
614
|
-
return { timeoutSeconds };
|
|
615
|
-
}
|
|
616
|
-
}
|
|
617
|
-
}
|
|
618
|
-
await world.queue(`__wkf_workflow_${workflowName}`, {
|
|
619
|
-
runId: workflowRunId,
|
|
620
|
-
traceCarrier: await serializeTraceCarrier(),
|
|
621
|
-
});
|
|
622
|
-
});
|
|
623
|
-
});
|
|
624
|
-
});
|
|
625
191
|
// this is a no-op placeholder as the client is
|
|
626
192
|
// expecting this to be present but we aren't actually using it
|
|
627
193
|
export function runStep() { }
|
|
628
|
-
//# sourceMappingURL=
|
|
194
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicnVudGltZS5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9ydW50aW1lLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxvQkFBb0IsRUFBRSxNQUFNLGtCQUFrQixDQUFDO0FBQ3hELE9BQU8sRUFBRSxpQkFBaUIsRUFBRSxNQUFNLDRCQUE0QixDQUFDO0FBQy9ELE9BQU8sRUFFTCxvQkFBb0IsRUFDcEIsMkJBQTJCLEdBRTVCLE1BQU0saUJBQWlCLENBQUM7QUFDekIsT0FBTyxFQUFFLGtCQUFrQixFQUFFLE1BQU0sYUFBYSxDQUFDO0FBQ2pELE9BQU8sRUFBRSxhQUFhLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDNUMsT0FBTyxFQUNMLHVCQUF1QixFQUN2QixnQkFBZ0IsRUFDaEIsd0JBQXdCLEVBQ3hCLHVCQUF1QixFQUN2QixlQUFlLEdBQ2hCLE1BQU0sc0JBQXNCLENBQUM7QUFDOUIsT0FBTyxFQUFFLGdCQUFnQixFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFDbkUsT0FBTyxFQUFFLFFBQVEsRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBQ2hFLE9BQU8sRUFBRSxlQUFlLEVBQUUsTUFBTSxpQkFBaUIsQ0FBQztBQUNsRCxPQUFPLEtBQUssU0FBUyxNQUFNLHFDQUFxQyxDQUFDO0FBQ2pFLE9BQU8sRUFBRSxvQkFBb0IsRUFBRSxLQUFLLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxnQkFBZ0IsQ0FBQztBQUMvRSxPQUFPLEVBQUUsWUFBWSxFQUFFLGFBQWEsRUFBRSxNQUFNLFlBQVksQ0FBQztBQUN6RCxPQUFPLEVBQUUsOEJBQThCLEVBQUUsTUFBTSxXQUFXLENBQUM7QUFDM0QsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLGVBQWUsQ0FBQztBQUc1QyxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFDakQsT0FBTyxFQUlMLFdBQVcsR0FDWixNQUFNLHNCQUFzQixDQUFDO0FBQzlCLE9BQU8sRUFDTCxjQUFjLEVBQ2QsVUFBVSxFQUNWLGFBQWEsR0FDZCxNQUFNLDBCQUEwQixDQUFDO0FBQ2xDLE9BQU8sRUFBcUIsS0FBSyxFQUFFLE1BQU0sb0JBQW9CLENBQUM7QUFDOUQsT0FBTyxFQUFFLGNBQWMsRUFBRSxNQUFNLDJCQUEyQixDQUFDO0FBQzNELE9BQU8sRUFDTCxXQUFXLEVBQ1gsUUFBUSxFQUNSLGdCQUFnQixFQUNoQixRQUFRLEdBQ1QsTUFBTSxvQkFBb0IsQ0FBQztBQUM1QixPQUFPLEVBQ0wsTUFBTSxFQUNOLEdBQUcsR0FFSixNQUFNLGtCQUFrQixDQUFDO0FBRTFCOzs7Ozs7O0dBT0c7QUFDSCxNQUFNLFVBQVUsa0JBQWtCLENBQ2hDLFlBQW9CO0lBRXBCLE1BQU0sT0FBTyxHQUFHLGdCQUFnQixFQUFFLENBQUMsa0JBQWtCLENBQ25ELGlCQUFpQixFQUNqQixLQUFLLEVBQUUsUUFBUSxFQUFFLFFBQVEsRUFBRSxFQUFFO1FBQzNCLDBDQUEwQztRQUMxQyx5RkFBeUY7UUFDekYseUZBQXlGO1FBQ3pGLG9GQUFvRjtRQUNwRixNQUFNLFdBQVcsR0FBRyx1QkFBdUIsQ0FBQyxRQUFRLENBQUMsQ0FBQztRQUN0RCxJQUFJLFdBQVcsRUFBRSxDQUFDO1lBQ2hCLE1BQU0sd0JBQXdCLENBQUMsV0FBVyxFQUFFLFVBQVUsQ0FBQyxDQUFDO1lBQ3hELE9BQU87UUFDVCxDQUFDO1FBRUQsTUFBTSxFQUNKLEtBQUssRUFDTCxZQUFZLEVBQUUsWUFBWSxFQUMxQixXQUFXLEdBQ1osR0FBRywyQkFBMkIsQ0FBQyxLQUFLLENBQUMsUUFBUSxDQUFDLENBQUM7UUFDaEQsZ0RBQWdEO1FBQ2hELE1BQU0sWUFBWSxHQUFHLFFBQVEsQ0FBQyxTQUFTLENBQUMsS0FBSyxDQUFDLGlCQUFpQixDQUFDLE1BQU0sQ0FBQyxDQUFDO1FBQ3hFLE1BQU0sU0FBUyxHQUFHLE1BQU0sb0JBQW9CLEVBQUUsQ0FBQztRQUUvQywyREFBMkQ7UUFDM0QsT0FBTyxNQUFNLGdCQUFnQixDQUFDLFlBQVksRUFBRSxLQUFLLElBQUksRUFBRTtZQUNyRCxNQUFNLEtBQUssR0FBRyxRQUFRLEVBQUUsQ0FBQztZQUN6QixPQUFPLEtBQUssQ0FDVixZQUFZLFlBQVksRUFBRSxFQUMxQixFQUFFLEtBQUssRUFBRSxTQUFTLEVBQUUsRUFDcEIsS0FBSyxFQUFFLElBQUksRUFBRSxFQUFFO2dCQUNiLElBQUksRUFBRSxhQUFhLENBQUM7b0JBQ2xCLEdBQUcsU0FBUyxDQUFDLFlBQVksQ0FBQyxZQUFZLENBQUM7b0JBQ3ZDLEdBQUcsU0FBUyxDQUFDLGlCQUFpQixDQUFDLFNBQVMsQ0FBQztvQkFDekMsR0FBRyxTQUFTLENBQUMsU0FBUyxDQUFDLFFBQVEsQ0FBQyxTQUFTLENBQUM7b0JBQzFDLEdBQUcsU0FBUyxDQUFDLGNBQWMsQ0FBQyxRQUFRLENBQUMsU0FBUyxDQUFDO29CQUMvQyxHQUFHLGdCQUFnQixDQUFDLEVBQUUsV0FBVyxFQUFFLENBQUM7aUJBQ3JDLENBQUMsQ0FBQztnQkFFSCxpRUFBaUU7Z0JBRWpFLElBQUksRUFBRSxhQUFhLENBQUM7b0JBQ2xCLEdBQUcsU0FBUyxDQUFDLGFBQWEsQ0FBQyxLQUFLLENBQUM7b0JBQ2pDLEdBQUcsU0FBUyxDQUFDLHVCQUF1QixDQUFDLENBQUMsQ0FBQyxZQUFZLENBQUM7aUJBQ3JELENBQUMsQ0FBQztnQkFFSCxJQUFJLGlCQUFpQixHQUFHLENBQUMsQ0FBQyxDQUFDO2dCQUMzQixJQUFJLENBQUM7b0JBQ0gsSUFBSSxXQUFXLEdBQUcsTUFBTSxLQUFLLENBQUMsSUFBSSxDQUFDLEdBQUcsQ0FBQyxLQUFLLENBQUMsQ0FBQztvQkFFOUMsSUFBSSxXQUFXLENBQUMsTUFBTSxLQUFLLFNBQVMsRUFBRSxDQUFDO3dCQUNyQyxxRUFBcUU7d0JBQ3JFLE1BQU0sTUFBTSxHQUFHLE1BQU0sS0FBSyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsS0FBSyxFQUFFOzRCQUM5QyxTQUFTLEVBQUUsYUFBYTs0QkFDeEIsV0FBVyxFQUFFLG9CQUFvQjt5QkFDbEMsQ0FBQyxDQUFDO3dCQUNILHdFQUF3RTt3QkFDeEUsSUFBSSxDQUFDLE1BQU0sQ0FBQyxHQUFHLEVBQUUsQ0FBQzs0QkFDaEIsTUFBTSxJQUFJLG9CQUFvQixDQUM1Qiw0RUFBNEUsS0FBSyxJQUFJLENBQ3RGLENBQUM7d0JBQ0osQ0FBQzt3QkFDRCxXQUFXLEdBQUcsTUFBTSxDQUFDLEdBQUcsQ0FBQztvQkFDM0IsQ0FBQztvQkFFRCxrRUFBa0U7b0JBQ2xFLHFCQUFxQjtvQkFDckIsSUFBSSxDQUFDLFdBQVcsQ0FBQyxTQUFTLEVBQUUsQ0FBQzt3QkFDM0IsTUFBTSxJQUFJLG9CQUFvQixDQUM1QixpQkFBaUIsS0FBSyxnQ0FBZ0MsQ0FDdkQsQ0FBQztvQkFDSixDQUFDO29CQUNELGlCQUFpQixHQUFHLENBQUMsV0FBVyxDQUFDLFNBQVMsQ0FBQztvQkFFM0MsSUFBSSxFQUFFLGFBQWEsQ0FBQzt3QkFDbEIsR0FBRyxTQUFTLENBQUMsaUJBQWlCLENBQUMsV0FBVyxDQUFDLE1BQU0sQ0FBQzt3QkFDbEQsR0FBRyxTQUFTLENBQUMsaUJBQWlCLENBQUMsaUJBQWlCLENBQUM7cUJBQ2xELENBQUMsQ0FBQztvQkFFSCxJQUFJLFdBQVcsQ0FBQyxNQUFNLEtBQUssU0FBUyxFQUFFLENBQUM7d0JBQ3JDLDhEQUE4RDt3QkFDOUQsT0FBTyxDQUFDLElBQUksQ0FDVixhQUFhLEtBQUssaUJBQWlCLFdBQVcsQ0FBQyxNQUFNLGFBQWEsQ0FDbkUsQ0FBQzt3QkFFRiw4RUFBOEU7d0JBQzlFLCtFQUErRTt3QkFDL0UsZ0ZBQWdGO3dCQUNoRixtRkFBbUY7d0JBQ25GLGdEQUFnRDt3QkFFaEQsT0FBTztvQkFDVCxDQUFDO29CQUVELDZDQUE2QztvQkFDN0MsTUFBTSxNQUFNLEdBQUcsTUFBTSx1QkFBdUIsQ0FBQyxXQUFXLENBQUMsS0FBSyxDQUFDLENBQUM7b0JBRWhFLCtEQUErRDtvQkFDL0QsTUFBTSxHQUFHLEdBQUcsSUFBSSxDQUFDLEdBQUcsRUFBRSxDQUFDO29CQUV2Qix5RUFBeUU7b0JBQ3pFLE1BQU0sZ0JBQWdCLEdBQUcsSUFBSSxHQUFHLENBQzlCLE1BQU07eUJBQ0gsTUFBTSxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDLENBQUMsU0FBUyxLQUFLLGdCQUFnQixDQUFDO3lCQUMvQyxHQUFHLENBQUMsQ0FBQyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxhQUFhLENBQUMsQ0FDL0IsQ0FBQztvQkFFRix5Q0FBeUM7b0JBQ3pDLE1BQU0sZUFBZSxHQUFHLE1BQU07eUJBQzNCLE1BQU0sQ0FDTCxDQUFDLENBQUMsRUFBNkMsRUFBRSxDQUMvQyxDQUFDLENBQUMsU0FBUyxLQUFLLGNBQWM7d0JBQzlCLENBQUMsQ0FBQyxhQUFhLEtBQUssU0FBUzt3QkFDN0IsQ0FBQyxnQkFBZ0IsQ0FBQyxHQUFHLENBQUMsQ0FBQyxDQUFDLGFBQWEsQ0FBQzt3QkFDdEMsR0FBRyxJQUFLLENBQUMsQ0FBQyxTQUFTLENBQUMsUUFBaUIsQ0FBQyxPQUFPLEVBQUUsQ0FDbEQ7eUJBQ0EsR0FBRyxDQUFDLENBQUMsQ0FBQyxFQUFFLEVBQUUsQ0FBQyxDQUFDO3dCQUNYLFNBQVMsRUFBRSxnQkFBeUI7d0JBQ3BDLFdBQVcsRUFBRSxvQkFBb0I7d0JBQ2pDLGFBQWEsRUFBRSxDQUFDLENBQUMsYUFBYTtxQkFDL0IsQ0FBQyxDQUFDLENBQUM7b0JBRU4sbUNBQW1DO29CQUNuQyxLQUFLLE1BQU0sU0FBUyxJQUFJLGVBQWUsRUFBRSxDQUFDO3dCQUN4QyxNQUFNLE1BQU0sR0FBRyxNQUFNLEtBQUssQ0FBQyxNQUFNLENBQUMsTUFBTSxDQUFDLEtBQUssRUFBRSxTQUFTLENBQUMsQ0FBQzt3QkFDM0QsK0RBQStEO3dCQUMvRCxNQUFNLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFNLENBQUMsQ0FBQztvQkFDN0IsQ0FBQztvQkFFRCxNQUFNLE1BQU0sR0FBRyxNQUFNLFdBQVcsQ0FDOUIsWUFBWSxFQUNaLFdBQVcsRUFDWCxNQUFNLENBQ1AsQ0FBQztvQkFFRixtRUFBbUU7b0JBQ25FLE1BQU0sS0FBSyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsS0FBSyxFQUFFO3dCQUMvQixTQUFTLEVBQUUsZUFBZTt3QkFDMUIsV0FBVyxFQUFFLG9CQUFvQjt3QkFDakMsU0FBUyxFQUFFOzRCQUNULE1BQU0sRUFBRSxNQUFNO3lCQUNmO3FCQUNGLENBQUMsQ0FBQztvQkFFSCxJQUFJLEVBQUUsYUFBYSxDQUFDO3dCQUNsQixHQUFHLFNBQVMsQ0FBQyxpQkFBaUIsQ0FBQyxXQUFXLENBQUM7d0JBQzNDLEdBQUcsU0FBUyxDQUFDLG1CQUFtQixDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUM7cUJBQ2hELENBQUMsQ0FBQztnQkFDTCxDQUFDO2dCQUFDLE9BQU8sR0FBRyxFQUFFLENBQUM7b0JBQ2IsSUFBSSxrQkFBa0IsQ0FBQyxFQUFFLENBQUMsR0FBRyxDQUFDLEVBQUUsQ0FBQzt3QkFDL0IsTUFBTSxpQkFBaUIsR0FBRyw4QkFBOEIsQ0FDdEQsS0FBSyxFQUNMLEdBQUcsQ0FBQyxTQUFTLEVBQ2IsR0FBRyxDQUFDLFNBQVMsRUFDYixHQUFHLENBQUMsU0FBUyxDQUNkLENBQUM7d0JBQ0YsSUFBSSxpQkFBaUIsRUFBRSxDQUFDOzRCQUN0QixhQUFhLENBQUMsS0FBSyxDQUFDLGlCQUFpQixDQUFDLENBQUM7d0JBQ3pDLENBQUM7d0JBRUQsTUFBTSxNQUFNLEdBQUcsTUFBTSxnQkFBZ0IsQ0FBQzs0QkFDcEMsVUFBVSxFQUFFLEdBQUc7NEJBQ2YsS0FBSzs0QkFDTCxLQUFLOzRCQUNMLFlBQVk7NEJBQ1osaUJBQWlCOzRCQUNqQixJQUFJO3lCQUNMLENBQUMsQ0FBQzt3QkFFSCxJQUFJLE1BQU0sQ0FBQyxjQUFjLEtBQUssU0FBUyxFQUFFLENBQUM7NEJBQ3hDLE9BQU8sRUFBRSxjQUFjLEVBQUUsTUFBTSxDQUFDLGNBQWMsRUFBRSxDQUFDO3dCQUNuRCxDQUFDO29CQUNILENBQUM7eUJBQU0sQ0FBQzt3QkFDTixrR0FBa0c7d0JBQ2xHLGlHQUFpRzt3QkFDakcsb0NBQW9DO3dCQUVwQyxNQUFNLFNBQVMsR0FBRyxZQUFZLENBQUMsR0FBRyxDQUFDLENBQUM7d0JBQ3BDLE1BQU0sWUFBWSxHQUNoQixHQUFHLFlBQVksS0FBSyxDQUFDLENBQUMsQ0FBQyxHQUFHLENBQUMsT0FBTyxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUM7d0JBQ25ELElBQUksVUFBVSxHQUFHLGFBQWEsQ0FBQyxHQUFHLENBQUMsQ0FBQzt3QkFFcEMsd0VBQXdFO3dCQUN4RSxJQUFJLFVBQVUsRUFBRSxDQUFDOzRCQUNmLE1BQU0sVUFBVSxHQUFHLGlCQUFpQixDQUFDLFlBQVksQ0FBQyxDQUFDOzRCQUNuRCxNQUFNLFFBQVEsR0FBRyxVQUFVLEVBQUUsSUFBSSxJQUFJLFlBQVksQ0FBQzs0QkFDbEQsVUFBVSxHQUFHLGVBQWUsQ0FDMUIsVUFBVSxFQUNWLFFBQVEsRUFDUixZQUFZLENBQ2IsQ0FBQzt3QkFDSixDQUFDO3dCQUVELE9BQU8sQ0FBQyxLQUFLLENBQ1gsR0FBRyxTQUFTLG1CQUFtQixLQUFLLGtCQUFrQixVQUFVLEVBQUUsQ0FDbkUsQ0FBQzt3QkFDRiwrREFBK0Q7d0JBQy9ELE1BQU0sS0FBSyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsS0FBSyxFQUFFOzRCQUMvQixTQUFTLEVBQUUsWUFBWTs0QkFDdkIsV0FBVyxFQUFFLG9CQUFvQjs0QkFDakMsU0FBUyxFQUFFO2dDQUNULEtBQUssRUFBRTtvQ0FDTCxPQUFPLEVBQUUsWUFBWTtvQ0FDckIsS0FBSyxFQUFFLFVBQVU7aUNBQ2xCO2dDQUNELGdEQUFnRDs2QkFDakQ7eUJBQ0YsQ0FBQyxDQUFDO3dCQUVILElBQUksRUFBRSxhQUFhLENBQUM7NEJBQ2xCLEdBQUcsU0FBUyxDQUFDLGlCQUFpQixDQUFDLFFBQVEsQ0FBQzs0QkFDeEMsR0FBRyxTQUFTLENBQUMsaUJBQWlCLENBQUMsU0FBUyxDQUFDOzRCQUN6QyxHQUFHLFNBQVMsQ0FBQyxvQkFBb0IsQ0FBQyxNQUFNLENBQUMsR0FBRyxDQUFDLENBQUM7eUJBQy9DLENBQUMsQ0FBQztvQkFDTCxDQUFDO2dCQUNILENBQUM7WUFDSCxDQUFDLENBQ0YsQ0FBQyxDQUFDLHVCQUF1QjtRQUM1QixDQUFDLENBQUMsQ0FBQztJQUNMLENBQUMsQ0FDRixDQUFDO0lBRUYsT0FBTyxlQUFlLENBQUMsT0FBTyxDQUFDLENBQUM7QUFDbEMsQ0FBQztBQUVELCtDQUErQztBQUMvQywrREFBK0Q7QUFDL0QsTUFBTSxVQUFVLE9BQU8sS0FBSSxDQUFDIn0=
|