@outputai/core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +11 -0
- package/bin/healthcheck.mjs +36 -0
- package/bin/healthcheck.spec.js +90 -0
- package/bin/worker.sh +26 -0
- package/package.json +67 -0
- package/src/activity_integration/context.d.ts +27 -0
- package/src/activity_integration/context.js +17 -0
- package/src/activity_integration/context.spec.js +42 -0
- package/src/activity_integration/events.d.ts +7 -0
- package/src/activity_integration/events.js +10 -0
- package/src/activity_integration/index.d.ts +9 -0
- package/src/activity_integration/index.js +3 -0
- package/src/activity_integration/tracing.d.ts +32 -0
- package/src/activity_integration/tracing.js +37 -0
- package/src/async_storage.js +19 -0
- package/src/bus.js +3 -0
- package/src/consts.js +32 -0
- package/src/errors.d.ts +15 -0
- package/src/errors.js +14 -0
- package/src/hooks/index.d.ts +28 -0
- package/src/hooks/index.js +32 -0
- package/src/index.d.ts +49 -0
- package/src/index.js +4 -0
- package/src/interface/evaluation_result.d.ts +173 -0
- package/src/interface/evaluation_result.js +215 -0
- package/src/interface/evaluator.d.ts +70 -0
- package/src/interface/evaluator.js +34 -0
- package/src/interface/evaluator.spec.js +565 -0
- package/src/interface/index.d.ts +9 -0
- package/src/interface/index.js +26 -0
- package/src/interface/step.d.ts +138 -0
- package/src/interface/step.js +22 -0
- package/src/interface/types.d.ts +27 -0
- package/src/interface/validations/runtime.js +20 -0
- package/src/interface/validations/runtime.spec.js +29 -0
- package/src/interface/validations/schema_utils.js +8 -0
- package/src/interface/validations/schema_utils.spec.js +67 -0
- package/src/interface/validations/static.js +136 -0
- package/src/interface/validations/static.spec.js +366 -0
- package/src/interface/webhook.d.ts +84 -0
- package/src/interface/webhook.js +64 -0
- package/src/interface/webhook.spec.js +122 -0
- package/src/interface/workflow.d.ts +273 -0
- package/src/interface/workflow.js +128 -0
- package/src/interface/workflow.spec.js +467 -0
- package/src/interface/workflow_context.js +31 -0
- package/src/interface/workflow_utils.d.ts +76 -0
- package/src/interface/workflow_utils.js +50 -0
- package/src/interface/workflow_utils.spec.js +190 -0
- package/src/interface/zod_integration.spec.js +646 -0
- package/src/internal_activities/index.js +66 -0
- package/src/internal_activities/index.spec.js +102 -0
- package/src/logger.js +73 -0
- package/src/tracing/internal_interface.js +71 -0
- package/src/tracing/processors/local/index.js +111 -0
- package/src/tracing/processors/local/index.spec.js +149 -0
- package/src/tracing/processors/s3/configs.js +31 -0
- package/src/tracing/processors/s3/configs.spec.js +64 -0
- package/src/tracing/processors/s3/index.js +114 -0
- package/src/tracing/processors/s3/index.spec.js +153 -0
- package/src/tracing/processors/s3/redis_client.js +62 -0
- package/src/tracing/processors/s3/redis_client.spec.js +185 -0
- package/src/tracing/processors/s3/s3_client.js +27 -0
- package/src/tracing/processors/s3/s3_client.spec.js +62 -0
- package/src/tracing/tools/build_trace_tree.js +83 -0
- package/src/tracing/tools/build_trace_tree.spec.js +135 -0
- package/src/tracing/tools/utils.js +21 -0
- package/src/tracing/tools/utils.spec.js +14 -0
- package/src/tracing/trace_engine.js +97 -0
- package/src/tracing/trace_engine.spec.js +199 -0
- package/src/utils/index.d.ts +134 -0
- package/src/utils/index.js +2 -0
- package/src/utils/resolve_invocation_dir.js +34 -0
- package/src/utils/resolve_invocation_dir.spec.js +102 -0
- package/src/utils/utils.js +211 -0
- package/src/utils/utils.spec.js +448 -0
- package/src/worker/bundler_options.js +43 -0
- package/src/worker/catalog_workflow/catalog.js +114 -0
- package/src/worker/catalog_workflow/index.js +54 -0
- package/src/worker/catalog_workflow/index.spec.js +196 -0
- package/src/worker/catalog_workflow/workflow.js +24 -0
- package/src/worker/configs.js +49 -0
- package/src/worker/configs.spec.js +130 -0
- package/src/worker/index.js +89 -0
- package/src/worker/index.spec.js +177 -0
- package/src/worker/interceptors/activity.js +62 -0
- package/src/worker/interceptors/activity.spec.js +212 -0
- package/src/worker/interceptors/workflow.js +70 -0
- package/src/worker/interceptors/workflow.spec.js +167 -0
- package/src/worker/interceptors.js +10 -0
- package/src/worker/loader.js +151 -0
- package/src/worker/loader.spec.js +236 -0
- package/src/worker/loader_tools.js +132 -0
- package/src/worker/loader_tools.spec.js +156 -0
- package/src/worker/log_hooks.js +95 -0
- package/src/worker/log_hooks.spec.js +217 -0
- package/src/worker/sandboxed_utils.js +18 -0
- package/src/worker/shutdown.js +26 -0
- package/src/worker/shutdown.spec.js +82 -0
- package/src/worker/sinks.js +74 -0
- package/src/worker/start_catalog.js +36 -0
- package/src/worker/start_catalog.spec.js +118 -0
- package/src/worker/webpack_loaders/consts.js +9 -0
- package/src/worker/webpack_loaders/tools.js +548 -0
- package/src/worker/webpack_loaders/tools.spec.js +330 -0
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.js +221 -0
- package/src/worker/webpack_loaders/workflow_rewriter/collect_target_imports.spec.js +336 -0
- package/src/worker/webpack_loaders/workflow_rewriter/index.mjs +61 -0
- package/src/worker/webpack_loaders/workflow_rewriter/index.spec.js +216 -0
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.js +196 -0
- package/src/worker/webpack_loaders/workflow_rewriter/rewrite_fn_bodies.spec.js +123 -0
- package/src/worker/webpack_loaders/workflow_validator/index.mjs +205 -0
- package/src/worker/webpack_loaders/workflow_validator/index.spec.js +613 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { describe, it, expect, vi } from 'vitest';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
|
|
4
|
+
// Provide the same symbol to the module under test and to the test
|
|
5
|
+
const METADATA_ACCESS_SYMBOL = Symbol( '__metadata' );
|
|
6
|
+
vi.mock( '#consts', () => ( {
|
|
7
|
+
METADATA_ACCESS_SYMBOL
|
|
8
|
+
} ) );
|
|
9
|
+
|
|
10
|
+
const setMetadata = ( target, values ) =>
|
|
11
|
+
Object.defineProperty( target, METADATA_ACCESS_SYMBOL, { value: values, writable: false, enumerable: false, configurable: false } );
|
|
12
|
+
|
|
13
|
+
describe( 'createCatalog', () => {
|
|
14
|
+
it( 'builds catalog with activities grouped by workflow path and returns Catalog with CatalogWorkflow entries', async () => {
|
|
15
|
+
const { createCatalog } = await import( './index.js' );
|
|
16
|
+
|
|
17
|
+
const workflows = [
|
|
18
|
+
{
|
|
19
|
+
name: 'flow1',
|
|
20
|
+
path: '/flows/flow1/workflow.js',
|
|
21
|
+
description: 'desc-flow1',
|
|
22
|
+
inputSchema: z.object( { in: z.literal( 'f1' ) } ),
|
|
23
|
+
outputSchema: z.object( { out: z.literal( 'f1' ) } )
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
name: 'flow2',
|
|
27
|
+
path: '/flows/flow2/workflow.js',
|
|
28
|
+
description: 'desc-flow2',
|
|
29
|
+
inputSchema: z.object( { in: z.literal( 'f2' ) } ),
|
|
30
|
+
outputSchema: z.object( { out: z.literal( 'f2' ) } )
|
|
31
|
+
}
|
|
32
|
+
];
|
|
33
|
+
|
|
34
|
+
const activity1 = () => {};
|
|
35
|
+
setMetadata( activity1, {
|
|
36
|
+
name: 'A1',
|
|
37
|
+
path: '/flows/flow1#A1',
|
|
38
|
+
description: 'desc-a1',
|
|
39
|
+
inputSchema: z.object( { in: z.literal( 'a1' ) } ),
|
|
40
|
+
outputSchema: z.object( { out: z.literal( 'a1' ) } )
|
|
41
|
+
} );
|
|
42
|
+
|
|
43
|
+
const activity2 = () => {};
|
|
44
|
+
setMetadata( activity2, {
|
|
45
|
+
name: 'A2',
|
|
46
|
+
path: '/flows/flow1#A2',
|
|
47
|
+
description: 'desc-a2',
|
|
48
|
+
inputSchema: z.object( { in: z.literal( 'a2' ) } ),
|
|
49
|
+
outputSchema: z.object( { out: z.literal( 'a2' ) } )
|
|
50
|
+
} );
|
|
51
|
+
|
|
52
|
+
const activity3 = () => {};
|
|
53
|
+
setMetadata( activity3, {
|
|
54
|
+
name: 'B1',
|
|
55
|
+
path: '/flows/flow2#B1',
|
|
56
|
+
description: 'desc-b1',
|
|
57
|
+
inputSchema: z.object( { in: z.literal( 'b1' ) } ),
|
|
58
|
+
outputSchema: z.object( { out: z.literal( 'b1' ) } )
|
|
59
|
+
} );
|
|
60
|
+
|
|
61
|
+
const activity4 = () => {};
|
|
62
|
+
setMetadata( activity4, {
|
|
63
|
+
name: 'X',
|
|
64
|
+
path: '/other#X',
|
|
65
|
+
description: 'desc-x',
|
|
66
|
+
inputSchema: z.object( { in: z.literal( 'x' ) } ),
|
|
67
|
+
outputSchema: z.object( { out: z.literal( 'x' ) } )
|
|
68
|
+
} );
|
|
69
|
+
|
|
70
|
+
const activities = {
|
|
71
|
+
'/flows/flow1#A1': activity1,
|
|
72
|
+
'/flows/flow1#A2': activity2,
|
|
73
|
+
'/flows/flow2#B1': activity3,
|
|
74
|
+
'/other#X': activity4
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const catalog = createCatalog( { workflows, activities } );
|
|
78
|
+
|
|
79
|
+
const mapped = catalog.workflows.map( w => ( {
|
|
80
|
+
name: w.name,
|
|
81
|
+
path: w.path,
|
|
82
|
+
description: w.description,
|
|
83
|
+
inputSchema: w.inputSchema,
|
|
84
|
+
outputSchema: w.outputSchema,
|
|
85
|
+
activities: w.activities.map( a => ( {
|
|
86
|
+
name: a.name,
|
|
87
|
+
description: a.description,
|
|
88
|
+
inputSchema: a.inputSchema,
|
|
89
|
+
outputSchema: a.outputSchema
|
|
90
|
+
} ) )
|
|
91
|
+
} ) );
|
|
92
|
+
|
|
93
|
+
expect( mapped ).toEqual( [
|
|
94
|
+
{
|
|
95
|
+
name: 'flow1',
|
|
96
|
+
path: '/flows/flow1/workflow.js',
|
|
97
|
+
description: 'desc-flow1',
|
|
98
|
+
inputSchema: {
|
|
99
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
100
|
+
type: 'object',
|
|
101
|
+
properties: { in: { type: 'string', const: 'f1' } },
|
|
102
|
+
required: [ 'in' ],
|
|
103
|
+
additionalProperties: false
|
|
104
|
+
},
|
|
105
|
+
outputSchema: {
|
|
106
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
107
|
+
type: 'object',
|
|
108
|
+
properties: { out: { type: 'string', const: 'f1' } },
|
|
109
|
+
required: [ 'out' ],
|
|
110
|
+
additionalProperties: false
|
|
111
|
+
},
|
|
112
|
+
activities: [
|
|
113
|
+
{
|
|
114
|
+
name: 'A1',
|
|
115
|
+
description: 'desc-a1',
|
|
116
|
+
inputSchema: {
|
|
117
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
118
|
+
type: 'object',
|
|
119
|
+
properties: { in: { type: 'string', const: 'a1' } },
|
|
120
|
+
required: [ 'in' ],
|
|
121
|
+
additionalProperties: false
|
|
122
|
+
},
|
|
123
|
+
outputSchema: {
|
|
124
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
125
|
+
type: 'object',
|
|
126
|
+
properties: { out: { type: 'string', const: 'a1' } },
|
|
127
|
+
required: [ 'out' ],
|
|
128
|
+
additionalProperties: false
|
|
129
|
+
}
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
name: 'A2',
|
|
133
|
+
description: 'desc-a2',
|
|
134
|
+
inputSchema: {
|
|
135
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
136
|
+
type: 'object',
|
|
137
|
+
properties: { in: { type: 'string', const: 'a2' } },
|
|
138
|
+
required: [ 'in' ],
|
|
139
|
+
additionalProperties: false
|
|
140
|
+
},
|
|
141
|
+
outputSchema: {
|
|
142
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
143
|
+
type: 'object',
|
|
144
|
+
properties: { out: { type: 'string', const: 'a2' } },
|
|
145
|
+
required: [ 'out' ],
|
|
146
|
+
additionalProperties: false
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
]
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
name: 'flow2',
|
|
153
|
+
path: '/flows/flow2/workflow.js',
|
|
154
|
+
description: 'desc-flow2',
|
|
155
|
+
inputSchema: {
|
|
156
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
157
|
+
type: 'object',
|
|
158
|
+
properties: { in: { type: 'string', const: 'f2' } },
|
|
159
|
+
required: [ 'in' ],
|
|
160
|
+
additionalProperties: false
|
|
161
|
+
},
|
|
162
|
+
outputSchema: {
|
|
163
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
164
|
+
type: 'object',
|
|
165
|
+
properties: { out: { type: 'string', const: 'f2' } },
|
|
166
|
+
required: [ 'out' ],
|
|
167
|
+
additionalProperties: false
|
|
168
|
+
},
|
|
169
|
+
activities: [
|
|
170
|
+
{
|
|
171
|
+
name: 'B1',
|
|
172
|
+
description: 'desc-b1',
|
|
173
|
+
inputSchema: {
|
|
174
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
175
|
+
type: 'object',
|
|
176
|
+
properties: { in: { type: 'string', const: 'b1' } },
|
|
177
|
+
required: [ 'in' ],
|
|
178
|
+
additionalProperties: false
|
|
179
|
+
},
|
|
180
|
+
outputSchema: {
|
|
181
|
+
$schema: 'https://json-schema.org/draft/2020-12/schema',
|
|
182
|
+
type: 'object',
|
|
183
|
+
properties: { out: { type: 'string', const: 'b1' } },
|
|
184
|
+
required: [ 'out' ],
|
|
185
|
+
additionalProperties: false
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
]
|
|
189
|
+
}
|
|
190
|
+
] );
|
|
191
|
+
|
|
192
|
+
// Original inputs are not mutated
|
|
193
|
+
expect( workflows[0].path ).toBe( '/flows/flow1/workflow.js' );
|
|
194
|
+
expect( workflows[1].path ).toBe( '/flows/flow2/workflow.js' );
|
|
195
|
+
} );
|
|
196
|
+
} );
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { defineQuery, setHandler, condition, defineUpdate } from '@temporalio/workflow';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* This is a special workflow, unique to each worker, which holds the meta information of all other workflows in that worker.
|
|
5
|
+
*
|
|
6
|
+
* The information is set in the startup and is accessible via a query called 'get'.
|
|
7
|
+
*
|
|
8
|
+
* @param {object} catalog - The catalog information
|
|
9
|
+
*/
|
|
10
|
+
export default async function catalogWorkflow( catalog ) {
|
|
11
|
+
const state = { canEnd: false };
|
|
12
|
+
|
|
13
|
+
// Returns the catalog
|
|
14
|
+
setHandler( defineQuery( 'get' ), () => catalog );
|
|
15
|
+
|
|
16
|
+
// Politely respond to a ping
|
|
17
|
+
setHandler( defineQuery( 'ping' ), () => 'pong' );
|
|
18
|
+
|
|
19
|
+
// Listen to this update to complete the workflow
|
|
20
|
+
setHandler( defineUpdate( 'complete' ), () => state.canEnd = true );
|
|
21
|
+
|
|
22
|
+
// Wait indefinitely, until the state changes
|
|
23
|
+
await condition( () => state.canEnd );
|
|
24
|
+
};
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import * as z from 'zod';
|
|
2
|
+
import { isStringboolTrue } from '#utils';
|
|
3
|
+
|
|
4
|
+
class InvalidEnvVarsErrors extends Error { }
|
|
5
|
+
|
|
6
|
+
const coalesceEmptyString = v => v === '' ? undefined : v;
|
|
7
|
+
|
|
8
|
+
const envVarSchema = z.object( {
|
|
9
|
+
OUTPUT_CATALOG_ID: z.string().regex( /^[a-z0-9_.@-]+$/i ),
|
|
10
|
+
TEMPORAL_ADDRESS: z.string().default( 'localhost:7233' ),
|
|
11
|
+
TEMPORAL_API_KEY: z.string().optional(),
|
|
12
|
+
TEMPORAL_NAMESPACE: z.string().optional().default( 'default' ),
|
|
13
|
+
// Worker concurrency — tune these via env vars to adjust for your workload.
|
|
14
|
+
// Each step (API, LLM, etc.) call is one activity. Lower this to reduce memory pressure.
|
|
15
|
+
TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 40 ) ),
|
|
16
|
+
// Workflows are lightweight state machines — this can be high.
|
|
17
|
+
TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_EXECUTIONS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 200 ) ),
|
|
18
|
+
// LRU cache for sticky workflow execution. Lower values free memory faster after surges.
|
|
19
|
+
TEMPORAL_MAX_CACHED_WORKFLOWS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 1000 ) ),
|
|
20
|
+
// How aggressively the worker pulls tasks from Temporal.
|
|
21
|
+
TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_POLLS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 5 ) ),
|
|
22
|
+
TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_POLLS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 5 ) ),
|
|
23
|
+
// Activity configs
|
|
24
|
+
// How often the worker sends a heartbeat to the Temporal Service during activity execution
|
|
25
|
+
OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 2 * 60 * 1000 ) ), // 2min
|
|
26
|
+
// Whether to send activity heartbeats (enabled by default)
|
|
27
|
+
OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: z.transform( v => v === undefined ? true : isStringboolTrue( v ) ),
|
|
28
|
+
// Time to allow for hooks to flush before shutdown
|
|
29
|
+
OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY: z.preprocess( coalesceEmptyString, z.coerce.number().int().positive().default( 3000 ) )
|
|
30
|
+
} );
|
|
31
|
+
|
|
32
|
+
const { data: envVars, error } = envVarSchema.safeParse( process.env );
|
|
33
|
+
if ( error ) {
|
|
34
|
+
throw new InvalidEnvVarsErrors( z.prettifyError( error ) );
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export const address = envVars.TEMPORAL_ADDRESS;
|
|
38
|
+
export const apiKey = envVars.TEMPORAL_API_KEY;
|
|
39
|
+
export const maxConcurrentActivityTaskExecutions = envVars.TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS;
|
|
40
|
+
export const maxConcurrentWorkflowTaskExecutions = envVars.TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_EXECUTIONS;
|
|
41
|
+
export const maxCachedWorkflows = envVars.TEMPORAL_MAX_CACHED_WORKFLOWS;
|
|
42
|
+
export const maxConcurrentActivityTaskPolls = envVars.TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_POLLS;
|
|
43
|
+
export const maxConcurrentWorkflowTaskPolls = envVars.TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_POLLS;
|
|
44
|
+
export const namespace = envVars.TEMPORAL_NAMESPACE;
|
|
45
|
+
export const taskQueue = envVars.OUTPUT_CATALOG_ID;
|
|
46
|
+
export const catalogId = envVars.OUTPUT_CATALOG_ID;
|
|
47
|
+
export const activityHeartbeatIntervalMs = envVars.OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS;
|
|
48
|
+
export const activityHeartbeatEnabled = envVars.OUTPUT_ACTIVITY_HEARTBEAT_ENABLED;
|
|
49
|
+
export const processFailureShutdownDelay = envVars.OUTPUT_PROCESS_FAILURE_SHUTDOWN_DELAY;
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
const CONFIG_KEYS = [
|
|
4
|
+
'OUTPUT_CATALOG_ID',
|
|
5
|
+
'TEMPORAL_ADDRESS',
|
|
6
|
+
'TEMPORAL_API_KEY',
|
|
7
|
+
'TEMPORAL_NAMESPACE',
|
|
8
|
+
'TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS',
|
|
9
|
+
'TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_EXECUTIONS',
|
|
10
|
+
'TEMPORAL_MAX_CACHED_WORKFLOWS',
|
|
11
|
+
'TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_POLLS',
|
|
12
|
+
'TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_POLLS',
|
|
13
|
+
'OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS',
|
|
14
|
+
'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED'
|
|
15
|
+
];
|
|
16
|
+
|
|
17
|
+
const setEnv = ( overrides = {} ) => {
|
|
18
|
+
process.env.OUTPUT_CATALOG_ID = overrides.OUTPUT_CATALOG_ID ?? 'test-catalog';
|
|
19
|
+
CONFIG_KEYS.forEach( key => {
|
|
20
|
+
if ( overrides[key] !== undefined ) {
|
|
21
|
+
process.env[key] = String( overrides[key] );
|
|
22
|
+
}
|
|
23
|
+
} );
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
const clearEnv = () => {
|
|
27
|
+
CONFIG_KEYS.forEach( key => delete process.env[key] );
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
async function loadConfigs() {
|
|
31
|
+
vi.resetModules();
|
|
32
|
+
return import( './configs.js' );
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
describe( 'worker/configs', () => {
|
|
36
|
+
beforeEach( () => clearEnv() );
|
|
37
|
+
afterEach( () => clearEnv() );
|
|
38
|
+
|
|
39
|
+
it( 'throws when OUTPUT_CATALOG_ID is missing', async () => {
|
|
40
|
+
clearEnv();
|
|
41
|
+
vi.resetModules();
|
|
42
|
+
|
|
43
|
+
await expect( import( './configs.js' ) ).rejects.toThrow();
|
|
44
|
+
} );
|
|
45
|
+
|
|
46
|
+
it( 'throws when OUTPUT_CATALOG_ID does not match regex', async () => {
|
|
47
|
+
setEnv( { OUTPUT_CATALOG_ID: 'invalid space' } );
|
|
48
|
+
vi.resetModules();
|
|
49
|
+
|
|
50
|
+
await expect( import( './configs.js' ) ).rejects.toThrow();
|
|
51
|
+
} );
|
|
52
|
+
|
|
53
|
+
it( 'uses defaults when only OUTPUT_CATALOG_ID is set', async () => {
|
|
54
|
+
setEnv();
|
|
55
|
+
const configs = await loadConfigs();
|
|
56
|
+
|
|
57
|
+
expect( configs.address ).toBe( 'localhost:7233' );
|
|
58
|
+
expect( configs.namespace ).toBe( 'default' );
|
|
59
|
+
expect( configs.maxConcurrentActivityTaskExecutions ).toBe( 40 );
|
|
60
|
+
expect( configs.maxConcurrentWorkflowTaskExecutions ).toBe( 200 );
|
|
61
|
+
expect( configs.maxCachedWorkflows ).toBe( 1000 );
|
|
62
|
+
expect( configs.maxConcurrentActivityTaskPolls ).toBe( 5 );
|
|
63
|
+
expect( configs.maxConcurrentWorkflowTaskPolls ).toBe( 5 );
|
|
64
|
+
expect( configs.activityHeartbeatIntervalMs ).toBe( 2 * 60 * 1000 );
|
|
65
|
+
expect( configs.activityHeartbeatEnabled ).toBe( true );
|
|
66
|
+
expect( configs.taskQueue ).toBe( 'test-catalog' );
|
|
67
|
+
expect( configs.catalogId ).toBe( 'test-catalog' );
|
|
68
|
+
} );
|
|
69
|
+
|
|
70
|
+
it( 'treats empty string for optional number as default (preprocess)', async () => {
|
|
71
|
+
setEnv( { TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS: '' } );
|
|
72
|
+
const configs = await loadConfigs();
|
|
73
|
+
|
|
74
|
+
expect( configs.maxConcurrentActivityTaskExecutions ).toBe( 40 );
|
|
75
|
+
} );
|
|
76
|
+
|
|
77
|
+
it( 'parses custom numeric env vars', async () => {
|
|
78
|
+
setEnv( {
|
|
79
|
+
TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS: '10',
|
|
80
|
+
TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASK_EXECUTIONS: '50',
|
|
81
|
+
TEMPORAL_MAX_CACHED_WORKFLOWS: '500',
|
|
82
|
+
OUTPUT_ACTIVITY_HEARTBEAT_INTERVAL_MS: '60000'
|
|
83
|
+
} );
|
|
84
|
+
const configs = await loadConfigs();
|
|
85
|
+
|
|
86
|
+
expect( configs.maxConcurrentActivityTaskExecutions ).toBe( 10 );
|
|
87
|
+
expect( configs.maxConcurrentWorkflowTaskExecutions ).toBe( 50 );
|
|
88
|
+
expect( configs.maxCachedWorkflows ).toBe( 500 );
|
|
89
|
+
expect( configs.activityHeartbeatIntervalMs ).toBe( 60000 );
|
|
90
|
+
} );
|
|
91
|
+
|
|
92
|
+
it( 'throws when optional number is zero or negative', async () => {
|
|
93
|
+
setEnv( { TEMPORAL_MAX_CONCURRENT_ACTIVITY_TASK_EXECUTIONS: '0' } );
|
|
94
|
+
vi.resetModules();
|
|
95
|
+
|
|
96
|
+
await expect( import( './configs.js' ) ).rejects.toThrow();
|
|
97
|
+
} );
|
|
98
|
+
|
|
99
|
+
it( 'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: "true"|"1"|"on" → true', async () => {
|
|
100
|
+
for ( const val of [ 'true', '1', 'on' ] ) {
|
|
101
|
+
setEnv( { OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: val } );
|
|
102
|
+
const configs = await loadConfigs();
|
|
103
|
+
expect( configs.activityHeartbeatEnabled ).toBe( true );
|
|
104
|
+
clearEnv();
|
|
105
|
+
}
|
|
106
|
+
} );
|
|
107
|
+
|
|
108
|
+
it( 'OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: "false"|other → false, undefined → true', async () => {
|
|
109
|
+
setEnv( { OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: 'false' } );
|
|
110
|
+
const configsFalse = await loadConfigs();
|
|
111
|
+
expect( configsFalse.activityHeartbeatEnabled ).toBe( false );
|
|
112
|
+
|
|
113
|
+
setEnv( { OUTPUT_ACTIVITY_HEARTBEAT_ENABLED: '0' } );
|
|
114
|
+
const configsZero = await loadConfigs();
|
|
115
|
+
expect( configsZero.activityHeartbeatEnabled ).toBe( false );
|
|
116
|
+
|
|
117
|
+
clearEnv();
|
|
118
|
+
setEnv(); // only OUTPUT_CATALOG_ID; OUTPUT_ACTIVITY_HEARTBEAT_ENABLED absent → default true
|
|
119
|
+
const configsDefault = await loadConfigs();
|
|
120
|
+
expect( configsDefault.activityHeartbeatEnabled ).toBe( true );
|
|
121
|
+
} );
|
|
122
|
+
|
|
123
|
+
it( 'parses TEMPORAL_ADDRESS and TEMPORAL_NAMESPACE', async () => {
|
|
124
|
+
setEnv( { TEMPORAL_ADDRESS: 'temporal:7233', TEMPORAL_NAMESPACE: 'my-ns' } );
|
|
125
|
+
const configs = await loadConfigs();
|
|
126
|
+
|
|
127
|
+
expect( configs.address ).toBe( 'temporal:7233' );
|
|
128
|
+
expect( configs.namespace ).toBe( 'my-ns' );
|
|
129
|
+
} );
|
|
130
|
+
} );
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { Worker, NativeConnection } from '@temporalio/worker';
|
|
2
|
+
import * as configs from './configs.js';
|
|
3
|
+
import { loadActivities, loadHooks, loadWorkflows, createWorkflowsEntryPoint } from './loader.js';
|
|
4
|
+
import { sinks } from './sinks.js';
|
|
5
|
+
import { createCatalog } from './catalog_workflow/index.js';
|
|
6
|
+
import { init as initTracing } from '#tracing';
|
|
7
|
+
import { webpackConfigHook } from './bundler_options.js';
|
|
8
|
+
import { initInterceptors } from './interceptors.js';
|
|
9
|
+
import { createChildLogger } from '#logger';
|
|
10
|
+
import { registerShutdown } from './shutdown.js';
|
|
11
|
+
import { startCatalog } from './start_catalog.js';
|
|
12
|
+
import { messageBus } from '#bus';
|
|
13
|
+
import './log_hooks.js';
|
|
14
|
+
import { BusEventType } from '#consts';
|
|
15
|
+
|
|
16
|
+
const log = createChildLogger( 'Worker' );
|
|
17
|
+
|
|
18
|
+
// Get caller directory from command line arguments
|
|
19
|
+
const callerDir = process.argv[2];
|
|
20
|
+
|
|
21
|
+
( async () => {
|
|
22
|
+
const {
|
|
23
|
+
address,
|
|
24
|
+
apiKey,
|
|
25
|
+
namespace,
|
|
26
|
+
taskQueue,
|
|
27
|
+
maxConcurrentWorkflowTaskExecutions,
|
|
28
|
+
maxConcurrentActivityTaskExecutions,
|
|
29
|
+
maxCachedWorkflows,
|
|
30
|
+
maxConcurrentActivityTaskPolls,
|
|
31
|
+
maxConcurrentWorkflowTaskPolls
|
|
32
|
+
} = configs;
|
|
33
|
+
|
|
34
|
+
log.info( 'Loading config...', { callerDir } );
|
|
35
|
+
await loadHooks( callerDir );
|
|
36
|
+
|
|
37
|
+
log.info( 'Loading workflows...', { callerDir } );
|
|
38
|
+
const workflows = await loadWorkflows( callerDir );
|
|
39
|
+
|
|
40
|
+
log.info( 'Loading activities...', { callerDir } );
|
|
41
|
+
const activities = await loadActivities( callerDir, workflows );
|
|
42
|
+
|
|
43
|
+
log.info( 'Creating worker entry point...' );
|
|
44
|
+
const workflowsPath = createWorkflowsEntryPoint( workflows );
|
|
45
|
+
|
|
46
|
+
log.info( 'Initializing tracing...' );
|
|
47
|
+
await initTracing();
|
|
48
|
+
|
|
49
|
+
log.info( 'Creating workflows catalog...' );
|
|
50
|
+
const catalog = createCatalog( { workflows, activities } );
|
|
51
|
+
|
|
52
|
+
log.info( 'Connecting Temporal...' );
|
|
53
|
+
// Enable TLS when connecting to remote Temporal (API key present)
|
|
54
|
+
const connection = await NativeConnection.connect( { address, tls: Boolean( apiKey ), apiKey } );
|
|
55
|
+
|
|
56
|
+
log.info( 'Creating worker...' );
|
|
57
|
+
const worker = await Worker.create( {
|
|
58
|
+
connection,
|
|
59
|
+
namespace,
|
|
60
|
+
taskQueue,
|
|
61
|
+
workflowsPath,
|
|
62
|
+
activities,
|
|
63
|
+
sinks,
|
|
64
|
+
interceptors: initInterceptors( { activities, workflows } ),
|
|
65
|
+
maxConcurrentWorkflowTaskExecutions,
|
|
66
|
+
maxConcurrentActivityTaskExecutions,
|
|
67
|
+
maxCachedWorkflows,
|
|
68
|
+
maxConcurrentActivityTaskPolls,
|
|
69
|
+
maxConcurrentWorkflowTaskPolls,
|
|
70
|
+
bundlerOptions: { webpackConfigHook }
|
|
71
|
+
} );
|
|
72
|
+
|
|
73
|
+
registerShutdown( { worker, log } );
|
|
74
|
+
|
|
75
|
+
log.info( 'Running worker...' );
|
|
76
|
+
await Promise.all( [ worker.run(), startCatalog( { connection, namespace, catalog } ) ] );
|
|
77
|
+
|
|
78
|
+
log.info( 'Closing connection...' );
|
|
79
|
+
await connection.close();
|
|
80
|
+
|
|
81
|
+
log.info( 'Bye' );
|
|
82
|
+
|
|
83
|
+
process.exit( 0 );
|
|
84
|
+
} )().catch( error => {
|
|
85
|
+
log.error( 'Fatal error', { message: error.message, stack: error.stack } );
|
|
86
|
+
messageBus.emit( BusEventType.RUNTIME_ERROR, { error } );
|
|
87
|
+
log.info( `Exiting in ${configs.processFailureShutdownDelay}ms` );
|
|
88
|
+
setTimeout( () => process.exit( 1 ), configs.processFailureShutdownDelay );
|
|
89
|
+
} );
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
|
|
3
|
+
const mockLog = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
|
4
|
+
vi.mock( '#logger', () => ( { createChildLogger: () => mockLog } ) );
|
|
5
|
+
|
|
6
|
+
vi.mock( '#consts', async importOriginal => {
|
|
7
|
+
const actual = await importOriginal();
|
|
8
|
+
return { ...actual };
|
|
9
|
+
} );
|
|
10
|
+
|
|
11
|
+
vi.mock( '#tracing', () => ( { init: vi.fn().mockResolvedValue( undefined ) } ) );
|
|
12
|
+
|
|
13
|
+
const configValues = {
|
|
14
|
+
address: 'localhost:7233',
|
|
15
|
+
apiKey: undefined,
|
|
16
|
+
namespace: 'default',
|
|
17
|
+
taskQueue: 'test-queue',
|
|
18
|
+
catalogId: 'test-catalog',
|
|
19
|
+
maxConcurrentWorkflowTaskExecutions: 200,
|
|
20
|
+
maxConcurrentActivityTaskExecutions: 40,
|
|
21
|
+
maxCachedWorkflows: 1000,
|
|
22
|
+
maxConcurrentActivityTaskPolls: 5,
|
|
23
|
+
maxConcurrentWorkflowTaskPolls: 5,
|
|
24
|
+
processFailureShutdownDelay: 0
|
|
25
|
+
};
|
|
26
|
+
vi.mock( './configs.js', () => configValues );
|
|
27
|
+
|
|
28
|
+
const messageBusMock = { on: vi.fn(), emit: vi.fn() };
|
|
29
|
+
vi.mock( '#bus', () => ( { messageBus: messageBusMock } ) );
|
|
30
|
+
|
|
31
|
+
const loadWorkflowsMock = vi.fn().mockResolvedValue( [] );
|
|
32
|
+
const loadActivitiesMock = vi.fn().mockResolvedValue( {} );
|
|
33
|
+
const loadHooksMock = vi.fn().mockResolvedValue( undefined );
|
|
34
|
+
const createWorkflowsEntryPointMock = vi.fn().mockReturnValue( '/fake/workflows/path.js' );
|
|
35
|
+
vi.mock( './loader.js', () => ( {
|
|
36
|
+
loadWorkflows: loadWorkflowsMock,
|
|
37
|
+
loadActivities: loadActivitiesMock,
|
|
38
|
+
loadHooks: loadHooksMock,
|
|
39
|
+
createWorkflowsEntryPoint: createWorkflowsEntryPointMock
|
|
40
|
+
} ) );
|
|
41
|
+
|
|
42
|
+
vi.mock( './sinks.js', () => ( { sinks: {} } ) );
|
|
43
|
+
|
|
44
|
+
const createCatalogMock = vi.fn().mockReturnValue( { workflows: [], activities: {} } );
|
|
45
|
+
vi.mock( './catalog_workflow/index.js', () => ( { createCatalog: createCatalogMock } ) );
|
|
46
|
+
|
|
47
|
+
vi.mock( './bundler_options.js', () => ( { webpackConfigHook: vi.fn() } ) );
|
|
48
|
+
|
|
49
|
+
const initInterceptorsMock = vi.fn().mockReturnValue( [] );
|
|
50
|
+
vi.mock( './interceptors.js', () => ( { initInterceptors: initInterceptorsMock } ) );
|
|
51
|
+
|
|
52
|
+
const startCatalogMock = vi.fn().mockResolvedValue( undefined );
|
|
53
|
+
vi.mock( './start_catalog.js', () => ( { startCatalog: startCatalogMock } ) );
|
|
54
|
+
|
|
55
|
+
const registerShutdownMock = vi.fn();
|
|
56
|
+
vi.mock( './shutdown.js', () => ( { registerShutdown: registerShutdownMock } ) );
|
|
57
|
+
|
|
58
|
+
vi.mock( './log_hooks.js', () => ( {} ) );
|
|
59
|
+
|
|
60
|
+
const runState = { resolve: null };
|
|
61
|
+
const runPromise = new Promise( r => {
|
|
62
|
+
runState.resolve = r;
|
|
63
|
+
} );
|
|
64
|
+
const shutdownMock = vi.fn();
|
|
65
|
+
const mockConnection = { close: vi.fn().mockResolvedValue( undefined ) };
|
|
66
|
+
const mockWorker = { run: () => runPromise, shutdown: shutdownMock };
|
|
67
|
+
|
|
68
|
+
vi.mock( '@temporalio/worker', () => ( {
|
|
69
|
+
Worker: { create: vi.fn().mockResolvedValue( mockWorker ) },
|
|
70
|
+
NativeConnection: { connect: vi.fn().mockResolvedValue( mockConnection ) }
|
|
71
|
+
} ) );
|
|
72
|
+
|
|
73
|
+
describe( 'worker/index', () => {
|
|
74
|
+
const exitMock = vi.fn();
|
|
75
|
+
const originalArgv = process.argv;
|
|
76
|
+
const originalExit = process.exit;
|
|
77
|
+
|
|
78
|
+
beforeEach( () => {
|
|
79
|
+
vi.clearAllMocks();
|
|
80
|
+
process.argv = [ ...originalArgv.slice( 0, 2 ), '/test/caller/dir' ];
|
|
81
|
+
process.exit = exitMock;
|
|
82
|
+
} );
|
|
83
|
+
|
|
84
|
+
afterEach( () => {
|
|
85
|
+
process.argv = originalArgv;
|
|
86
|
+
process.exit = originalExit;
|
|
87
|
+
configValues.apiKey = undefined;
|
|
88
|
+
} );
|
|
89
|
+
|
|
90
|
+
it( 'loads configs, workflows, activities and creates worker with correct options', async () => {
|
|
91
|
+
const { Worker, NativeConnection } = await import( '@temporalio/worker' );
|
|
92
|
+
const { init: initTracing } = await import( '#tracing' );
|
|
93
|
+
|
|
94
|
+
import( './index.js' );
|
|
95
|
+
|
|
96
|
+
await vi.waitFor( () => {
|
|
97
|
+
expect( loadHooksMock ).toHaveBeenCalledWith( '/test/caller/dir' );
|
|
98
|
+
} );
|
|
99
|
+
expect( loadWorkflowsMock ).toHaveBeenCalledWith( '/test/caller/dir' );
|
|
100
|
+
expect( loadActivitiesMock ).toHaveBeenCalledWith( '/test/caller/dir', [] );
|
|
101
|
+
expect( createWorkflowsEntryPointMock ).toHaveBeenCalledWith( [] );
|
|
102
|
+
expect( initTracing ).toHaveBeenCalled();
|
|
103
|
+
expect( createCatalogMock ).toHaveBeenCalledWith( { workflows: [], activities: {} } );
|
|
104
|
+
expect( NativeConnection.connect ).toHaveBeenCalledWith( {
|
|
105
|
+
address: configValues.address,
|
|
106
|
+
tls: false,
|
|
107
|
+
apiKey: undefined
|
|
108
|
+
} );
|
|
109
|
+
expect( Worker.create ).toHaveBeenCalledWith( expect.objectContaining( {
|
|
110
|
+
namespace: configValues.namespace,
|
|
111
|
+
taskQueue: configValues.taskQueue,
|
|
112
|
+
workflowsPath: '/fake/workflows/path.js',
|
|
113
|
+
activities: {},
|
|
114
|
+
maxConcurrentWorkflowTaskExecutions: configValues.maxConcurrentWorkflowTaskExecutions,
|
|
115
|
+
maxConcurrentActivityTaskExecutions: configValues.maxConcurrentActivityTaskExecutions,
|
|
116
|
+
maxCachedWorkflows: configValues.maxCachedWorkflows,
|
|
117
|
+
maxConcurrentActivityTaskPolls: configValues.maxConcurrentActivityTaskPolls,
|
|
118
|
+
maxConcurrentWorkflowTaskPolls: configValues.maxConcurrentWorkflowTaskPolls
|
|
119
|
+
} ) );
|
|
120
|
+
expect( initInterceptorsMock ).toHaveBeenCalledWith( { activities: {}, workflows: [] } );
|
|
121
|
+
expect( registerShutdownMock ).toHaveBeenCalledWith( { worker: mockWorker, log: mockLog } );
|
|
122
|
+
expect( startCatalogMock ).toHaveBeenCalledWith( {
|
|
123
|
+
connection: mockConnection,
|
|
124
|
+
namespace: configValues.namespace,
|
|
125
|
+
catalog: { workflows: [], activities: {} }
|
|
126
|
+
} );
|
|
127
|
+
|
|
128
|
+
runState.resolve();
|
|
129
|
+
await vi.waitFor( () => {
|
|
130
|
+
expect( mockConnection.close ).toHaveBeenCalled();
|
|
131
|
+
} );
|
|
132
|
+
expect( exitMock ).toHaveBeenCalledWith( 0 );
|
|
133
|
+
} );
|
|
134
|
+
|
|
135
|
+
it( 'enables TLS when apiKey is set', async () => {
|
|
136
|
+
configValues.apiKey = 'secret';
|
|
137
|
+
vi.resetModules();
|
|
138
|
+
|
|
139
|
+
const { NativeConnection } = await import( '@temporalio/worker' );
|
|
140
|
+
import( './index.js' );
|
|
141
|
+
|
|
142
|
+
await vi.waitFor( () => {
|
|
143
|
+
expect( NativeConnection.connect ).toHaveBeenCalledWith( expect.objectContaining( {
|
|
144
|
+
tls: true,
|
|
145
|
+
apiKey: 'secret'
|
|
146
|
+
} ) );
|
|
147
|
+
} );
|
|
148
|
+
runState.resolve();
|
|
149
|
+
await vi.waitFor( () => expect( exitMock ).toHaveBeenCalled() );
|
|
150
|
+
} );
|
|
151
|
+
|
|
152
|
+
it( 'calls registerShutdown with worker and log', async () => {
|
|
153
|
+
vi.resetModules();
|
|
154
|
+
|
|
155
|
+
import( './index.js' );
|
|
156
|
+
|
|
157
|
+
await vi.waitFor( () => {
|
|
158
|
+
expect( registerShutdownMock ).toHaveBeenCalledWith( { worker: mockWorker, log: mockLog } );
|
|
159
|
+
} );
|
|
160
|
+
runState.resolve();
|
|
161
|
+
await vi.waitFor( () => expect( exitMock ).toHaveBeenCalled() );
|
|
162
|
+
} );
|
|
163
|
+
|
|
164
|
+
it( 'calls process.exit(1) on fatal error', async () => {
|
|
165
|
+
loadWorkflowsMock.mockRejectedValueOnce( new Error( 'load failed' ) );
|
|
166
|
+
vi.resetModules();
|
|
167
|
+
|
|
168
|
+
import( './index.js' );
|
|
169
|
+
|
|
170
|
+
await vi.waitFor( () => {
|
|
171
|
+
expect( mockLog.error ).toHaveBeenCalledWith( 'Fatal error', expect.any( Object ) );
|
|
172
|
+
} );
|
|
173
|
+
await vi.waitFor( () => {
|
|
174
|
+
expect( exitMock ).toHaveBeenCalledWith( 1 );
|
|
175
|
+
} );
|
|
176
|
+
} );
|
|
177
|
+
} );
|