@pgflow/core 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{CHANGELOG.md → dist/CHANGELOG.md} +6 -0
- package/package.json +8 -5
- package/__tests__/mocks/index.ts +0 -1
- package/__tests__/mocks/postgres.ts +0 -37
- package/__tests__/types/PgflowSqlClient.test-d.ts +0 -59
- package/docs/options_for_flow_and_steps.md +0 -75
- package/docs/pgflow-blob-reference-system.md +0 -179
- package/eslint.config.cjs +0 -22
- package/example-flow.mermaid +0 -5
- package/example-flow.svg +0 -1
- package/flow-lifecycle.mermaid +0 -83
- package/flow-lifecycle.svg +0 -1
- package/out-tsc/vitest/__tests__/mocks/index.d.ts +0 -2
- package/out-tsc/vitest/__tests__/mocks/index.d.ts.map +0 -1
- package/out-tsc/vitest/__tests__/mocks/postgres.d.ts +0 -15
- package/out-tsc/vitest/__tests__/mocks/postgres.d.ts.map +0 -1
- package/out-tsc/vitest/__tests__/types/PgflowSqlClient.test-d.d.ts +0 -2
- package/out-tsc/vitest/__tests__/types/PgflowSqlClient.test-d.d.ts.map +0 -1
- package/out-tsc/vitest/tsconfig.spec.tsbuildinfo +0 -1
- package/out-tsc/vitest/vite.config.d.ts +0 -3
- package/out-tsc/vitest/vite.config.d.ts.map +0 -1
- package/pkgs/core/dist/index.js +0 -54
- package/pkgs/core/dist/pkgs/core/LICENSE.md +0 -660
- package/pkgs/core/dist/pkgs/core/README.md +0 -373
- package/pkgs/dsl/dist/index.js +0 -123
- package/pkgs/dsl/dist/pkgs/dsl/README.md +0 -11
- package/pkgs/edge-worker/dist/index.js +0 -953
- package/pkgs/edge-worker/dist/index.js.map +0 -7
- package/pkgs/edge-worker/dist/pkgs/edge-worker/LICENSE.md +0 -660
- package/pkgs/edge-worker/dist/pkgs/edge-worker/README.md +0 -46
- package/pkgs/example-flows/dist/index.js +0 -152
- package/pkgs/example-flows/dist/pkgs/example-flows/README.md +0 -11
- package/project.json +0 -125
- package/prompts/architect.md +0 -87
- package/prompts/condition.md +0 -33
- package/prompts/declarative_sql.md +0 -15
- package/prompts/deps_in_payloads.md +0 -20
- package/prompts/dsl-multi-arg.ts +0 -48
- package/prompts/dsl-options.md +0 -39
- package/prompts/dsl-single-arg.ts +0 -51
- package/prompts/dsl-two-arg.ts +0 -61
- package/prompts/dsl.md +0 -119
- package/prompts/fanout_steps.md +0 -1
- package/prompts/json_schemas.md +0 -36
- package/prompts/one_shot.md +0 -286
- package/prompts/pgtap.md +0 -229
- package/prompts/sdk.md +0 -59
- package/prompts/step_types.md +0 -62
- package/prompts/versioning.md +0 -16
- package/queries/fail_permanently.sql +0 -17
- package/queries/fail_task.sql +0 -21
- package/queries/sequential.sql +0 -47
- package/queries/two_roots_left_right.sql +0 -59
- package/schema.svg +0 -1
- package/scripts/colorize-pgtap-output.awk +0 -72
- package/scripts/run-test-with-colors +0 -5
- package/scripts/watch-test +0 -7
- package/src/PgflowSqlClient.ts +0 -85
- package/src/database-types.ts +0 -759
- package/src/index.ts +0 -3
- package/src/types.ts +0 -103
- package/supabase/config.toml +0 -32
- package/supabase/seed.sql +0 -202
- package/supabase/tests/add_step/basic_step_addition.test.sql +0 -29
- package/supabase/tests/add_step/circular_dependency.test.sql +0 -21
- package/supabase/tests/add_step/flow_isolation.test.sql +0 -26
- package/supabase/tests/add_step/idempotent_step_addition.test.sql +0 -20
- package/supabase/tests/add_step/invalid_step_slug.test.sql +0 -16
- package/supabase/tests/add_step/nonexistent_dependency.test.sql +0 -16
- package/supabase/tests/add_step/nonexistent_flow.test.sql +0 -13
- package/supabase/tests/add_step/options.test.sql +0 -66
- package/supabase/tests/add_step/step_with_dependency.test.sql +0 -36
- package/supabase/tests/add_step/step_with_multiple_dependencies.test.sql +0 -46
- package/supabase/tests/complete_task/archives_message.test.sql +0 -67
- package/supabase/tests/complete_task/completes_run_if_no_more_remaining_steps.test.sql +0 -62
- package/supabase/tests/complete_task/completes_task_and_updates_dependents.test.sql +0 -64
- package/supabase/tests/complete_task/decrements_remaining_steps_if_completing_step.test.sql +0 -62
- package/supabase/tests/complete_task/saves_output_when_completing_run.test.sql +0 -57
- package/supabase/tests/create_flow/flow_creation.test.sql +0 -27
- package/supabase/tests/create_flow/idempotency_and_duplicates.test.sql +0 -26
- package/supabase/tests/create_flow/invalid_slug.test.sql +0 -13
- package/supabase/tests/create_flow/options.test.sql +0 -57
- package/supabase/tests/fail_task/exponential_backoff.test.sql +0 -70
- package/supabase/tests/fail_task/mark_as_failed_if_no_retries_available.test.sql +0 -49
- package/supabase/tests/fail_task/respects_flow_retry_settings.test.sql +0 -48
- package/supabase/tests/fail_task/respects_step_retry_settings.test.sql +0 -48
- package/supabase/tests/fail_task/retry_task_if_retries_available.test.sql +0 -39
- package/supabase/tests/is_valid_slug.test.sql +0 -72
- package/supabase/tests/poll_for_tasks/builds_proper_input_from_deps_outputs.test.sql +0 -35
- package/supabase/tests/poll_for_tasks/hides_messages.test.sql +0 -35
- package/supabase/tests/poll_for_tasks/increments_attempts_count.test.sql +0 -35
- package/supabase/tests/poll_for_tasks/multiple_task_processing.test.sql +0 -24
- package/supabase/tests/poll_for_tasks/polls_only_queued_tasks.test.sql +0 -35
- package/supabase/tests/poll_for_tasks/reads_messages.test.sql +0 -38
- package/supabase/tests/poll_for_tasks/returns_no_tasks_if_no_step_task_for_message.test.sql +0 -34
- package/supabase/tests/poll_for_tasks/returns_no_tasks_if_queue_is_empty.test.sql +0 -19
- package/supabase/tests/poll_for_tasks/returns_no_tasks_when_qty_set_to_0.test.sql +0 -22
- package/supabase/tests/poll_for_tasks/sets_vt_delay_based_on_opt_timeout.test.sql +0 -41
- package/supabase/tests/poll_for_tasks/tasks_reapppear_if_not_processed_in_time.test.sql +0 -59
- package/supabase/tests/start_flow/creates_run.test.sql +0 -24
- package/supabase/tests/start_flow/creates_step_states_for_all_steps.test.sql +0 -25
- package/supabase/tests/start_flow/creates_step_tasks_only_for_root_steps.test.sql +0 -54
- package/supabase/tests/start_flow/returns_run.test.sql +0 -24
- package/supabase/tests/start_flow/sends_messages_on_the_queue.test.sql +0 -50
- package/supabase/tests/start_flow/starts_only_root_steps.test.sql +0 -21
- package/supabase/tests/step_dsl_is_idempotent.test.sql +0 -34
- package/tsconfig.json +0 -16
- package/tsconfig.lib.json +0 -26
- package/tsconfig.spec.json +0 -35
- package/vite.config.ts +0 -57
|
@@ -1,953 +0,0 @@
|
|
|
1
|
-
// ../edge-worker/src/core/ExecutionController.ts
|
|
2
|
-
import { newQueue } from "@henrygd/queue";
|
|
3
|
-
|
|
4
|
-
// ../edge-worker/src/core/Logger.ts
|
|
5
|
-
import pino from "pino";
|
|
6
|
-
function getLogLevelFromEnv() {
|
|
7
|
-
const validLevels = [
|
|
8
|
-
"DEBUG",
|
|
9
|
-
"INFO",
|
|
10
|
-
"ERROR"
|
|
11
|
-
];
|
|
12
|
-
const logLevel = Deno.env.get("EDGE_WORKER_LOG_LEVEL")?.toUpperCase();
|
|
13
|
-
if (logLevel && !validLevels.includes(logLevel)) {
|
|
14
|
-
console.warn(`Invalid log level "${logLevel}". Using "INFO" instead.`);
|
|
15
|
-
return "info";
|
|
16
|
-
}
|
|
17
|
-
return logLevel?.toLowerCase() || "info";
|
|
18
|
-
}
|
|
19
|
-
function setupLogger(workerId) {
|
|
20
|
-
const level = getLogLevelFromEnv();
|
|
21
|
-
const loggerOptions = {
|
|
22
|
-
level,
|
|
23
|
-
formatters: {
|
|
24
|
-
bindings: () => ({ worker_id: workerId })
|
|
25
|
-
},
|
|
26
|
-
serializers: pino.stdSerializers,
|
|
27
|
-
transport: {
|
|
28
|
-
target: "pino-pretty",
|
|
29
|
-
options: {
|
|
30
|
-
colorize: true,
|
|
31
|
-
messageFormat: "[{module}] {msg}"
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
};
|
|
35
|
-
pino(loggerOptions);
|
|
36
|
-
}
|
|
37
|
-
function getLogger(module) {
|
|
38
|
-
return pino().child({ module });
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// ../edge-worker/src/core/ExecutionController.ts
|
|
42
|
-
var ExecutionController = class {
|
|
43
|
-
constructor(executorFactory, abortSignal, config) {
|
|
44
|
-
this.logger = getLogger("ExecutionController");
|
|
45
|
-
this.signal = abortSignal;
|
|
46
|
-
this.createExecutor = executorFactory;
|
|
47
|
-
this.promiseQueue = newQueue(config.maxConcurrent);
|
|
48
|
-
}
|
|
49
|
-
async start(record) {
|
|
50
|
-
const executor = this.createExecutor(record, this.signal);
|
|
51
|
-
this.logger.info(`Scheduling execution of task ${executor.msgId}`);
|
|
52
|
-
return await this.promiseQueue.add(async () => {
|
|
53
|
-
try {
|
|
54
|
-
this.logger.debug(`Executing task ${executor.msgId}...`);
|
|
55
|
-
await executor.execute();
|
|
56
|
-
this.logger.debug(`Execution successful for ${executor.msgId}`);
|
|
57
|
-
} catch (error) {
|
|
58
|
-
this.logger.error(`Execution failed for ${executor.msgId}:`, error);
|
|
59
|
-
throw error;
|
|
60
|
-
}
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
async awaitCompletion() {
|
|
64
|
-
const active = this.promiseQueue.active();
|
|
65
|
-
const all = this.promiseQueue.size();
|
|
66
|
-
this.logger.debug(
|
|
67
|
-
`Awaiting completion of all tasks... (active/all: ${active}}/${all})`
|
|
68
|
-
);
|
|
69
|
-
await this.promiseQueue.done();
|
|
70
|
-
}
|
|
71
|
-
};
|
|
72
|
-
|
|
73
|
-
// ../edge-worker/src/queue/MessageExecutor.ts
|
|
74
|
-
var AbortError = class extends Error {
|
|
75
|
-
constructor() {
|
|
76
|
-
super("Operation aborted");
|
|
77
|
-
this.name = "AbortError";
|
|
78
|
-
}
|
|
79
|
-
};
|
|
80
|
-
var MessageExecutor = class {
|
|
81
|
-
constructor(queue, record, messageHandler, signal, retryLimit, retryDelay) {
|
|
82
|
-
this.queue = queue;
|
|
83
|
-
this.record = record;
|
|
84
|
-
this.messageHandler = messageHandler;
|
|
85
|
-
this.signal = signal;
|
|
86
|
-
this.retryLimit = retryLimit;
|
|
87
|
-
this.retryDelay = retryDelay;
|
|
88
|
-
this.logger = getLogger("MessageExecutor");
|
|
89
|
-
}
|
|
90
|
-
get msgId() {
|
|
91
|
-
return this.record.msg_id;
|
|
92
|
-
}
|
|
93
|
-
async execute() {
|
|
94
|
-
try {
|
|
95
|
-
if (this.signal.aborted) {
|
|
96
|
-
throw new AbortError();
|
|
97
|
-
}
|
|
98
|
-
this.signal.throwIfAborted();
|
|
99
|
-
this.logger.debug(`Executing task ${this.msgId}...`);
|
|
100
|
-
await this.messageHandler(this.record.message);
|
|
101
|
-
this.logger.debug(
|
|
102
|
-
`Task ${this.msgId} completed successfully, archiving...`
|
|
103
|
-
);
|
|
104
|
-
await this.queue.archive(this.msgId);
|
|
105
|
-
this.logger.debug(`Archived task ${this.msgId} successfully`);
|
|
106
|
-
} catch (error) {
|
|
107
|
-
await this.handleExecutionError(error);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
/**
|
|
111
|
-
* Handles the error that occurred during execution.
|
|
112
|
-
*
|
|
113
|
-
* If the error is an AbortError, it means that the worker was aborted and stopping,
|
|
114
|
-
* the message will reappear after the visibility timeout and be picked up by another worker.
|
|
115
|
-
*
|
|
116
|
-
* Otherwise, it proceeds with retry or archiving forever.
|
|
117
|
-
*/
|
|
118
|
-
async handleExecutionError(error) {
|
|
119
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
120
|
-
this.logger.debug(`Aborted execution for ${this.msgId}`);
|
|
121
|
-
} else {
|
|
122
|
-
this.logger.debug(`Task ${this.msgId} failed with error: ${error}`);
|
|
123
|
-
await this.retryOrArchive();
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Retries the message if it is available.
|
|
128
|
-
* Otherwise, archives the message forever and stops processing it.
|
|
129
|
-
*/
|
|
130
|
-
async retryOrArchive() {
|
|
131
|
-
if (this.retryAvailable) {
|
|
132
|
-
this.logger.debug(`Retrying ${this.msgId} in ${this.retryDelay} seconds`);
|
|
133
|
-
await this.queue.setVt(this.msgId, this.retryDelay);
|
|
134
|
-
} else {
|
|
135
|
-
this.logger.debug(`Archiving ${this.msgId} forever`);
|
|
136
|
-
await this.queue.archive(this.msgId);
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
/**
|
|
140
|
-
* Returns true if the message can be retried.
|
|
141
|
-
*/
|
|
142
|
-
get retryAvailable() {
|
|
143
|
-
const readCountLimit = this.retryLimit + 1;
|
|
144
|
-
return this.record.read_ct < readCountLimit;
|
|
145
|
-
}
|
|
146
|
-
};
|
|
147
|
-
|
|
148
|
-
// ../edge-worker/src/core/Queries.ts
|
|
149
|
-
var Queries = class {
|
|
150
|
-
constructor(sql) {
|
|
151
|
-
this.sql = sql;
|
|
152
|
-
}
|
|
153
|
-
async onWorkerStarted({
|
|
154
|
-
queueName,
|
|
155
|
-
workerId,
|
|
156
|
-
edgeFunctionName
|
|
157
|
-
}) {
|
|
158
|
-
const [worker] = await this.sql`
|
|
159
|
-
INSERT INTO edge_worker.workers (queue_name, worker_id, function_name)
|
|
160
|
-
VALUES (${queueName}, ${workerId}, ${edgeFunctionName})
|
|
161
|
-
RETURNING *;
|
|
162
|
-
`;
|
|
163
|
-
return worker;
|
|
164
|
-
}
|
|
165
|
-
async onWorkerStopped(workerRow) {
|
|
166
|
-
const [worker] = await this.sql`
|
|
167
|
-
UPDATE edge_worker.workers AS w
|
|
168
|
-
SET stopped_at = clock_timestamp(), last_heartbeat_at = clock_timestamp()
|
|
169
|
-
WHERE w.worker_id = ${workerRow.worker_id}
|
|
170
|
-
RETURNING *;
|
|
171
|
-
`;
|
|
172
|
-
return worker;
|
|
173
|
-
}
|
|
174
|
-
async sendHeartbeat(workerRow) {
|
|
175
|
-
await this.sql`
|
|
176
|
-
UPDATE edge_worker.workers AS w
|
|
177
|
-
SET last_heartbeat_at = clock_timestamp()
|
|
178
|
-
WHERE w.worker_id = ${workerRow.worker_id}
|
|
179
|
-
RETURNING *;
|
|
180
|
-
`;
|
|
181
|
-
}
|
|
182
|
-
};
|
|
183
|
-
|
|
184
|
-
// ../edge-worker/src/queue/Queue.ts
|
|
185
|
-
var Queue = class {
|
|
186
|
-
constructor(sql, queueName) {
|
|
187
|
-
this.sql = sql;
|
|
188
|
-
this.queueName = queueName;
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* Creates a queue if it doesn't exist.
|
|
192
|
-
* If the queue already exists, this method does nothing.
|
|
193
|
-
*/
|
|
194
|
-
async safeCreate() {
|
|
195
|
-
return await this.sql`
|
|
196
|
-
select * from pgmq.create(${this.queueName})
|
|
197
|
-
where not exists (
|
|
198
|
-
select 1 from pgmq.list_queues() where queue_name = ${this.queueName}
|
|
199
|
-
);
|
|
200
|
-
`;
|
|
201
|
-
}
|
|
202
|
-
/**
|
|
203
|
-
* Drops a queue if it exists.
|
|
204
|
-
* If the queue doesn't exist, this method does nothing.
|
|
205
|
-
*/
|
|
206
|
-
async safeDrop() {
|
|
207
|
-
return await this.sql`
|
|
208
|
-
select * from pgmq.drop_queue(${this.queueName})
|
|
209
|
-
where exists (
|
|
210
|
-
select 1 from pgmq.list_queues() where queue_name = ${this.queueName}
|
|
211
|
-
);
|
|
212
|
-
`;
|
|
213
|
-
}
|
|
214
|
-
async archive(msgId) {
|
|
215
|
-
await this.sql`
|
|
216
|
-
SELECT pgmq.archive(queue_name => ${this.queueName}, msg_id => ${msgId}::bigint);
|
|
217
|
-
`;
|
|
218
|
-
}
|
|
219
|
-
async archiveBatch(msgIds) {
|
|
220
|
-
await this.sql`
|
|
221
|
-
SELECT pgmq.archive(queue_name => ${this.queueName}, msg_ids => ${msgIds}::bigint[]);
|
|
222
|
-
`;
|
|
223
|
-
}
|
|
224
|
-
async send(message) {
|
|
225
|
-
const msgJson = JSON.stringify(message);
|
|
226
|
-
await this.sql`
|
|
227
|
-
SELECT pgmq.send(queue_name => ${this.queueName}, msg => ${msgJson}::jsonb)
|
|
228
|
-
`;
|
|
229
|
-
}
|
|
230
|
-
async readWithPoll(batchSize = 20, visibilityTimeout = 2, maxPollSeconds = 5, pollIntervalMs = 200) {
|
|
231
|
-
return await this.sql`
|
|
232
|
-
SELECT *
|
|
233
|
-
FROM edge_worker.read_with_poll(
|
|
234
|
-
queue_name => ${this.queueName},
|
|
235
|
-
vt => ${visibilityTimeout},
|
|
236
|
-
qty => ${batchSize},
|
|
237
|
-
max_poll_seconds => ${maxPollSeconds},
|
|
238
|
-
poll_interval_ms => ${pollIntervalMs}
|
|
239
|
-
);
|
|
240
|
-
`;
|
|
241
|
-
}
|
|
242
|
-
/**
|
|
243
|
-
* Sets the visibility timeout of a message to the current time plus the given offset.
|
|
244
|
-
*
|
|
245
|
-
* This is an inlined version of the pgmq.set_vt in order to fix the bug.
|
|
246
|
-
* The original uses now() instead of clock_timestamp() which is problematic in transactions.
|
|
247
|
-
* See more details here: https://github.com/tembo-io/pgmq/issues/367
|
|
248
|
-
*
|
|
249
|
-
* The only change made is now() replaced with clock_timestamp().
|
|
250
|
-
*/
|
|
251
|
-
async setVt(msgId, vtOffsetSeconds) {
|
|
252
|
-
const records = await this.sql`
|
|
253
|
-
UPDATE ${this.sql("pgmq.q_" + this.queueName)}
|
|
254
|
-
SET vt = (clock_timestamp() + make_interval(secs => ${vtOffsetSeconds}))
|
|
255
|
-
WHERE msg_id = ${msgId}::bigint
|
|
256
|
-
RETURNING *;
|
|
257
|
-
`;
|
|
258
|
-
return records[0];
|
|
259
|
-
}
|
|
260
|
-
};
|
|
261
|
-
|
|
262
|
-
// ../edge-worker/src/queue/ReadWithPollPoller.ts
|
|
263
|
-
var ReadWithPollPoller = class {
|
|
264
|
-
constructor(queue, signal, config) {
|
|
265
|
-
this.queue = queue;
|
|
266
|
-
this.signal = signal;
|
|
267
|
-
this.config = config;
|
|
268
|
-
}
|
|
269
|
-
async poll() {
|
|
270
|
-
if (this.isAborted()) {
|
|
271
|
-
return [];
|
|
272
|
-
}
|
|
273
|
-
return await this.queue.readWithPoll(
|
|
274
|
-
this.config.batchSize,
|
|
275
|
-
this.config.visibilityTimeout,
|
|
276
|
-
this.config.maxPollSeconds,
|
|
277
|
-
this.config.pollIntervalMs
|
|
278
|
-
);
|
|
279
|
-
}
|
|
280
|
-
isAborted() {
|
|
281
|
-
return this.signal.aborted;
|
|
282
|
-
}
|
|
283
|
-
};
|
|
284
|
-
|
|
285
|
-
// ../edge-worker/src/core/Worker.ts
|
|
286
|
-
var Worker = class {
|
|
287
|
-
constructor(batchProcessor, lifecycle, sql) {
|
|
288
|
-
this.logger = getLogger("Worker");
|
|
289
|
-
this.abortController = new AbortController();
|
|
290
|
-
this.sql = sql;
|
|
291
|
-
this.lifecycle = lifecycle;
|
|
292
|
-
this.batchProcessor = batchProcessor;
|
|
293
|
-
}
|
|
294
|
-
async startOnlyOnce(workerBootstrap) {
|
|
295
|
-
if (this.lifecycle.isRunning) {
|
|
296
|
-
this.logger.debug("Worker already running, ignoring start request");
|
|
297
|
-
return;
|
|
298
|
-
}
|
|
299
|
-
await this.start(workerBootstrap);
|
|
300
|
-
}
|
|
301
|
-
async start(workerBootstrap) {
|
|
302
|
-
setupLogger(workerBootstrap.workerId);
|
|
303
|
-
try {
|
|
304
|
-
await this.lifecycle.acknowledgeStart(workerBootstrap);
|
|
305
|
-
while (this.isMainLoopActive) {
|
|
306
|
-
try {
|
|
307
|
-
await this.lifecycle.sendHeartbeat();
|
|
308
|
-
} catch (error) {
|
|
309
|
-
this.logger.error(`Error sending heartbeat: ${error}`);
|
|
310
|
-
}
|
|
311
|
-
try {
|
|
312
|
-
await this.batchProcessor.processBatch();
|
|
313
|
-
} catch (error) {
|
|
314
|
-
this.logger.error(`Error processing batch: ${error}`);
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
} catch (error) {
|
|
318
|
-
this.logger.error(`Error in worker main loop: ${error}`);
|
|
319
|
-
throw error;
|
|
320
|
-
}
|
|
321
|
-
}
|
|
322
|
-
async stop() {
|
|
323
|
-
if (this.lifecycle.isStopping || this.lifecycle.isStopped) {
|
|
324
|
-
return;
|
|
325
|
-
}
|
|
326
|
-
this.lifecycle.transitionToStopping();
|
|
327
|
-
try {
|
|
328
|
-
this.logger.info("-> Stopped accepting new messages");
|
|
329
|
-
this.abortController.abort();
|
|
330
|
-
this.logger.info("-> Waiting for pending tasks to complete...");
|
|
331
|
-
await this.batchProcessor.awaitCompletion();
|
|
332
|
-
this.logger.info("-> Pending tasks completed!");
|
|
333
|
-
this.lifecycle.acknowledgeStop();
|
|
334
|
-
this.logger.info("-> Closing SQL connection...");
|
|
335
|
-
await this.sql.end();
|
|
336
|
-
this.logger.info("-> SQL connection closed!");
|
|
337
|
-
} catch (error) {
|
|
338
|
-
this.logger.info(`Error during worker stop: ${error}`);
|
|
339
|
-
throw error;
|
|
340
|
-
}
|
|
341
|
-
}
|
|
342
|
-
get edgeFunctionName() {
|
|
343
|
-
return this.lifecycle.edgeFunctionName;
|
|
344
|
-
}
|
|
345
|
-
/**
|
|
346
|
-
* Returns true if worker state is Running and worker was not stopped
|
|
347
|
-
*/
|
|
348
|
-
get isMainLoopActive() {
|
|
349
|
-
return this.lifecycle.isRunning && !this.isAborted;
|
|
350
|
-
}
|
|
351
|
-
get isAborted() {
|
|
352
|
-
return this.abortController.signal.aborted;
|
|
353
|
-
}
|
|
354
|
-
};
|
|
355
|
-
|
|
356
|
-
// ../edge-worker/src/queue/createQueueWorker.ts
|
|
357
|
-
import postgres from "postgres";
|
|
358
|
-
|
|
359
|
-
// ../edge-worker/src/core/Heartbeat.ts
|
|
360
|
-
var Heartbeat = class {
|
|
361
|
-
constructor(interval, queries, workerRow) {
|
|
362
|
-
this.interval = interval;
|
|
363
|
-
this.queries = queries;
|
|
364
|
-
this.workerRow = workerRow;
|
|
365
|
-
this.logger = getLogger("Heartbeat");
|
|
366
|
-
this.lastHeartbeat = 0;
|
|
367
|
-
}
|
|
368
|
-
async send() {
|
|
369
|
-
const now = Date.now();
|
|
370
|
-
if (now - this.lastHeartbeat >= this.interval) {
|
|
371
|
-
await this.queries.sendHeartbeat(this.workerRow);
|
|
372
|
-
this.logger.debug("OK");
|
|
373
|
-
this.lastHeartbeat = now;
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
};
|
|
377
|
-
|
|
378
|
-
// ../edge-worker/src/core/WorkerState.ts
|
|
379
|
-
var Transitions = {
|
|
380
|
-
["created" /* Created */]: ["starting" /* Starting */],
|
|
381
|
-
["starting" /* Starting */]: ["running" /* Running */],
|
|
382
|
-
["running" /* Running */]: ["stopping" /* Stopping */],
|
|
383
|
-
["stopping" /* Stopping */]: ["stopped" /* Stopped */],
|
|
384
|
-
["stopped" /* Stopped */]: []
|
|
385
|
-
// Terminal state - no valid transitions from here
|
|
386
|
-
};
|
|
387
|
-
var TransitionError = class extends Error {
|
|
388
|
-
constructor(options) {
|
|
389
|
-
super(`Cannot transition from ${options.from} to ${options.to}`);
|
|
390
|
-
}
|
|
391
|
-
};
|
|
392
|
-
var WorkerState = class {
|
|
393
|
-
constructor() {
|
|
394
|
-
this.logger = getLogger("WorkerState");
|
|
395
|
-
this.state = "created" /* Created */;
|
|
396
|
-
}
|
|
397
|
-
get current() {
|
|
398
|
-
return this.state;
|
|
399
|
-
}
|
|
400
|
-
get isCreated() {
|
|
401
|
-
return this.state === "created" /* Created */;
|
|
402
|
-
}
|
|
403
|
-
get isStarting() {
|
|
404
|
-
return this.state === "starting" /* Starting */;
|
|
405
|
-
}
|
|
406
|
-
get isRunning() {
|
|
407
|
-
return this.state === "running" /* Running */;
|
|
408
|
-
}
|
|
409
|
-
get isStopping() {
|
|
410
|
-
return this.state === "stopping" /* Stopping */;
|
|
411
|
-
}
|
|
412
|
-
get isStopped() {
|
|
413
|
-
return this.state === "stopped" /* Stopped */;
|
|
414
|
-
}
|
|
415
|
-
transitionTo(state) {
|
|
416
|
-
this.logger.debug(
|
|
417
|
-
`[WorkerState] Starting transition to '${state}' (current state: ${this.state})`
|
|
418
|
-
);
|
|
419
|
-
if (this.state === state) {
|
|
420
|
-
return;
|
|
421
|
-
}
|
|
422
|
-
if (Transitions[this.state].includes(state)) {
|
|
423
|
-
this.state = state;
|
|
424
|
-
this.logger.debug(`[WorkerState] Transitioned to '${state}'`);
|
|
425
|
-
} else {
|
|
426
|
-
throw new TransitionError({
|
|
427
|
-
from: this.state,
|
|
428
|
-
to: state
|
|
429
|
-
});
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
};
|
|
433
|
-
|
|
434
|
-
// ../edge-worker/src/core/WorkerLifecycle.ts
|
|
435
|
-
var WorkerLifecycle = class {
|
|
436
|
-
constructor(queries, queue) {
|
|
437
|
-
this.workerState = new WorkerState();
|
|
438
|
-
this.logger = getLogger("WorkerLifecycle");
|
|
439
|
-
this.queries = queries;
|
|
440
|
-
this.queue = queue;
|
|
441
|
-
}
|
|
442
|
-
async acknowledgeStart(workerBootstrap) {
|
|
443
|
-
this.workerState.transitionTo("starting" /* Starting */);
|
|
444
|
-
this.logger.info(`Ensuring queue '${this.queue.queueName}' exists...`);
|
|
445
|
-
await this.queue.safeCreate();
|
|
446
|
-
this.workerRow = await this.queries.onWorkerStarted({
|
|
447
|
-
queueName: this.queueName,
|
|
448
|
-
...workerBootstrap
|
|
449
|
-
});
|
|
450
|
-
this.heartbeat = new Heartbeat(5e3, this.queries, this.workerRow);
|
|
451
|
-
this.workerState.transitionTo("running" /* Running */);
|
|
452
|
-
}
|
|
453
|
-
acknowledgeStop() {
|
|
454
|
-
this.workerState.transitionTo("stopping" /* Stopping */);
|
|
455
|
-
if (!this.workerRow) {
|
|
456
|
-
throw new Error("Cannot stop worker: workerRow not set");
|
|
457
|
-
}
|
|
458
|
-
try {
|
|
459
|
-
this.logger.debug("Acknowledging worker stop...");
|
|
460
|
-
this.workerState.transitionTo("stopped" /* Stopped */);
|
|
461
|
-
this.logger.debug("Worker stop acknowledged");
|
|
462
|
-
} catch (error) {
|
|
463
|
-
this.logger.debug(`Error acknowledging worker stop: ${error}`);
|
|
464
|
-
throw error;
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
get edgeFunctionName() {
|
|
468
|
-
return this.workerRow?.function_name;
|
|
469
|
-
}
|
|
470
|
-
get queueName() {
|
|
471
|
-
return this.queue.queueName;
|
|
472
|
-
}
|
|
473
|
-
async sendHeartbeat() {
|
|
474
|
-
await this.heartbeat?.send();
|
|
475
|
-
}
|
|
476
|
-
get isRunning() {
|
|
477
|
-
return this.workerState.isRunning;
|
|
478
|
-
}
|
|
479
|
-
get isStopping() {
|
|
480
|
-
return this.workerState.isStopping;
|
|
481
|
-
}
|
|
482
|
-
get isStopped() {
|
|
483
|
-
return this.workerState.isStopped;
|
|
484
|
-
}
|
|
485
|
-
transitionToStopping() {
|
|
486
|
-
this.workerState.transitionTo("stopping" /* Stopping */);
|
|
487
|
-
}
|
|
488
|
-
};
|
|
489
|
-
|
|
490
|
-
// ../edge-worker/src/core/BatchProcessor.ts
|
|
491
|
-
var BatchProcessor = class {
|
|
492
|
-
constructor(executionController, poller, signal) {
|
|
493
|
-
this.executionController = executionController;
|
|
494
|
-
this.poller = poller;
|
|
495
|
-
this.signal = signal;
|
|
496
|
-
this.logger = getLogger("BatchProcessor");
|
|
497
|
-
this.executionController = executionController;
|
|
498
|
-
this.signal = signal;
|
|
499
|
-
this.poller = poller;
|
|
500
|
-
}
|
|
501
|
-
async processBatch() {
|
|
502
|
-
this.logger.debug("Polling for new batch of messages...");
|
|
503
|
-
const messageRecords = await this.poller.poll();
|
|
504
|
-
if (this.signal.aborted) {
|
|
505
|
-
this.logger.info("Discarding messageRecords because worker is stopping");
|
|
506
|
-
return;
|
|
507
|
-
}
|
|
508
|
-
this.logger.debug(`Starting ${messageRecords.length} messages`);
|
|
509
|
-
const startPromises = messageRecords.map(
|
|
510
|
-
(message) => this.executionController.start(message)
|
|
511
|
-
);
|
|
512
|
-
await Promise.all(startPromises);
|
|
513
|
-
}
|
|
514
|
-
async awaitCompletion() {
|
|
515
|
-
return await this.executionController.awaitCompletion();
|
|
516
|
-
}
|
|
517
|
-
};
|
|
518
|
-
|
|
519
|
-
// ../edge-worker/src/queue/createQueueWorker.ts
|
|
520
|
-
function createQueueWorker(handler, config) {
|
|
521
|
-
const abortController = new AbortController();
|
|
522
|
-
const abortSignal = abortController.signal;
|
|
523
|
-
const sql = config.sql || postgres(config.connectionString || "", {
|
|
524
|
-
max: config.maxPgConnections,
|
|
525
|
-
prepare: false
|
|
526
|
-
});
|
|
527
|
-
const queue = new Queue(sql, config.queueName || "tasks");
|
|
528
|
-
const queries = new Queries(sql);
|
|
529
|
-
const lifecycle = new WorkerLifecycle(queries, queue);
|
|
530
|
-
const executorFactory = (record, signal) => {
|
|
531
|
-
return new MessageExecutor(
|
|
532
|
-
queue,
|
|
533
|
-
record,
|
|
534
|
-
handler,
|
|
535
|
-
signal,
|
|
536
|
-
config.retryLimit || 5,
|
|
537
|
-
config.retryDelay || 3
|
|
538
|
-
);
|
|
539
|
-
};
|
|
540
|
-
const poller = new ReadWithPollPoller(queue, abortSignal, {
|
|
541
|
-
batchSize: config.batchSize || config.maxConcurrent || 10,
|
|
542
|
-
maxPollSeconds: config.maxPollSeconds || 5,
|
|
543
|
-
pollIntervalMs: config.pollIntervalMs || 200,
|
|
544
|
-
visibilityTimeout: config.visibilityTimeout || 3
|
|
545
|
-
});
|
|
546
|
-
const executionController = new ExecutionController(
|
|
547
|
-
executorFactory,
|
|
548
|
-
abortSignal,
|
|
549
|
-
{
|
|
550
|
-
maxConcurrent: config.maxConcurrent || 10
|
|
551
|
-
}
|
|
552
|
-
);
|
|
553
|
-
const batchProcessor = new BatchProcessor(
|
|
554
|
-
executionController,
|
|
555
|
-
poller,
|
|
556
|
-
abortSignal
|
|
557
|
-
);
|
|
558
|
-
return new Worker(batchProcessor, lifecycle, sql);
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
// ../edge-worker/src/spawnNewEdgeFunction.ts
|
|
562
|
-
var SUPABASE_URL = Deno.env.get("SUPABASE_URL");
|
|
563
|
-
var SUPABASE_ANON_KEY = Deno.env.get("SUPABASE_ANON_KEY");
|
|
564
|
-
var logger = getLogger("spawnNewEdgeFunction");
|
|
565
|
-
async function spawnNewEdgeFunction(functionName = "edge-worker") {
|
|
566
|
-
if (!functionName) {
|
|
567
|
-
throw new Error("functionName cannot be null or empty");
|
|
568
|
-
}
|
|
569
|
-
logger.debug("Spawning a new Edge Function...");
|
|
570
|
-
const response = await fetch(`${SUPABASE_URL}/functions/v1/${functionName}`, {
|
|
571
|
-
method: "POST",
|
|
572
|
-
headers: {
|
|
573
|
-
Authorization: `Bearer ${SUPABASE_ANON_KEY}`,
|
|
574
|
-
"Content-Type": "application/json"
|
|
575
|
-
}
|
|
576
|
-
});
|
|
577
|
-
logger.debug("Edge Function spawned successfully!");
|
|
578
|
-
if (!response.ok) {
|
|
579
|
-
throw new Error(
|
|
580
|
-
`Edge function returned non-OK status: ${response.status} ${response.statusText}`
|
|
581
|
-
);
|
|
582
|
-
}
|
|
583
|
-
}
|
|
584
|
-
|
|
585
|
-
// ../edge-worker/src/EdgeWorker.ts
|
|
586
|
-
var EdgeWorker = class {
|
|
587
|
-
static {
|
|
588
|
-
this.logger = getLogger("EdgeWorker");
|
|
589
|
-
}
|
|
590
|
-
static {
|
|
591
|
-
this.wasCalled = false;
|
|
592
|
-
}
|
|
593
|
-
/**
|
|
594
|
-
* Start the EdgeWorker with the given message handler and configuration.
|
|
595
|
-
*
|
|
596
|
-
* @param handler - Function that processes each message from the queue
|
|
597
|
-
* @param config - Configuration options for the worker
|
|
598
|
-
*
|
|
599
|
-
* @example
|
|
600
|
-
* ```typescript
|
|
601
|
-
* EdgeWorker.start(handler, {
|
|
602
|
-
* // name of the queue to poll for messages
|
|
603
|
-
* queueName: 'tasks',
|
|
604
|
-
*
|
|
605
|
-
* // how many tasks are processed at the same time
|
|
606
|
-
* maxConcurrent: 10,
|
|
607
|
-
*
|
|
608
|
-
* // how many connections to the database are opened
|
|
609
|
-
* maxPgConnections: 4,
|
|
610
|
-
*
|
|
611
|
-
* // in-worker polling interval
|
|
612
|
-
* maxPollSeconds: 5,
|
|
613
|
-
*
|
|
614
|
-
* // in-database polling interval
|
|
615
|
-
* pollIntervalMs: 200,
|
|
616
|
-
*
|
|
617
|
-
* // how long to wait before retrying a failed job
|
|
618
|
-
* retryDelay: 5,
|
|
619
|
-
*
|
|
620
|
-
* // how many times to retry a failed job
|
|
621
|
-
* retryLimit: 5,
|
|
622
|
-
*
|
|
623
|
-
* // how long a job is invisible after reading
|
|
624
|
-
* // if not successful, will reappear after this time
|
|
625
|
-
* visibilityTimeout: 3,
|
|
626
|
-
* });
|
|
627
|
-
* ```
|
|
628
|
-
*/
|
|
629
|
-
static start(handler, config = {}) {
|
|
630
|
-
this.ensureFirstCall();
|
|
631
|
-
const connectionString = config.connectionString || this.getConnectionString();
|
|
632
|
-
const completeConfig = {
|
|
633
|
-
// Pass through any config options first
|
|
634
|
-
...config,
|
|
635
|
-
// Then override with defaults for missing values
|
|
636
|
-
queueName: config.queueName || "tasks",
|
|
637
|
-
maxConcurrent: config.maxConcurrent ?? 10,
|
|
638
|
-
maxPgConnections: config.maxPgConnections ?? 4,
|
|
639
|
-
maxPollSeconds: config.maxPollSeconds ?? 5,
|
|
640
|
-
pollIntervalMs: config.pollIntervalMs ?? 200,
|
|
641
|
-
retryDelay: config.retryDelay ?? 5,
|
|
642
|
-
retryLimit: config.retryLimit ?? 5,
|
|
643
|
-
visibilityTimeout: config.visibilityTimeout ?? 3,
|
|
644
|
-
// Ensure connectionString is always set
|
|
645
|
-
connectionString
|
|
646
|
-
};
|
|
647
|
-
this.setupRequestHandler(handler, completeConfig);
|
|
648
|
-
}
|
|
649
|
-
static ensureFirstCall() {
|
|
650
|
-
if (this.wasCalled) {
|
|
651
|
-
throw new Error("EdgeWorker.start() can only be called once");
|
|
652
|
-
}
|
|
653
|
-
this.wasCalled = true;
|
|
654
|
-
}
|
|
655
|
-
static getConnectionString() {
|
|
656
|
-
const connectionString = Deno.env.get("EDGE_WORKER_DB_URL");
|
|
657
|
-
if (!connectionString) {
|
|
658
|
-
const message = "EDGE_WORKER_DB_URL is not set!\nSee https://pgflow.pages.dev/edge-worker/prepare-environment/#prepare-connection-string";
|
|
659
|
-
throw new Error(message);
|
|
660
|
-
}
|
|
661
|
-
return connectionString;
|
|
662
|
-
}
|
|
663
|
-
static setupShutdownHandler(worker) {
|
|
664
|
-
globalThis.onbeforeunload = async () => {
|
|
665
|
-
if (worker.edgeFunctionName) {
|
|
666
|
-
await spawnNewEdgeFunction(worker.edgeFunctionName);
|
|
667
|
-
}
|
|
668
|
-
worker.stop();
|
|
669
|
-
};
|
|
670
|
-
EdgeRuntime.waitUntil(new Promise(() => {
|
|
671
|
-
}));
|
|
672
|
-
}
|
|
673
|
-
static setupRequestHandler(handler, workerConfig) {
|
|
674
|
-
let worker = null;
|
|
675
|
-
Deno.serve({}, (req) => {
|
|
676
|
-
if (!worker) {
|
|
677
|
-
const edgeFunctionName = this.extractFunctionName(req);
|
|
678
|
-
const sbExecutionId = Deno.env.get("SB_EXECUTION_ID");
|
|
679
|
-
setupLogger(sbExecutionId);
|
|
680
|
-
this.logger.info(`HTTP Request: ${edgeFunctionName}`);
|
|
681
|
-
worker = createQueueWorker(handler, workerConfig);
|
|
682
|
-
worker.startOnlyOnce({
|
|
683
|
-
edgeFunctionName,
|
|
684
|
-
workerId: sbExecutionId
|
|
685
|
-
});
|
|
686
|
-
this.setupShutdownHandler(worker);
|
|
687
|
-
}
|
|
688
|
-
return new Response("ok", {
|
|
689
|
-
headers: { "Content-Type": "application/json" }
|
|
690
|
-
});
|
|
691
|
-
});
|
|
692
|
-
}
|
|
693
|
-
static extractFunctionName(req) {
|
|
694
|
-
return new URL(req.url).pathname.replace(/^\/+|\/+$/g, "");
|
|
695
|
-
}
|
|
696
|
-
};
|
|
697
|
-
|
|
698
|
-
// ../edge-worker/src/flow/StepTaskPoller.ts
|
|
699
|
-
var StepTaskPoller = class {
|
|
700
|
-
constructor(adapter, signal, config) {
|
|
701
|
-
this.adapter = adapter;
|
|
702
|
-
this.signal = signal;
|
|
703
|
-
this.config = config;
|
|
704
|
-
this.logger = getLogger("StepTaskPoller");
|
|
705
|
-
}
|
|
706
|
-
async poll() {
|
|
707
|
-
if (this.isAborted()) {
|
|
708
|
-
this.logger.debug("Polling aborted, returning empty array");
|
|
709
|
-
return [];
|
|
710
|
-
}
|
|
711
|
-
this.logger.debug(
|
|
712
|
-
`Polling for flow tasks with batch size ${this.config.batchSize}`
|
|
713
|
-
);
|
|
714
|
-
try {
|
|
715
|
-
const tasks = await this.adapter.pollForTasks(
|
|
716
|
-
this.config.queueName,
|
|
717
|
-
this.config.batchSize
|
|
718
|
-
);
|
|
719
|
-
this.logger.debug(`Retrieved ${tasks.length} flow tasks`);
|
|
720
|
-
return tasks;
|
|
721
|
-
} catch (err) {
|
|
722
|
-
this.logger.error(`Error polling for flow tasks: ${err}`);
|
|
723
|
-
return [];
|
|
724
|
-
}
|
|
725
|
-
}
|
|
726
|
-
isAborted() {
|
|
727
|
-
return this.signal.aborted;
|
|
728
|
-
}
|
|
729
|
-
};
|
|
730
|
-
|
|
731
|
-
// ../edge-worker/src/flow/StepTaskExecutor.ts
|
|
732
|
-
var AbortError2 = class extends Error {
|
|
733
|
-
constructor() {
|
|
734
|
-
super("Operation aborted");
|
|
735
|
-
this.name = "AbortError";
|
|
736
|
-
}
|
|
737
|
-
};
|
|
738
|
-
var StepTaskExecutor = class {
|
|
739
|
-
constructor(flow, task, adapter, signal) {
|
|
740
|
-
this.flow = flow;
|
|
741
|
-
this.task = task;
|
|
742
|
-
this.adapter = adapter;
|
|
743
|
-
this.signal = signal;
|
|
744
|
-
this.logger = getLogger("StepTaskExecutor");
|
|
745
|
-
}
|
|
746
|
-
get msgId() {
|
|
747
|
-
return this.task.msg_id;
|
|
748
|
-
}
|
|
749
|
-
async execute() {
|
|
750
|
-
try {
|
|
751
|
-
if (this.signal.aborted) {
|
|
752
|
-
throw new AbortError2();
|
|
753
|
-
}
|
|
754
|
-
this.signal.throwIfAborted();
|
|
755
|
-
const stepSlug = this.task.step_slug;
|
|
756
|
-
this.logger.debug(
|
|
757
|
-
`Executing step task ${this.task.msg_id} for step ${stepSlug}`
|
|
758
|
-
);
|
|
759
|
-
const stepDef = this.flow.getStepDefinition(stepSlug);
|
|
760
|
-
if (!stepDef) {
|
|
761
|
-
throw new Error(`No step definition found for slug=${stepSlug}`);
|
|
762
|
-
}
|
|
763
|
-
const result = await stepDef.handler(this.task.input);
|
|
764
|
-
this.logger.debug(
|
|
765
|
-
`step task ${this.task.msg_id} completed successfully, marking as complete`
|
|
766
|
-
);
|
|
767
|
-
await this.adapter.completeTask(this.task, result);
|
|
768
|
-
this.logger.debug(`step task ${this.task.msg_id} marked as complete`);
|
|
769
|
-
} catch (error) {
|
|
770
|
-
await this.handleExecutionError(error);
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
/**
|
|
774
|
-
* Handles the error that occurred during execution.
|
|
775
|
-
*
|
|
776
|
-
* If the error is an AbortError, it means that the worker was aborted and stopping,
|
|
777
|
-
* the task will be picked up by another worker later.
|
|
778
|
-
*
|
|
779
|
-
* Otherwise, it marks the task as failed.
|
|
780
|
-
*/
|
|
781
|
-
async handleExecutionError(error) {
|
|
782
|
-
if (error instanceof Error && error.name === "AbortError") {
|
|
783
|
-
this.logger.debug(`Aborted execution for step task ${this.task.msg_id}`);
|
|
784
|
-
} else {
|
|
785
|
-
this.logger.error(
|
|
786
|
-
`step task ${this.task.msg_id} failed with error: ${error}`
|
|
787
|
-
);
|
|
788
|
-
await this.adapter.failTask(this.task, error);
|
|
789
|
-
}
|
|
790
|
-
}
|
|
791
|
-
};
|
|
792
|
-
|
|
793
|
-
// src/PgflowSqlClient.ts
|
|
794
|
-
var PgflowSqlClient = class {
|
|
795
|
-
constructor(sql) {
|
|
796
|
-
this.sql = sql;
|
|
797
|
-
}
|
|
798
|
-
async pollForTasks(queueName, batchSize = 20, visibilityTimeout = 2, maxPollSeconds = 5, pollIntervalMs = 200) {
|
|
799
|
-
return await this.sql`
|
|
800
|
-
SELECT *
|
|
801
|
-
FROM pgflow.poll_for_tasks(
|
|
802
|
-
queue_name => ${queueName},
|
|
803
|
-
vt => ${visibilityTimeout},
|
|
804
|
-
qty => ${batchSize},
|
|
805
|
-
max_poll_seconds => ${maxPollSeconds},
|
|
806
|
-
poll_interval_ms => ${pollIntervalMs}
|
|
807
|
-
);
|
|
808
|
-
`;
|
|
809
|
-
}
|
|
810
|
-
async completeTask(stepTask, output) {
|
|
811
|
-
await this.sql`
|
|
812
|
-
SELECT pgflow.complete_task(
|
|
813
|
-
run_id => ${stepTask.run_id}::uuid,
|
|
814
|
-
step_slug => ${stepTask.step_slug}::text,
|
|
815
|
-
task_index => ${0}::int,
|
|
816
|
-
output => ${this.sql.json(output || null)}::jsonb
|
|
817
|
-
);
|
|
818
|
-
`;
|
|
819
|
-
}
|
|
820
|
-
async failTask(stepTask, error) {
|
|
821
|
-
const errorString = typeof error === "string" ? error : error instanceof Error ? error.message : JSON.stringify(error);
|
|
822
|
-
await this.sql`
|
|
823
|
-
SELECT pgflow.fail_task(
|
|
824
|
-
run_id => ${stepTask.run_id}::uuid,
|
|
825
|
-
step_slug => ${stepTask.step_slug}::text,
|
|
826
|
-
task_index => ${0}::int,
|
|
827
|
-
error_message => ${errorString}::text
|
|
828
|
-
);
|
|
829
|
-
`;
|
|
830
|
-
}
|
|
831
|
-
async startFlow(flow, input) {
|
|
832
|
-
const results = await this.sql`
|
|
833
|
-
SELECT * FROM pgflow.start_flow(${flow.slug}::text, ${this.sql.json(
|
|
834
|
-
input
|
|
835
|
-
)}::jsonb);
|
|
836
|
-
`;
|
|
837
|
-
if (results.length === 0) {
|
|
838
|
-
throw new Error(`Failed to start flow ${flow.slug}`);
|
|
839
|
-
}
|
|
840
|
-
const [flowRun] = results;
|
|
841
|
-
return flowRun;
|
|
842
|
-
}
|
|
843
|
-
};
|
|
844
|
-
|
|
845
|
-
// ../edge-worker/src/flow/createFlowWorker.ts
|
|
846
|
-
import postgres2 from "postgres";
|
|
847
|
-
|
|
848
|
-
// ../edge-worker/src/flow/FlowWorkerLifecycle.ts
|
|
849
|
-
var FlowWorkerLifecycle = class {
|
|
850
|
-
constructor(queries, flow) {
|
|
851
|
-
this.workerState = new WorkerState();
|
|
852
|
-
this.logger = getLogger("FlowWorkerLifecycle");
|
|
853
|
-
this.queries = queries;
|
|
854
|
-
this.flow = flow;
|
|
855
|
-
}
|
|
856
|
-
async acknowledgeStart(workerBootstrap) {
|
|
857
|
-
this.workerState.transitionTo("starting" /* Starting */);
|
|
858
|
-
this.workerRow = await this.queries.onWorkerStarted({
|
|
859
|
-
queueName: this.queueName,
|
|
860
|
-
...workerBootstrap
|
|
861
|
-
});
|
|
862
|
-
this.heartbeat = new Heartbeat(5e3, this.queries, this.workerRow);
|
|
863
|
-
this.workerState.transitionTo("running" /* Running */);
|
|
864
|
-
}
|
|
865
|
-
acknowledgeStop() {
|
|
866
|
-
this.workerState.transitionTo("stopping" /* Stopping */);
|
|
867
|
-
if (!this.workerRow) {
|
|
868
|
-
throw new Error("Cannot stop worker: workerRow not set");
|
|
869
|
-
}
|
|
870
|
-
try {
|
|
871
|
-
this.logger.debug("Acknowledging worker stop...");
|
|
872
|
-
this.workerState.transitionTo("stopped" /* Stopped */);
|
|
873
|
-
this.logger.debug("Worker stop acknowledged");
|
|
874
|
-
} catch (error) {
|
|
875
|
-
this.logger.debug(`Error acknowledging worker stop: ${error}`);
|
|
876
|
-
throw error;
|
|
877
|
-
}
|
|
878
|
-
}
|
|
879
|
-
get edgeFunctionName() {
|
|
880
|
-
return this.workerRow?.function_name;
|
|
881
|
-
}
|
|
882
|
-
get queueName() {
|
|
883
|
-
return this.flow.slug;
|
|
884
|
-
}
|
|
885
|
-
async sendHeartbeat() {
|
|
886
|
-
await this.heartbeat?.send();
|
|
887
|
-
}
|
|
888
|
-
get isRunning() {
|
|
889
|
-
return this.workerState.isRunning;
|
|
890
|
-
}
|
|
891
|
-
get isStopping() {
|
|
892
|
-
return this.workerState.isStopping;
|
|
893
|
-
}
|
|
894
|
-
get isStopped() {
|
|
895
|
-
return this.workerState.isStopped;
|
|
896
|
-
}
|
|
897
|
-
transitionToStopping() {
|
|
898
|
-
this.workerState.transitionTo("stopping" /* Stopping */);
|
|
899
|
-
}
|
|
900
|
-
};
|
|
901
|
-
|
|
902
|
-
// ../edge-worker/src/flow/createFlowWorker.ts
|
|
903
|
-
function createFlowWorker(flow, config) {
|
|
904
|
-
const logger2 = getLogger("createFlowWorker");
|
|
905
|
-
const abortController = new AbortController();
|
|
906
|
-
const abortSignal = abortController.signal;
|
|
907
|
-
if (!config.sql && !config.connectionString) {
|
|
908
|
-
throw new Error(
|
|
909
|
-
"Either 'sql' or 'connectionString' must be provided in FlowWorkerConfig."
|
|
910
|
-
);
|
|
911
|
-
}
|
|
912
|
-
const sql = config.sql || postgres2(config.connectionString, {
|
|
913
|
-
max: config.maxPgConnections,
|
|
914
|
-
prepare: false
|
|
915
|
-
});
|
|
916
|
-
const pgflowAdapter = new PgflowSqlClient(sql);
|
|
917
|
-
const queueName = flow.slug || "tasks";
|
|
918
|
-
logger2.debug(`Using queue name: ${queueName}`);
|
|
919
|
-
const queries = new Queries(sql);
|
|
920
|
-
const lifecycle = new FlowWorkerLifecycle(queries, flow);
|
|
921
|
-
const pollerConfig = {
|
|
922
|
-
batchSize: config.batchSize || 10,
|
|
923
|
-
queueName: flow.slug
|
|
924
|
-
};
|
|
925
|
-
const poller = new StepTaskPoller(
|
|
926
|
-
pgflowAdapter,
|
|
927
|
-
abortSignal,
|
|
928
|
-
pollerConfig
|
|
929
|
-
);
|
|
930
|
-
const executorFactory = (record, signal) => {
|
|
931
|
-
return new StepTaskExecutor(flow, record, pgflowAdapter, signal);
|
|
932
|
-
};
|
|
933
|
-
const executionController = new ExecutionController(
|
|
934
|
-
executorFactory,
|
|
935
|
-
abortSignal,
|
|
936
|
-
{
|
|
937
|
-
maxConcurrent: config.maxConcurrent || 10
|
|
938
|
-
}
|
|
939
|
-
);
|
|
940
|
-
const batchProcessor = new BatchProcessor(
|
|
941
|
-
executionController,
|
|
942
|
-
poller,
|
|
943
|
-
abortSignal
|
|
944
|
-
);
|
|
945
|
-
return new Worker(batchProcessor, lifecycle, sql);
|
|
946
|
-
}
|
|
947
|
-
export {
|
|
948
|
-
EdgeWorker,
|
|
949
|
-
FlowWorkerLifecycle,
|
|
950
|
-
createFlowWorker,
|
|
951
|
-
createQueueWorker
|
|
952
|
-
};
|
|
953
|
-
//# sourceMappingURL=index.js.map
|