@pgflow/edge-worker 0.1.14 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +4 -4
- package/dist/CHANGELOG.md +0 -268
- package/dist/EdgeWorker.d.ts +0 -121
- package/dist/EdgeWorker.d.ts.map +0 -1
- package/dist/EdgeWorker.js +0 -168
- package/dist/LICENSE.md +0 -660
- package/dist/README.md +0 -46
- package/dist/core/BatchProcessor.d.ts +0 -13
- package/dist/core/BatchProcessor.d.ts.map +0 -1
- package/dist/core/BatchProcessor.js +0 -29
- package/dist/core/ExecutionController.d.ts +0 -15
- package/dist/core/ExecutionController.d.ts.map +0 -1
- package/dist/core/ExecutionController.js +0 -34
- package/dist/core/Heartbeat.d.ts +0 -13
- package/dist/core/Heartbeat.d.ts.map +0 -1
- package/dist/core/Heartbeat.js +0 -21
- package/dist/core/Queries.d.ts +0 -14
- package/dist/core/Queries.d.ts.map +0 -1
- package/dist/core/Queries.js +0 -31
- package/dist/core/Worker.d.ts +0 -21
- package/dist/core/Worker.d.ts.map +0 -1
- package/dist/core/Worker.js +0 -79
- package/dist/core/WorkerLifecycle.d.ts +0 -26
- package/dist/core/WorkerLifecycle.d.ts.map +0 -1
- package/dist/core/WorkerLifecycle.js +0 -69
- package/dist/core/WorkerState.d.ts +0 -37
- package/dist/core/WorkerState.d.ts.map +0 -1
- package/dist/core/WorkerState.js +0 -70
- package/dist/core/types.d.ts +0 -39
- package/dist/core/types.d.ts.map +0 -1
- package/dist/core/types.js +0 -1
- package/dist/flow/FlowWorkerLifecycle.d.ts +0 -26
- package/dist/flow/FlowWorkerLifecycle.d.ts.map +0 -1
- package/dist/flow/FlowWorkerLifecycle.js +0 -64
- package/dist/flow/StepTaskExecutor.d.ts +0 -28
- package/dist/flow/StepTaskExecutor.d.ts.map +0 -1
- package/dist/flow/StepTaskExecutor.js +0 -71
- package/dist/flow/StepTaskPoller.d.ts +0 -21
- package/dist/flow/StepTaskPoller.d.ts.map +0 -1
- package/dist/flow/StepTaskPoller.js +0 -34
- package/dist/flow/createFlowWorker.d.ts +0 -42
- package/dist/flow/createFlowWorker.d.ts.map +0 -1
- package/dist/flow/createFlowWorker.js +0 -56
- package/dist/flow/types.d.ts +0 -2
- package/dist/flow/types.d.ts.map +0 -1
- package/dist/flow/types.js +0 -1
- package/dist/index.d.ts +0 -10
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -8
- package/dist/package.json +0 -33
- package/dist/platform/DenoAdapter.d.ts +0 -39
- package/dist/platform/DenoAdapter.d.ts.map +0 -1
- package/dist/platform/DenoAdapter.js +0 -126
- package/dist/platform/createAdapter.d.ts +0 -6
- package/dist/platform/createAdapter.d.ts.map +0 -1
- package/dist/platform/createAdapter.js +0 -16
- package/dist/platform/index.d.ts +0 -4
- package/dist/platform/index.d.ts.map +0 -1
- package/dist/platform/index.js +0 -3
- package/dist/platform/logging.d.ts +0 -10
- package/dist/platform/logging.d.ts.map +0 -1
- package/dist/platform/logging.js +0 -68
- package/dist/platform/types.d.ts +0 -37
- package/dist/platform/types.d.ts.map +0 -1
- package/dist/platform/types.js +0 -1
- package/dist/queue/MessageExecutor.d.ts +0 -43
- package/dist/queue/MessageExecutor.d.ts.map +0 -1
- package/dist/queue/MessageExecutor.js +0 -95
- package/dist/queue/Queue.d.ts +0 -35
- package/dist/queue/Queue.d.ts.map +0 -1
- package/dist/queue/Queue.js +0 -87
- package/dist/queue/ReadWithPollPoller.d.ts +0 -20
- package/dist/queue/ReadWithPollPoller.d.ts.map +0 -1
- package/dist/queue/ReadWithPollPoller.js +0 -25
- package/dist/queue/createQueueWorker.d.ts +0 -75
- package/dist/queue/createQueueWorker.d.ts.map +0 -1
- package/dist/queue/createQueueWorker.js +0 -47
- package/dist/queue/types.d.ts +0 -17
- package/dist/queue/types.d.ts.map +0 -1
- package/dist/queue/types.js +0 -1
- package/dist/tsconfig.lib.tsbuildinfo +0 -1
|
@@ -1,64 +0,0 @@
|
|
|
1
|
-
import { Heartbeat } from '../core/Heartbeat.js';
|
|
2
|
-
import { States, WorkerState } from '../core/WorkerState.js';
|
|
3
|
-
/**
|
|
4
|
-
* A specialized WorkerLifecycle for Flow-based workers that is aware of the Flow's step types
|
|
5
|
-
*/
|
|
6
|
-
export class FlowWorkerLifecycle {
|
|
7
|
-
workerState;
|
|
8
|
-
heartbeat;
|
|
9
|
-
logger;
|
|
10
|
-
queries;
|
|
11
|
-
workerRow;
|
|
12
|
-
flow;
|
|
13
|
-
constructor(queries, flow, logger) {
|
|
14
|
-
this.queries = queries;
|
|
15
|
-
this.flow = flow;
|
|
16
|
-
this.logger = logger;
|
|
17
|
-
this.workerState = new WorkerState(logger);
|
|
18
|
-
}
|
|
19
|
-
async acknowledgeStart(workerBootstrap) {
|
|
20
|
-
this.workerState.transitionTo(States.Starting);
|
|
21
|
-
this.workerRow = await this.queries.onWorkerStarted({
|
|
22
|
-
queueName: this.queueName,
|
|
23
|
-
...workerBootstrap,
|
|
24
|
-
});
|
|
25
|
-
this.heartbeat = new Heartbeat(5000, this.queries, this.workerRow, this.logger);
|
|
26
|
-
this.workerState.transitionTo(States.Running);
|
|
27
|
-
}
|
|
28
|
-
acknowledgeStop() {
|
|
29
|
-
this.workerState.transitionTo(States.Stopping);
|
|
30
|
-
if (!this.workerRow) {
|
|
31
|
-
throw new Error('Cannot stop worker: workerRow not set');
|
|
32
|
-
}
|
|
33
|
-
try {
|
|
34
|
-
this.logger.debug('Acknowledging worker stop...');
|
|
35
|
-
this.workerState.transitionTo(States.Stopped);
|
|
36
|
-
this.logger.debug('Worker stop acknowledged');
|
|
37
|
-
}
|
|
38
|
-
catch (error) {
|
|
39
|
-
this.logger.debug(`Error acknowledging worker stop: ${error}`);
|
|
40
|
-
throw error;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
get edgeFunctionName() {
|
|
44
|
-
return this.workerRow?.function_name;
|
|
45
|
-
}
|
|
46
|
-
get queueName() {
|
|
47
|
-
return this.flow.slug;
|
|
48
|
-
}
|
|
49
|
-
async sendHeartbeat() {
|
|
50
|
-
await this.heartbeat?.send();
|
|
51
|
-
}
|
|
52
|
-
get isRunning() {
|
|
53
|
-
return this.workerState.isRunning;
|
|
54
|
-
}
|
|
55
|
-
get isStopping() {
|
|
56
|
-
return this.workerState.isStopping;
|
|
57
|
-
}
|
|
58
|
-
get isStopped() {
|
|
59
|
-
return this.workerState.isStopped;
|
|
60
|
-
}
|
|
61
|
-
transitionToStopping() {
|
|
62
|
-
this.workerState.transitionTo(States.Stopping);
|
|
63
|
-
}
|
|
64
|
-
}
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import type { AnyFlow } from '@pgflow/dsl';
|
|
2
|
-
import type { StepTaskRecord, IPgflowClient } from './types.js';
|
|
3
|
-
import type { IExecutor } from '../core/types.js';
|
|
4
|
-
import type { Logger } from '../platform/types.js';
|
|
5
|
-
/**
|
|
6
|
-
* An executor that processes step tasks using an IPgflowClient
|
|
7
|
-
* with strong typing for the flow's step handlers
|
|
8
|
-
*/
|
|
9
|
-
export declare class StepTaskExecutor<TFlow extends AnyFlow> implements IExecutor {
|
|
10
|
-
private readonly flow;
|
|
11
|
-
private readonly task;
|
|
12
|
-
private readonly adapter;
|
|
13
|
-
private readonly signal;
|
|
14
|
-
private logger;
|
|
15
|
-
constructor(flow: TFlow, task: StepTaskRecord<TFlow>, adapter: IPgflowClient<TFlow>, signal: AbortSignal, logger: Logger);
|
|
16
|
-
get msgId(): number;
|
|
17
|
-
execute(): Promise<void>;
|
|
18
|
-
/**
|
|
19
|
-
* Handles the error that occurred during execution.
|
|
20
|
-
*
|
|
21
|
-
* If the error is an AbortError, it means that the worker was aborted and stopping,
|
|
22
|
-
* the task will be picked up by another worker later.
|
|
23
|
-
*
|
|
24
|
-
* Otherwise, it marks the task as failed.
|
|
25
|
-
*/
|
|
26
|
-
private handleExecutionError;
|
|
27
|
-
}
|
|
28
|
-
//# sourceMappingURL=StepTaskExecutor.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"StepTaskExecutor.d.ts","sourceRoot":"","sources":["../../src/flow/StepTaskExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AASnD;;;GAGG;AACH,qBAAa,gBAAgB,CAAC,KAAK,SAAS,OAAO,CAAE,YAAW,SAAS;IAIrE,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IANzB,OAAO,CAAC,MAAM,CAAS;gBAGJ,IAAI,EAAE,KAAK,EACX,IAAI,EAAE,cAAc,CAAC,KAAK,CAAC,EAC3B,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,EAC7B,MAAM,EAAE,WAAW,EACpC,MAAM,EAAE,MAAM;IAKhB,IAAI,KAAK,WAER;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAoC9B;;;;;;;OAOG;YACW,oBAAoB;CAYnC"}
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
class AbortError extends Error {
|
|
2
|
-
constructor() {
|
|
3
|
-
super('Operation aborted');
|
|
4
|
-
this.name = 'AbortError';
|
|
5
|
-
}
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* An executor that processes step tasks using an IPgflowClient
|
|
9
|
-
* with strong typing for the flow's step handlers
|
|
10
|
-
*/
|
|
11
|
-
export class StepTaskExecutor {
|
|
12
|
-
flow;
|
|
13
|
-
task;
|
|
14
|
-
adapter;
|
|
15
|
-
signal;
|
|
16
|
-
logger;
|
|
17
|
-
constructor(flow, task, adapter, signal, logger) {
|
|
18
|
-
this.flow = flow;
|
|
19
|
-
this.task = task;
|
|
20
|
-
this.adapter = adapter;
|
|
21
|
-
this.signal = signal;
|
|
22
|
-
this.logger = logger;
|
|
23
|
-
}
|
|
24
|
-
get msgId() {
|
|
25
|
-
return this.task.msg_id;
|
|
26
|
-
}
|
|
27
|
-
async execute() {
|
|
28
|
-
try {
|
|
29
|
-
if (this.signal.aborted) {
|
|
30
|
-
throw new AbortError();
|
|
31
|
-
}
|
|
32
|
-
// Check if already aborted before starting
|
|
33
|
-
this.signal.throwIfAborted();
|
|
34
|
-
const stepSlug = this.task.step_slug;
|
|
35
|
-
this.logger.debug(`Executing step task ${this.task.msg_id} for step ${stepSlug}`);
|
|
36
|
-
// Get the step handler from the flow with proper typing
|
|
37
|
-
const stepDef = this.flow.getStepDefinition(stepSlug);
|
|
38
|
-
if (!stepDef) {
|
|
39
|
-
throw new Error(`No step definition found for slug=${stepSlug}`);
|
|
40
|
-
}
|
|
41
|
-
// !!! HANDLER EXECUTION !!!
|
|
42
|
-
const result = await stepDef.handler(this.task.input);
|
|
43
|
-
// !!! HANDLER EXECUTION !!!
|
|
44
|
-
this.logger.debug(`step task ${this.task.msg_id} completed successfully, marking as complete`);
|
|
45
|
-
await this.adapter.completeTask(this.task, result);
|
|
46
|
-
this.logger.debug(`step task ${this.task.msg_id} marked as complete`);
|
|
47
|
-
}
|
|
48
|
-
catch (error) {
|
|
49
|
-
await this.handleExecutionError(error);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
/**
|
|
53
|
-
* Handles the error that occurred during execution.
|
|
54
|
-
*
|
|
55
|
-
* If the error is an AbortError, it means that the worker was aborted and stopping,
|
|
56
|
-
* the task will be picked up by another worker later.
|
|
57
|
-
*
|
|
58
|
-
* Otherwise, it marks the task as failed.
|
|
59
|
-
*/
|
|
60
|
-
async handleExecutionError(error) {
|
|
61
|
-
if (error instanceof Error && error.name === 'AbortError') {
|
|
62
|
-
this.logger.debug(`Aborted execution for step task ${this.task.msg_id}`);
|
|
63
|
-
// Do not mark as failed - the worker was aborted and stopping,
|
|
64
|
-
// the task will be picked up by another worker later
|
|
65
|
-
}
|
|
66
|
-
else {
|
|
67
|
-
this.logger.error(`step task ${this.task.msg_id} failed with error: ${error}`);
|
|
68
|
-
await this.adapter.failTask(this.task, error);
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
}
|
|
@@ -1,21 +0,0 @@
|
|
|
1
|
-
import type { StepTaskRecord, IPgflowClient } from './types.js';
|
|
2
|
-
import type { IPoller } from '../core/types.js';
|
|
3
|
-
import type { Logger } from '../platform/types.js';
|
|
4
|
-
import type { AnyFlow } from '@pgflow/dsl';
|
|
5
|
-
export interface StepTaskPollerConfig {
|
|
6
|
-
batchSize: number;
|
|
7
|
-
queueName: string;
|
|
8
|
-
}
|
|
9
|
-
/**
|
|
10
|
-
* A poller that retrieves flow tasks using an IPgflowClient
|
|
11
|
-
*/
|
|
12
|
-
export declare class StepTaskPoller<TFlow extends AnyFlow> implements IPoller<StepTaskRecord<TFlow>> {
|
|
13
|
-
private readonly adapter;
|
|
14
|
-
private readonly signal;
|
|
15
|
-
private readonly config;
|
|
16
|
-
private logger;
|
|
17
|
-
constructor(adapter: IPgflowClient<TFlow>, signal: AbortSignal, config: StepTaskPollerConfig, logger: Logger);
|
|
18
|
-
poll(): Promise<StepTaskRecord<TFlow>[]>;
|
|
19
|
-
private isAborted;
|
|
20
|
-
}
|
|
21
|
-
//# sourceMappingURL=StepTaskPoller.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"StepTaskPoller.d.ts","sourceRoot":"","sources":["../../src/flow/StepTaskPoller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAE3C,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,qBAAa,cAAc,CAAC,KAAK,SAAS,OAAO,CAC/C,YAAW,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;IAKvC,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IALzB,OAAO,CAAC,MAAM,CAAS;gBAGJ,OAAO,EAAE,aAAa,CAAC,KAAK,CAAC,EAC7B,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,oBAAoB,EAC7C,MAAM,EAAE,MAAM;IAKV,IAAI,IAAI,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC;IAuB9C,OAAO,CAAC,SAAS;CAGlB"}
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* A poller that retrieves flow tasks using an IPgflowClient
|
|
3
|
-
*/
|
|
4
|
-
export class StepTaskPoller {
|
|
5
|
-
adapter;
|
|
6
|
-
signal;
|
|
7
|
-
config;
|
|
8
|
-
logger;
|
|
9
|
-
constructor(adapter, signal, config, logger) {
|
|
10
|
-
this.adapter = adapter;
|
|
11
|
-
this.signal = signal;
|
|
12
|
-
this.config = config;
|
|
13
|
-
this.logger = logger;
|
|
14
|
-
}
|
|
15
|
-
async poll() {
|
|
16
|
-
if (this.isAborted()) {
|
|
17
|
-
this.logger.debug('Polling aborted, returning empty array');
|
|
18
|
-
return [];
|
|
19
|
-
}
|
|
20
|
-
this.logger.debug(`Polling for flow tasks with batch size ${this.config.batchSize}`);
|
|
21
|
-
try {
|
|
22
|
-
const tasks = await this.adapter.pollForTasks(this.config.queueName, this.config.batchSize);
|
|
23
|
-
this.logger.debug(`Retrieved ${tasks.length} flow tasks`);
|
|
24
|
-
return tasks;
|
|
25
|
-
}
|
|
26
|
-
catch (err) {
|
|
27
|
-
this.logger.error(`Error polling for flow tasks: ${err}`);
|
|
28
|
-
return [];
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
isAborted() {
|
|
32
|
-
return this.signal.aborted;
|
|
33
|
-
}
|
|
34
|
-
}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
import type { AnyFlow } from '@pgflow/dsl';
|
|
2
|
-
import type { Logger } from '../platform/types.js';
|
|
3
|
-
import { Worker } from '../core/Worker.js';
|
|
4
|
-
import postgres from 'postgres';
|
|
5
|
-
/**
|
|
6
|
-
* Configuration for the flow worker
|
|
7
|
-
*/
|
|
8
|
-
export type FlowWorkerConfig = {
|
|
9
|
-
/**
|
|
10
|
-
* How many tasks are processed at the same time
|
|
11
|
-
* @default 10
|
|
12
|
-
*/
|
|
13
|
-
maxConcurrent?: number;
|
|
14
|
-
/**
|
|
15
|
-
* PostgreSQL connection string.
|
|
16
|
-
* If not provided, it will be read from the EDGE_WORKER_DB_URL environment variable.
|
|
17
|
-
*/
|
|
18
|
-
connectionString?: string;
|
|
19
|
-
/**
|
|
20
|
-
* Optional SQL client instance
|
|
21
|
-
*/
|
|
22
|
-
sql?: postgres.Sql;
|
|
23
|
-
/**
|
|
24
|
-
* How many connections to the database are opened
|
|
25
|
-
* @default 4
|
|
26
|
-
*/
|
|
27
|
-
maxPgConnections?: number;
|
|
28
|
-
/**
|
|
29
|
-
* Batch size for polling messages
|
|
30
|
-
* @default 10
|
|
31
|
-
*/
|
|
32
|
-
batchSize?: number;
|
|
33
|
-
};
|
|
34
|
-
/**
|
|
35
|
-
* Creates a new Worker instance for processing flow tasks.
|
|
36
|
-
*
|
|
37
|
-
* @param flow - The Flow DSL definition
|
|
38
|
-
* @param config - Configuration options for the worker
|
|
39
|
-
* @returns A configured Worker instance ready to be started
|
|
40
|
-
*/
|
|
41
|
-
export declare function createFlowWorker<TFlow extends AnyFlow>(flow: TFlow, config: FlowWorkerConfig, createLogger: (module: string) => Logger): Worker;
|
|
42
|
-
//# sourceMappingURL=createFlowWorker.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"createFlowWorker.d.ts","sourceRoot":"","sources":["../../src/flow/createFlowWorker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAQ3C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,QAAQ,MAAM,UAAU,CAAC;AAIhC;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;OAEG;IACH,GAAG,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC;IAEnB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,SAAS,OAAO,EACpD,IAAI,EAAE,KAAK,EACX,MAAM,EAAE,gBAAgB,EACxB,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GACvC,MAAM,CAiFR"}
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import { ExecutionController } from '../core/ExecutionController.js';
|
|
2
|
-
import { StepTaskPoller } from './StepTaskPoller.js';
|
|
3
|
-
import { StepTaskExecutor } from './StepTaskExecutor.js';
|
|
4
|
-
import { PgflowSqlClient } from '@pgflow/core';
|
|
5
|
-
import { Queries } from '../core/Queries.js';
|
|
6
|
-
import { Worker } from '../core/Worker.js';
|
|
7
|
-
import postgres from 'postgres';
|
|
8
|
-
import { FlowWorkerLifecycle } from './FlowWorkerLifecycle.js';
|
|
9
|
-
import { BatchProcessor } from '../core/BatchProcessor.js';
|
|
10
|
-
/**
|
|
11
|
-
* Creates a new Worker instance for processing flow tasks.
|
|
12
|
-
*
|
|
13
|
-
* @param flow - The Flow DSL definition
|
|
14
|
-
* @param config - Configuration options for the worker
|
|
15
|
-
* @returns A configured Worker instance ready to be started
|
|
16
|
-
*/
|
|
17
|
-
export function createFlowWorker(flow, config, createLogger) {
|
|
18
|
-
const logger = createLogger('createFlowWorker');
|
|
19
|
-
// Create abort controller for graceful shutdown
|
|
20
|
-
const abortController = new AbortController();
|
|
21
|
-
const abortSignal = abortController.signal;
|
|
22
|
-
if (!config.sql && !config.connectionString) {
|
|
23
|
-
throw new Error("Either 'sql' or 'connectionString' must be provided in FlowWorkerConfig.");
|
|
24
|
-
}
|
|
25
|
-
const sql = config.sql ||
|
|
26
|
-
postgres(config.connectionString, {
|
|
27
|
-
max: config.maxPgConnections,
|
|
28
|
-
prepare: false,
|
|
29
|
-
});
|
|
30
|
-
// Create the pgflow adapter
|
|
31
|
-
const pgflowAdapter = new PgflowSqlClient(sql);
|
|
32
|
-
// Use flow slug as queue name, or fallback to 'tasks'
|
|
33
|
-
const queueName = flow.slug || 'tasks';
|
|
34
|
-
logger.debug(`Using queue name: ${queueName}`);
|
|
35
|
-
// Create specialized FlowWorkerLifecycle with the proxied queue and flow
|
|
36
|
-
const queries = new Queries(sql);
|
|
37
|
-
const lifecycle = new FlowWorkerLifecycle(queries, flow, createLogger('FlowWorkerLifecycle'));
|
|
38
|
-
// Create StepTaskPoller
|
|
39
|
-
const pollerConfig = {
|
|
40
|
-
batchSize: config.batchSize || 10,
|
|
41
|
-
queueName: flow.slug,
|
|
42
|
-
};
|
|
43
|
-
const poller = new StepTaskPoller(pgflowAdapter, abortSignal, pollerConfig, createLogger('StepTaskPoller'));
|
|
44
|
-
// Create executor factory with proper typing
|
|
45
|
-
const executorFactory = (record, signal) => {
|
|
46
|
-
return new StepTaskExecutor(flow, record, pgflowAdapter, signal, createLogger('StepTaskExecutor'));
|
|
47
|
-
};
|
|
48
|
-
// Create ExecutionController
|
|
49
|
-
const executionController = new ExecutionController(executorFactory, abortSignal, {
|
|
50
|
-
maxConcurrent: config.maxConcurrent || 10,
|
|
51
|
-
}, createLogger('ExecutionController'));
|
|
52
|
-
// Create BatchProcessor
|
|
53
|
-
const batchProcessor = new BatchProcessor(executionController, poller, abortSignal, createLogger('BatchProcessor'));
|
|
54
|
-
// Return Worker
|
|
55
|
-
return new Worker(batchProcessor, lifecycle, sql, createLogger('Worker'));
|
|
56
|
-
}
|
package/dist/flow/types.d.ts
DELETED
package/dist/flow/types.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/flow/types.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC"}
|
package/dist/flow/types.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export * from '@pgflow/core';
|
package/dist/index.d.ts
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
export { createQueueWorker } from './queue/createQueueWorker.js';
|
|
2
|
-
export { EdgeWorker } from './EdgeWorker.js';
|
|
3
|
-
export { createFlowWorker } from './flow/createFlowWorker.js';
|
|
4
|
-
export { FlowWorkerLifecycle } from './flow/FlowWorkerLifecycle.js';
|
|
5
|
-
export * from './platform/index.js';
|
|
6
|
-
export type { StepTaskRecord } from './flow/types.js';
|
|
7
|
-
export type { FlowWorkerConfig } from './flow/createFlowWorker.js';
|
|
8
|
-
export type { StepTaskPollerConfig } from './flow/StepTaskPoller.js';
|
|
9
|
-
export type { Json, IExecutor, IPoller, IMessage, ILifecycle, IBatchProcessor, } from './core/types.js';
|
|
10
|
-
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACjE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAG7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAGpE,cAAc,qBAAqB,CAAC;AAGpC,YAAY,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACtD,YAAY,EAAE,gBAAgB,EAAE,MAAM,4BAA4B,CAAC;AACnE,YAAY,EAAE,oBAAoB,EAAE,MAAM,0BAA0B,CAAC;AAGrE,YAAY,EACV,IAAI,EACJ,SAAS,EACT,OAAO,EACP,QAAQ,EACR,UAAU,EACV,eAAe,GAChB,MAAM,iBAAiB,CAAC"}
|
package/dist/index.js
DELETED
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
// Export existing queue-based worker
|
|
2
|
-
export { createQueueWorker } from './queue/createQueueWorker.js';
|
|
3
|
-
export { EdgeWorker } from './EdgeWorker.js';
|
|
4
|
-
// Export new flow-based worker
|
|
5
|
-
export { createFlowWorker } from './flow/createFlowWorker.js';
|
|
6
|
-
export { FlowWorkerLifecycle } from './flow/FlowWorkerLifecycle.js';
|
|
7
|
-
// Export platform adapters
|
|
8
|
-
export * from './platform/index.js';
|
package/dist/package.json
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"name": "@pgflow/edge-worker",
|
|
3
|
-
"version": "0.1.14",
|
|
4
|
-
"license": "AGPL-3.0",
|
|
5
|
-
"type": "module",
|
|
6
|
-
"main": "./dist/index.js",
|
|
7
|
-
"module": "./dist/index.js",
|
|
8
|
-
"types": "./dist/index.d.ts",
|
|
9
|
-
"exports": {
|
|
10
|
-
"./package.json": "./package.json",
|
|
11
|
-
".": {
|
|
12
|
-
"types": "./dist/index.d.ts",
|
|
13
|
-
"import": "./dist/index.js"
|
|
14
|
-
}
|
|
15
|
-
},
|
|
16
|
-
"files": [
|
|
17
|
-
"dist"
|
|
18
|
-
],
|
|
19
|
-
"dependencies": {
|
|
20
|
-
"@henrygd/queue": "^1.0.7",
|
|
21
|
-
"@pgflow/core": "workspace:*",
|
|
22
|
-
"@pgflow/dsl": "workspace:*",
|
|
23
|
-
"postgres": "3.4.5"
|
|
24
|
-
},
|
|
25
|
-
"devDependencies": {
|
|
26
|
-
"@types/deno": "npm:@teidesu/deno-types@1.45.2",
|
|
27
|
-
"@types/node": "~18.16.20",
|
|
28
|
-
"supabase": "2.21.1"
|
|
29
|
-
},
|
|
30
|
-
"publishConfig": {
|
|
31
|
-
"access": "public"
|
|
32
|
-
}
|
|
33
|
-
}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import type { CreateWorkerFn, Logger, PlatformAdapter } from './types.js';
|
|
2
|
-
/**
|
|
3
|
-
* Adapter for Deno runtime environment
|
|
4
|
-
*/
|
|
5
|
-
export declare class DenoAdapter implements PlatformAdapter {
|
|
6
|
-
private edgeFunctionName;
|
|
7
|
-
private worker;
|
|
8
|
-
private logger;
|
|
9
|
-
private loggingFactory;
|
|
10
|
-
constructor();
|
|
11
|
-
/**
|
|
12
|
-
* startWorker the platform adapter with a worker factory function
|
|
13
|
-
* @param createWorkerFn Function that creates a worker instance when called with a logger
|
|
14
|
-
*/
|
|
15
|
-
startWorker(createWorkerFn: CreateWorkerFn): Promise<void>;
|
|
16
|
-
stopWorker(): Promise<void>;
|
|
17
|
-
createLogger(module: string): Logger;
|
|
18
|
-
/**
|
|
19
|
-
* Ensures the config has a connectionString by using the environment value if needed
|
|
20
|
-
*/
|
|
21
|
-
getConnectionString(): string;
|
|
22
|
-
/**
|
|
23
|
-
* Get the Supabase URL for the current environment
|
|
24
|
-
*/
|
|
25
|
-
private getEnvVarOrThrow;
|
|
26
|
-
private spawnNewEdgeFunction;
|
|
27
|
-
private extractFunctionName;
|
|
28
|
-
private setupShutdownHandler;
|
|
29
|
-
/**
|
|
30
|
-
* Supabase EdgeRuntime exposes waitUntil method as a way to extend
|
|
31
|
-
* the lifetime of the function until the promise resolves.
|
|
32
|
-
*
|
|
33
|
-
* We leverage this to extend the lifetime to the absolute maximum,
|
|
34
|
-
* by passing a promise that never resolves.
|
|
35
|
-
*/
|
|
36
|
-
private extendLifetimeOfEdgeFunction;
|
|
37
|
-
private setupStartupHandler;
|
|
38
|
-
}
|
|
39
|
-
//# sourceMappingURL=DenoAdapter.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"DenoAdapter.d.ts","sourceRoot":"","sources":["../../src/platform/DenoAdapter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAI1E;;GAEG;AACH,qBAAa,WAAY,YAAW,eAAe;IACjD,OAAO,CAAC,gBAAgB,CAAuB;IAC/C,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAS;IAGvB,OAAO,CAAC,cAAc,CAA0B;;IAuBhD;;;OAGG;IACG,WAAW,CAAC,cAAc,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAM1D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAMjC,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAIpC;;OAEG;IACH,mBAAmB,IAAI,MAAM;IAI7B;;OAEG;IACH,OAAO,CAAC,gBAAgB;YAcV,oBAAoB;IA8BlC,OAAO,CAAC,mBAAmB;IAI3B,OAAO,CAAC,oBAAoB;IAY5B;;;;;;OAMG;IACH,OAAO,CAAC,4BAA4B;IAMpC,OAAO,CAAC,mBAAmB;CAwB5B"}
|
|
@@ -1,126 +0,0 @@
|
|
|
1
|
-
/// <reference types="./deno-types.d.ts" />
|
|
2
|
-
import { createLoggingFactory } from './logging.js';
|
|
3
|
-
/**
|
|
4
|
-
* Adapter for Deno runtime environment
|
|
5
|
-
*/
|
|
6
|
-
export class DenoAdapter {
|
|
7
|
-
edgeFunctionName = null;
|
|
8
|
-
worker = null;
|
|
9
|
-
logger;
|
|
10
|
-
// Logging factory with dynamic workerId support
|
|
11
|
-
loggingFactory = createLoggingFactory();
|
|
12
|
-
constructor() {
|
|
13
|
-
// Guard clause to ensure we're in a Deno environment
|
|
14
|
-
// This is just for type checking during build
|
|
15
|
-
// At runtime, this class should only be instantiated in Deno
|
|
16
|
-
if (typeof Deno === 'undefined' || typeof EdgeRuntime === 'undefined') {
|
|
17
|
-
throw new Error('DenoAdapter created in non-Deno environment - this is expected during build only');
|
|
18
|
-
}
|
|
19
|
-
// Set initial log level
|
|
20
|
-
const logLevel = this.getEnvVarOrThrow('EDGE_WORKER_LOG_LEVEL') || 'info';
|
|
21
|
-
console.log(`--- DenoAdapter: Raw log level from env: ${logLevel} ---`); // Raw console log
|
|
22
|
-
this.loggingFactory.setLogLevel(logLevel);
|
|
23
|
-
console.log('--- DenoAdapter: Log level set in factory ---'); // Raw console log
|
|
24
|
-
// startWorker logger with a default module name
|
|
25
|
-
this.logger = this.loggingFactory.createLogger('DenoAdapter');
|
|
26
|
-
this.logger.info('DenoAdapter logger instance created and working.'); // Use the created logger
|
|
27
|
-
}
|
|
28
|
-
/**
|
|
29
|
-
* startWorker the platform adapter with a worker factory function
|
|
30
|
-
* @param createWorkerFn Function that creates a worker instance when called with a logger
|
|
31
|
-
*/
|
|
32
|
-
async startWorker(createWorkerFn) {
|
|
33
|
-
this.extendLifetimeOfEdgeFunction();
|
|
34
|
-
this.setupShutdownHandler();
|
|
35
|
-
this.setupStartupHandler(createWorkerFn);
|
|
36
|
-
}
|
|
37
|
-
async stopWorker() {
|
|
38
|
-
if (this.worker) {
|
|
39
|
-
await this.worker.stop();
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
createLogger(module) {
|
|
43
|
-
return this.loggingFactory.createLogger(module);
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* Ensures the config has a connectionString by using the environment value if needed
|
|
47
|
-
*/
|
|
48
|
-
getConnectionString() {
|
|
49
|
-
return this.getEnvVarOrThrow('EDGE_WORKER_DB_URL');
|
|
50
|
-
}
|
|
51
|
-
/**
|
|
52
|
-
* Get the Supabase URL for the current environment
|
|
53
|
-
*/
|
|
54
|
-
getEnvVarOrThrow(name) {
|
|
55
|
-
const envVar = Deno.env.get(name);
|
|
56
|
-
if (!envVar) {
|
|
57
|
-
const message = `${name} is not set!\n` +
|
|
58
|
-
'See docs to learn how to prepare the environment:\n' +
|
|
59
|
-
'https://pgflow.pages.dev/edge-worker/prepare-environment';
|
|
60
|
-
throw new Error(message);
|
|
61
|
-
}
|
|
62
|
-
return envVar;
|
|
63
|
-
}
|
|
64
|
-
async spawnNewEdgeFunction() {
|
|
65
|
-
if (!this.edgeFunctionName) {
|
|
66
|
-
throw new Error('functionName cannot be null or empty');
|
|
67
|
-
}
|
|
68
|
-
const supabaseUrl = this.getEnvVarOrThrow('SUPABASE_URL');
|
|
69
|
-
const supabaseAnonKey = this.getEnvVarOrThrow('SUPABASE_ANON_KEY');
|
|
70
|
-
this.logger.debug('Spawning a new Edge Function...');
|
|
71
|
-
const response = await fetch(`${supabaseUrl}/functions/v1/${this.edgeFunctionName}`, {
|
|
72
|
-
method: 'POST',
|
|
73
|
-
headers: {
|
|
74
|
-
Authorization: `Bearer ${supabaseAnonKey}`,
|
|
75
|
-
'Content-Type': 'application/json',
|
|
76
|
-
},
|
|
77
|
-
});
|
|
78
|
-
this.logger.debug('Edge Function spawned successfully!');
|
|
79
|
-
if (!response.ok) {
|
|
80
|
-
throw new Error(`Edge function returned non-OK status: ${response.status} ${response.statusText}`);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
extractFunctionName(req) {
|
|
84
|
-
return new URL(req.url).pathname.replace(/^\/+|\/+$/g, '');
|
|
85
|
-
}
|
|
86
|
-
setupShutdownHandler() {
|
|
87
|
-
globalThis.onbeforeunload = async () => {
|
|
88
|
-
this.logger.info('Shutting down...');
|
|
89
|
-
if (this.worker) {
|
|
90
|
-
await this.spawnNewEdgeFunction();
|
|
91
|
-
}
|
|
92
|
-
await this.stopWorker();
|
|
93
|
-
};
|
|
94
|
-
}
|
|
95
|
-
/**
|
|
96
|
-
* Supabase EdgeRuntime exposes waitUntil method as a way to extend
|
|
97
|
-
* the lifetime of the function until the promise resolves.
|
|
98
|
-
*
|
|
99
|
-
* We leverage this to extend the lifetime to the absolute maximum,
|
|
100
|
-
* by passing a promise that never resolves.
|
|
101
|
-
*/
|
|
102
|
-
extendLifetimeOfEdgeFunction() {
|
|
103
|
-
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
|
104
|
-
const promiseThatNeverResolves = new Promise(() => { });
|
|
105
|
-
EdgeRuntime.waitUntil(promiseThatNeverResolves);
|
|
106
|
-
}
|
|
107
|
-
setupStartupHandler(createWorkerFn) {
|
|
108
|
-
Deno.serve({}, (req) => {
|
|
109
|
-
this.logger.info(`HTTP Request: ${this.edgeFunctionName}`);
|
|
110
|
-
if (!this.worker) {
|
|
111
|
-
this.edgeFunctionName = this.extractFunctionName(req);
|
|
112
|
-
const workerId = this.getEnvVarOrThrow('SB_EXECUTION_ID');
|
|
113
|
-
this.loggingFactory.setWorkerId(workerId);
|
|
114
|
-
// Create the worker using the factory function and the logger
|
|
115
|
-
this.worker = createWorkerFn(this.loggingFactory.createLogger);
|
|
116
|
-
this.worker.startOnlyOnce({
|
|
117
|
-
edgeFunctionName: this.edgeFunctionName,
|
|
118
|
-
workerId,
|
|
119
|
-
});
|
|
120
|
-
}
|
|
121
|
-
return new Response('ok', {
|
|
122
|
-
headers: { 'Content-Type': 'application/json' },
|
|
123
|
-
});
|
|
124
|
-
});
|
|
125
|
-
}
|
|
126
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"createAdapter.d.ts","sourceRoot":"","sources":["../../src/platform/createAdapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAGlD;;GAEG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,eAAe,CAAC,CAS9D"}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
import { DenoAdapter } from './DenoAdapter.js';
|
|
2
|
-
/**
|
|
3
|
-
* Creates the appropriate platform adapter based on the runtime environment
|
|
4
|
-
*/
|
|
5
|
-
export async function createAdapter() {
|
|
6
|
-
if (isDenoEnvironment()) {
|
|
7
|
-
const adapter = new DenoAdapter();
|
|
8
|
-
return adapter;
|
|
9
|
-
}
|
|
10
|
-
// For now, only support Deno
|
|
11
|
-
// Later add NodeAdapter, BrowserAdapter, etc.
|
|
12
|
-
throw new Error('Unsupported environment');
|
|
13
|
-
}
|
|
14
|
-
function isDenoEnvironment() {
|
|
15
|
-
return typeof Deno !== 'undefined';
|
|
16
|
-
}
|
package/dist/platform/index.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/platform/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/platform/index.js
DELETED
|
@@ -1,10 +0,0 @@
|
|
|
1
|
-
import type { Logger } from './types.js';
|
|
2
|
-
/**
|
|
3
|
-
* Creates a logging factory with dynamic workerId support
|
|
4
|
-
*/
|
|
5
|
-
export declare function createLoggingFactory(): {
|
|
6
|
-
createLogger: (module: string) => Logger;
|
|
7
|
-
setWorkerId: (workerId: string) => void;
|
|
8
|
-
setLogLevel: (newLogLevel: string) => void;
|
|
9
|
-
};
|
|
10
|
-
//# sourceMappingURL=logging.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"logging.d.ts","sourceRoot":"","sources":["../../src/platform/logging.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC;;GAEG;AACH,wBAAgB,oBAAoB;2BAcJ,MAAM,KAAG,MAAM;4BA2Dd,MAAM,KAAG,IAAI;+BAOV,MAAM,KAAG,IAAI;EAShD"}
|