@pgflow/edge-worker 0.0.9 → 0.0.10-prealpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/EdgeWorker.d.ts +73 -0
- package/dist/EdgeWorker.d.ts.map +1 -0
- package/dist/EdgeWorker.js +105 -0
- package/dist/core/BatchProcessor.d.ts +13 -0
- package/dist/core/BatchProcessor.d.ts.map +1 -0
- package/dist/core/BatchProcessor.js +29 -0
- package/dist/core/ExecutionController.d.ts +15 -0
- package/dist/core/ExecutionController.d.ts.map +1 -0
- package/dist/core/ExecutionController.js +34 -0
- package/dist/core/Heartbeat.d.ts +13 -0
- package/dist/core/Heartbeat.d.ts.map +1 -0
- package/dist/core/Heartbeat.js +21 -0
- package/dist/core/Queries.d.ts +14 -0
- package/dist/core/Queries.d.ts.map +1 -0
- package/dist/core/Queries.js +31 -0
- package/dist/core/Worker.d.ts +21 -0
- package/dist/core/Worker.d.ts.map +1 -0
- package/dist/core/Worker.js +79 -0
- package/dist/core/WorkerLifecycle.d.ts +26 -0
- package/dist/core/WorkerLifecycle.d.ts.map +1 -0
- package/dist/core/WorkerLifecycle.js +69 -0
- package/dist/core/WorkerState.d.ts +37 -0
- package/dist/core/WorkerState.d.ts.map +1 -0
- package/dist/core/WorkerState.js +70 -0
- package/dist/core/types.d.ts +39 -0
- package/dist/core/types.d.ts.map +1 -0
- package/dist/core/types.js +1 -0
- package/dist/flow/FlowWorkerLifecycle.d.ts +26 -0
- package/dist/flow/FlowWorkerLifecycle.d.ts.map +1 -0
- package/dist/flow/FlowWorkerLifecycle.js +64 -0
- package/dist/flow/StepTaskExecutor.d.ts +28 -0
- package/dist/flow/StepTaskExecutor.d.ts.map +1 -0
- package/dist/flow/StepTaskExecutor.js +71 -0
- package/dist/flow/StepTaskPoller.d.ts +21 -0
- package/dist/flow/StepTaskPoller.d.ts.map +1 -0
- package/dist/flow/StepTaskPoller.js +34 -0
- package/dist/flow/createFlowWorker.d.ts +24 -0
- package/dist/flow/createFlowWorker.d.ts.map +1 -0
- package/dist/flow/createFlowWorker.js +56 -0
- package/dist/flow/types.d.ts +2 -0
- package/dist/flow/types.d.ts.map +1 -0
- package/dist/flow/types.js +1 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -953
- package/dist/package.json +33 -0
- package/dist/platform/DenoAdapter.d.ts +23 -0
- package/dist/platform/DenoAdapter.d.ts.map +1 -0
- package/dist/platform/DenoAdapter.js +132 -0
- package/dist/platform/createAdapter.d.ts +7 -0
- package/dist/platform/createAdapter.d.ts.map +1 -0
- package/dist/platform/createAdapter.js +17 -0
- package/dist/platform/deno-types.d.ts +13 -0
- package/dist/platform/deno-types.d.ts.map +1 -0
- package/dist/platform/deno-types.js +6 -0
- package/dist/platform/index.d.ts +5 -0
- package/dist/platform/index.d.ts.map +1 -0
- package/dist/platform/index.js +4 -0
- package/dist/platform/types.d.ts +45 -0
- package/dist/platform/types.d.ts.map +1 -0
- package/dist/platform/types.js +1 -0
- package/dist/queue/MessageExecutor.d.ts +43 -0
- package/dist/queue/MessageExecutor.d.ts.map +1 -0
- package/dist/queue/MessageExecutor.js +95 -0
- package/dist/queue/Queue.d.ts +35 -0
- package/dist/queue/Queue.d.ts.map +1 -0
- package/dist/queue/Queue.js +87 -0
- package/dist/queue/ReadWithPollPoller.d.ts +20 -0
- package/dist/queue/ReadWithPollPoller.d.ts.map +1 -0
- package/dist/queue/ReadWithPollPoller.js +25 -0
- package/dist/queue/createQueueWorker.d.ts +74 -0
- package/dist/queue/createQueueWorker.d.ts.map +1 -0
- package/dist/queue/createQueueWorker.js +47 -0
- package/dist/queue/types.d.ts +13 -0
- package/dist/queue/types.d.ts.map +1 -0
- package/dist/queue/types.js +1 -0
- package/dist/tsconfig.lib.tsbuildinfo +1 -0
- package/package.json +15 -4
- package/dist/CHANGELOG.md +0 -44
- package/dist/LICENSE.md +0 -660
- package/dist/README.md +0 -46
- package/dist/index.js.map +0 -7
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { Json } from './core/types.js';
|
|
2
|
+
import { type QueueWorkerConfig } from './queue/createQueueWorker.js';
|
|
3
|
+
import type { PlatformAdapter } from './platform/types.js';
|
|
4
|
+
/**
|
|
5
|
+
* Configuration options for the EdgeWorker.
|
|
6
|
+
*/
|
|
7
|
+
export type EdgeWorkerConfig = QueueWorkerConfig;
|
|
8
|
+
/**
|
|
9
|
+
* EdgeWorker is the main entry point for creating and starting edge workers.
|
|
10
|
+
*
|
|
11
|
+
* It provides a simple interface for starting a worker that processes messages from a queue.
|
|
12
|
+
*
|
|
13
|
+
* @example
|
|
14
|
+
* ```typescript
|
|
15
|
+
* import { EdgeWorker } from '@pgflow/edge-worker';
|
|
16
|
+
*
|
|
17
|
+
* EdgeWorker.start(async (message) => {
|
|
18
|
+
* // Process the message
|
|
19
|
+
* console.log('Processing message:', message);
|
|
20
|
+
* }, {
|
|
21
|
+
* queueName: 'my-queue',
|
|
22
|
+
* maxConcurrent: 5,
|
|
23
|
+
* retryLimit: 3
|
|
24
|
+
* });
|
|
25
|
+
* ```
|
|
26
|
+
*/
|
|
27
|
+
export declare class EdgeWorker {
|
|
28
|
+
private static adapter;
|
|
29
|
+
private static wasCalled;
|
|
30
|
+
/**
|
|
31
|
+
* Start the EdgeWorker with the given message handler and configuration.
|
|
32
|
+
*
|
|
33
|
+
* @param handler - Function that processes each message from the queue
|
|
34
|
+
* @param config - Configuration options for the worker
|
|
35
|
+
*
|
|
36
|
+
* @example
|
|
37
|
+
* ```typescript
|
|
38
|
+
* EdgeWorker.start(handler, {
|
|
39
|
+
* // name of the queue to poll for messages
|
|
40
|
+
* queueName: 'tasks',
|
|
41
|
+
*
|
|
42
|
+
* // how many tasks are processed at the same time
|
|
43
|
+
* maxConcurrent: 10,
|
|
44
|
+
*
|
|
45
|
+
* // how many connections to the database are opened
|
|
46
|
+
* maxPgConnections: 4,
|
|
47
|
+
*
|
|
48
|
+
* // in-worker polling interval
|
|
49
|
+
* maxPollSeconds: 5,
|
|
50
|
+
*
|
|
51
|
+
* // in-database polling interval
|
|
52
|
+
* pollIntervalMs: 200,
|
|
53
|
+
*
|
|
54
|
+
* // how long to wait before retrying a failed job
|
|
55
|
+
* retryDelay: 5,
|
|
56
|
+
*
|
|
57
|
+
* // how many times to retry a failed job
|
|
58
|
+
* retryLimit: 5,
|
|
59
|
+
*
|
|
60
|
+
* // how long a job is invisible after reading
|
|
61
|
+
* // if not successful, will reappear after this time
|
|
62
|
+
* visibilityTimeout: 3,
|
|
63
|
+
* });
|
|
64
|
+
* ```
|
|
65
|
+
*/
|
|
66
|
+
static start<TPayload extends Json = Json>(handler: (message: TPayload) => Promise<void> | void, config?: EdgeWorkerConfig): Promise<PlatformAdapter>;
|
|
67
|
+
/**
|
|
68
|
+
* Stop the EdgeWorker and clean up resources.
|
|
69
|
+
*/
|
|
70
|
+
static stop(): Promise<void>;
|
|
71
|
+
private static ensureFirstCall;
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=EdgeWorker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"EdgeWorker.d.ts","sourceRoot":"","sources":["../src/EdgeWorker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAEL,KAAK,iBAAiB,EACvB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,KAAK,EACV,eAAe,EAGhB,MAAM,qBAAqB,CAAC;AAE7B;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,iBAAiB,CAAC;AAEjD;;;;;;;;;;;;;;;;;;GAkBG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAgC;IACtD,OAAO,CAAC,MAAM,CAAC,SAAS,CAAS;IAEjC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAmCG;WACU,KAAK,CAAC,QAAQ,SAAS,IAAI,GAAG,IAAI,EAC7C,OAAO,EAAE,CAAC,OAAO,EAAE,QAAQ,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,EACpD,MAAM,GAAE,gBAAqB;IAmC/B;;OAEG;WACU,IAAI;IAQjB,OAAO,CAAC,MAAM,CAAC,eAAe;CAQ/B"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { createQueueWorker, } from './queue/createQueueWorker.js';
|
|
2
|
+
import { createAdapter } from './platform/createAdapter.js';
|
|
3
|
+
/**
|
|
4
|
+
* EdgeWorker is the main entry point for creating and starting edge workers.
|
|
5
|
+
*
|
|
6
|
+
* It provides a simple interface for starting a worker that processes messages from a queue.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import { EdgeWorker } from '@pgflow/edge-worker';
|
|
11
|
+
*
|
|
12
|
+
* EdgeWorker.start(async (message) => {
|
|
13
|
+
* // Process the message
|
|
14
|
+
* console.log('Processing message:', message);
|
|
15
|
+
* }, {
|
|
16
|
+
* queueName: 'my-queue',
|
|
17
|
+
* maxConcurrent: 5,
|
|
18
|
+
* retryLimit: 3
|
|
19
|
+
* });
|
|
20
|
+
* ```
|
|
21
|
+
*/
|
|
22
|
+
export class EdgeWorker {
|
|
23
|
+
static adapter = null;
|
|
24
|
+
static wasCalled = false;
|
|
25
|
+
/**
|
|
26
|
+
* Start the EdgeWorker with the given message handler and configuration.
|
|
27
|
+
*
|
|
28
|
+
* @param handler - Function that processes each message from the queue
|
|
29
|
+
* @param config - Configuration options for the worker
|
|
30
|
+
*
|
|
31
|
+
* @example
|
|
32
|
+
* ```typescript
|
|
33
|
+
* EdgeWorker.start(handler, {
|
|
34
|
+
* // name of the queue to poll for messages
|
|
35
|
+
* queueName: 'tasks',
|
|
36
|
+
*
|
|
37
|
+
* // how many tasks are processed at the same time
|
|
38
|
+
* maxConcurrent: 10,
|
|
39
|
+
*
|
|
40
|
+
* // how many connections to the database are opened
|
|
41
|
+
* maxPgConnections: 4,
|
|
42
|
+
*
|
|
43
|
+
* // in-worker polling interval
|
|
44
|
+
* maxPollSeconds: 5,
|
|
45
|
+
*
|
|
46
|
+
* // in-database polling interval
|
|
47
|
+
* pollIntervalMs: 200,
|
|
48
|
+
*
|
|
49
|
+
* // how long to wait before retrying a failed job
|
|
50
|
+
* retryDelay: 5,
|
|
51
|
+
*
|
|
52
|
+
* // how many times to retry a failed job
|
|
53
|
+
* retryLimit: 5,
|
|
54
|
+
*
|
|
55
|
+
* // how long a job is invisible after reading
|
|
56
|
+
* // if not successful, will reappear after this time
|
|
57
|
+
* visibilityTimeout: 3,
|
|
58
|
+
* });
|
|
59
|
+
* ```
|
|
60
|
+
*/
|
|
61
|
+
static async start(handler, config = {}) {
|
|
62
|
+
this.ensureFirstCall();
|
|
63
|
+
// First, create the adapter
|
|
64
|
+
this.adapter = await createAdapter();
|
|
65
|
+
// Get environment info from adapter
|
|
66
|
+
const env = this.adapter.getEnv();
|
|
67
|
+
// Complete the config with default values and environment info
|
|
68
|
+
const completeConfig = {
|
|
69
|
+
...config,
|
|
70
|
+
queueName: config.queueName || 'tasks',
|
|
71
|
+
maxConcurrent: config.maxConcurrent ?? 10,
|
|
72
|
+
maxPgConnections: config.maxPgConnections ?? 4,
|
|
73
|
+
maxPollSeconds: config.maxPollSeconds ?? 5,
|
|
74
|
+
pollIntervalMs: config.pollIntervalMs ?? 200,
|
|
75
|
+
retryDelay: config.retryDelay ?? 5,
|
|
76
|
+
retryLimit: config.retryLimit ?? 5,
|
|
77
|
+
visibilityTimeout: config.visibilityTimeout ?? 3,
|
|
78
|
+
connectionString: config.connectionString || env.connectionString,
|
|
79
|
+
};
|
|
80
|
+
// Create a worker factory function that will be called by the adapter
|
|
81
|
+
const createWorkerFn = (createLoggerFn) => {
|
|
82
|
+
return createQueueWorker(handler, completeConfig, createLoggerFn);
|
|
83
|
+
};
|
|
84
|
+
// Initialize the adapter with the worker factory function
|
|
85
|
+
await this.adapter.initialize(createWorkerFn);
|
|
86
|
+
return this.adapter;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Stop the EdgeWorker and clean up resources.
|
|
90
|
+
*/
|
|
91
|
+
static async stop() {
|
|
92
|
+
if (this.adapter) {
|
|
93
|
+
await this.adapter.terminate();
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
throw new Error('EdgeWorker.start() must be called first');
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
static ensureFirstCall() {
|
|
100
|
+
if (this.wasCalled) {
|
|
101
|
+
throw new Error('EdgeWorker.start() can only be called once');
|
|
102
|
+
}
|
|
103
|
+
this.wasCalled = true;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { ExecutionController } from './ExecutionController.js';
|
|
2
|
+
import type { IMessage, IPoller } from './types.js';
|
|
3
|
+
import type { Logger } from '../platform/types.js';
|
|
4
|
+
export declare class BatchProcessor<TMessage extends IMessage> {
|
|
5
|
+
private executionController;
|
|
6
|
+
private poller;
|
|
7
|
+
private signal;
|
|
8
|
+
private logger;
|
|
9
|
+
constructor(executionController: ExecutionController<TMessage>, poller: IPoller<TMessage>, signal: AbortSignal, logger: Logger);
|
|
10
|
+
processBatch(): Promise<void>;
|
|
11
|
+
awaitCompletion(): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=BatchProcessor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"BatchProcessor.d.ts","sourceRoot":"","sources":["../../src/core/BatchProcessor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AACpE,OAAO,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,qBAAa,cAAc,CAAC,QAAQ,SAAS,QAAQ;IAIjD,OAAO,CAAC,mBAAmB;IAC3B,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IALhB,OAAO,CAAC,MAAM,CAAS;gBAGb,mBAAmB,EAAE,mBAAmB,CAAC,QAAQ,CAAC,EAClD,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,EACzB,MAAM,EAAE,WAAW,EAC3B,MAAM,EAAE,MAAM;IAQV,YAAY;IAiBZ,eAAe;CAGtB"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export class BatchProcessor {
|
|
2
|
+
executionController;
|
|
3
|
+
poller;
|
|
4
|
+
signal;
|
|
5
|
+
logger;
|
|
6
|
+
constructor(executionController, poller, signal, logger) {
|
|
7
|
+
this.executionController = executionController;
|
|
8
|
+
this.poller = poller;
|
|
9
|
+
this.signal = signal;
|
|
10
|
+
this.executionController = executionController;
|
|
11
|
+
this.signal = signal;
|
|
12
|
+
this.poller = poller;
|
|
13
|
+
this.logger = logger;
|
|
14
|
+
}
|
|
15
|
+
async processBatch() {
|
|
16
|
+
this.logger.debug('Polling for new batch of messages...');
|
|
17
|
+
const messageRecords = await this.poller.poll();
|
|
18
|
+
if (this.signal.aborted) {
|
|
19
|
+
this.logger.info('Discarding messageRecords because worker is stopping');
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
this.logger.debug(`Starting ${messageRecords.length} messages`);
|
|
23
|
+
const startPromises = messageRecords.map((message) => this.executionController.start(message));
|
|
24
|
+
await Promise.all(startPromises);
|
|
25
|
+
}
|
|
26
|
+
async awaitCompletion() {
|
|
27
|
+
return await this.executionController.awaitCompletion();
|
|
28
|
+
}
|
|
29
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { IExecutor, IMessage } from './types.js';
|
|
2
|
+
import type { Logger } from '../platform/types.js';
|
|
3
|
+
export interface ExecutionConfig {
|
|
4
|
+
maxConcurrent: number;
|
|
5
|
+
}
|
|
6
|
+
export declare class ExecutionController<TMessage extends IMessage> {
|
|
7
|
+
private logger;
|
|
8
|
+
private promiseQueue;
|
|
9
|
+
private signal;
|
|
10
|
+
private createExecutor;
|
|
11
|
+
constructor(executorFactory: (record: TMessage, signal: AbortSignal) => IExecutor, abortSignal: AbortSignal, config: ExecutionConfig, logger: Logger);
|
|
12
|
+
start(record: TMessage): Promise<void>;
|
|
13
|
+
awaitCompletion(): Promise<void>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=ExecutionController.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ExecutionController.d.ts","sourceRoot":"","sources":["../../src/core/ExecutionController.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,WAAW,eAAe;IAC9B,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,qBAAa,mBAAmB,CAAC,QAAQ,SAAS,QAAQ;IACxD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,cAAc,CAAuD;gBAG3E,eAAe,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,KAAK,SAAS,EACrE,WAAW,EAAE,WAAW,EACxB,MAAM,EAAE,eAAe,EACvB,MAAM,EAAE,MAAM;IAQV,KAAK,CAAC,MAAM,EAAE,QAAQ;IAiBtB,eAAe;CAStB"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { newQueue } from '@henrygd/queue';
|
|
2
|
+
export class ExecutionController {
|
|
3
|
+
logger;
|
|
4
|
+
promiseQueue;
|
|
5
|
+
signal;
|
|
6
|
+
createExecutor;
|
|
7
|
+
constructor(executorFactory, abortSignal, config, logger) {
|
|
8
|
+
this.signal = abortSignal;
|
|
9
|
+
this.createExecutor = executorFactory;
|
|
10
|
+
this.promiseQueue = newQueue(config.maxConcurrent);
|
|
11
|
+
this.logger = logger;
|
|
12
|
+
}
|
|
13
|
+
async start(record) {
|
|
14
|
+
const executor = this.createExecutor(record, this.signal);
|
|
15
|
+
this.logger.info(`Scheduling execution of task ${executor.msgId}`);
|
|
16
|
+
return await this.promiseQueue.add(async () => {
|
|
17
|
+
try {
|
|
18
|
+
this.logger.debug(`Executing task ${executor.msgId}...`);
|
|
19
|
+
await executor.execute();
|
|
20
|
+
this.logger.debug(`Execution successful for ${executor.msgId}`);
|
|
21
|
+
}
|
|
22
|
+
catch (error) {
|
|
23
|
+
this.logger.error(`Execution failed for ${executor.msgId}:`, error);
|
|
24
|
+
throw error;
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
async awaitCompletion() {
|
|
29
|
+
const active = this.promiseQueue.active();
|
|
30
|
+
const all = this.promiseQueue.size();
|
|
31
|
+
this.logger.debug(`Awaiting completion of all tasks... (active/all: ${active}}/${all})`);
|
|
32
|
+
await this.promiseQueue.done();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Queries } from './Queries.js';
|
|
2
|
+
import type { WorkerRow } from './types.js';
|
|
3
|
+
import type { Logger } from '../platform/types.js';
|
|
4
|
+
export declare class Heartbeat {
|
|
5
|
+
private interval;
|
|
6
|
+
private queries;
|
|
7
|
+
private workerRow;
|
|
8
|
+
private logger;
|
|
9
|
+
private lastHeartbeat;
|
|
10
|
+
constructor(interval: number, queries: Queries, workerRow: WorkerRow, logger: Logger);
|
|
11
|
+
send(): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=Heartbeat.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Heartbeat.d.ts","sourceRoot":"","sources":["../../src/core/Heartbeat.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,qBAAa,SAAS;IAKlB,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,OAAO;IACf,OAAO,CAAC,SAAS;IANnB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,aAAa,CAAK;gBAGhB,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,OAAO,EAChB,SAAS,EAAE,SAAS,EAC5B,MAAM,EAAE,MAAM;IAKV,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CAQ5B"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export class Heartbeat {
|
|
2
|
+
interval;
|
|
3
|
+
queries;
|
|
4
|
+
workerRow;
|
|
5
|
+
logger;
|
|
6
|
+
lastHeartbeat = 0;
|
|
7
|
+
constructor(interval, queries, workerRow, logger) {
|
|
8
|
+
this.interval = interval;
|
|
9
|
+
this.queries = queries;
|
|
10
|
+
this.workerRow = workerRow;
|
|
11
|
+
this.logger = logger;
|
|
12
|
+
}
|
|
13
|
+
async send() {
|
|
14
|
+
const now = Date.now();
|
|
15
|
+
if (now - this.lastHeartbeat >= this.interval) {
|
|
16
|
+
await this.queries.sendHeartbeat(this.workerRow);
|
|
17
|
+
this.logger.debug('OK');
|
|
18
|
+
this.lastHeartbeat = now;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type postgres from 'postgres';
|
|
2
|
+
import type { WorkerRow } from './types.js';
|
|
3
|
+
export declare class Queries {
|
|
4
|
+
private readonly sql;
|
|
5
|
+
constructor(sql: postgres.Sql);
|
|
6
|
+
onWorkerStarted({ queueName, workerId, edgeFunctionName, }: {
|
|
7
|
+
queueName: string;
|
|
8
|
+
workerId: string;
|
|
9
|
+
edgeFunctionName: string;
|
|
10
|
+
}): Promise<WorkerRow>;
|
|
11
|
+
onWorkerStopped(workerRow: WorkerRow): Promise<WorkerRow>;
|
|
12
|
+
sendHeartbeat(workerRow: WorkerRow): Promise<void>;
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=Queries.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Queries.d.ts","sourceRoot":"","sources":["../../src/core/Queries.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AACrC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,qBAAa,OAAO;IACN,OAAO,CAAC,QAAQ,CAAC,GAAG;gBAAH,GAAG,EAAE,QAAQ,CAAC,GAAG;IAExC,eAAe,CAAC,EACpB,SAAS,EACT,QAAQ,EACR,gBAAgB,GACjB,EAAE;QACD,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,gBAAgB,EAAE,MAAM,CAAC;KAC1B,GAAG,OAAO,CAAC,SAAS,CAAC;IAUhB,eAAe,CAAC,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAWzD,aAAa,CAAC,SAAS,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC;CAQzD"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export class Queries {
|
|
2
|
+
sql;
|
|
3
|
+
constructor(sql) {
|
|
4
|
+
this.sql = sql;
|
|
5
|
+
}
|
|
6
|
+
async onWorkerStarted({ queueName, workerId, edgeFunctionName, }) {
|
|
7
|
+
const [worker] = await this.sql `
|
|
8
|
+
INSERT INTO edge_worker.workers (queue_name, worker_id, function_name)
|
|
9
|
+
VALUES (${queueName}, ${workerId}, ${edgeFunctionName})
|
|
10
|
+
RETURNING *;
|
|
11
|
+
`;
|
|
12
|
+
return worker;
|
|
13
|
+
}
|
|
14
|
+
async onWorkerStopped(workerRow) {
|
|
15
|
+
const [worker] = await this.sql `
|
|
16
|
+
UPDATE edge_worker.workers AS w
|
|
17
|
+
SET stopped_at = clock_timestamp(), last_heartbeat_at = clock_timestamp()
|
|
18
|
+
WHERE w.worker_id = ${workerRow.worker_id}
|
|
19
|
+
RETURNING *;
|
|
20
|
+
`;
|
|
21
|
+
return worker;
|
|
22
|
+
}
|
|
23
|
+
async sendHeartbeat(workerRow) {
|
|
24
|
+
await this.sql `
|
|
25
|
+
UPDATE edge_worker.workers AS w
|
|
26
|
+
SET last_heartbeat_at = clock_timestamp()
|
|
27
|
+
WHERE w.worker_id = ${workerRow.worker_id}
|
|
28
|
+
RETURNING *;
|
|
29
|
+
`;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type postgres from 'postgres';
|
|
2
|
+
import type { IBatchProcessor, ILifecycle, WorkerBootstrap } from './types.js';
|
|
3
|
+
import type { Logger } from '../platform/types.js';
|
|
4
|
+
export declare class Worker {
|
|
5
|
+
private lifecycle;
|
|
6
|
+
private logger;
|
|
7
|
+
private abortController;
|
|
8
|
+
private batchProcessor;
|
|
9
|
+
private sql;
|
|
10
|
+
constructor(batchProcessor: IBatchProcessor, lifecycle: ILifecycle, sql: postgres.Sql, logger: Logger);
|
|
11
|
+
startOnlyOnce(workerBootstrap: WorkerBootstrap): Promise<void>;
|
|
12
|
+
private start;
|
|
13
|
+
stop(): Promise<void>;
|
|
14
|
+
get edgeFunctionName(): string | undefined;
|
|
15
|
+
/**
|
|
16
|
+
* Returns true if worker state is Running and worker was not stopped
|
|
17
|
+
*/
|
|
18
|
+
private get isMainLoopActive();
|
|
19
|
+
private get isAborted();
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=Worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"Worker.d.ts","sourceRoot":"","sources":["../../src/core/Worker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AACrC,OAAO,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,qBAAa,MAAM;IACjB,OAAO,CAAC,SAAS,CAAa;IAC9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,eAAe,CAAyB;IAEhD,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,GAAG,CAAe;gBAGxB,cAAc,EAAE,eAAe,EAC/B,SAAS,EAAE,UAAU,EACrB,GAAG,EAAE,QAAQ,CAAC,GAAG,EACjB,MAAM,EAAE,MAAM;IAQV,aAAa,CAAC,eAAe,EAAE,eAAe;YAStC,KAAK;IAyBb,IAAI;IA2BV,IAAI,gBAAgB,uBAEnB;IAED;;OAEG;IACH,OAAO,KAAK,gBAAgB,GAE3B;IAED,OAAO,KAAK,SAAS,GAEpB;CACF"}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
export class Worker {
|
|
2
|
+
lifecycle;
|
|
3
|
+
logger;
|
|
4
|
+
abortController = new AbortController();
|
|
5
|
+
batchProcessor;
|
|
6
|
+
sql;
|
|
7
|
+
constructor(batchProcessor, lifecycle, sql, logger) {
|
|
8
|
+
this.sql = sql;
|
|
9
|
+
this.lifecycle = lifecycle;
|
|
10
|
+
this.batchProcessor = batchProcessor;
|
|
11
|
+
this.logger = logger;
|
|
12
|
+
}
|
|
13
|
+
async startOnlyOnce(workerBootstrap) {
|
|
14
|
+
if (this.lifecycle.isRunning) {
|
|
15
|
+
this.logger.debug('Worker already running, ignoring start request');
|
|
16
|
+
return;
|
|
17
|
+
}
|
|
18
|
+
await this.start(workerBootstrap);
|
|
19
|
+
}
|
|
20
|
+
async start(workerBootstrap) {
|
|
21
|
+
try {
|
|
22
|
+
await this.lifecycle.acknowledgeStart(workerBootstrap);
|
|
23
|
+
while (this.isMainLoopActive) {
|
|
24
|
+
try {
|
|
25
|
+
await this.lifecycle.sendHeartbeat();
|
|
26
|
+
}
|
|
27
|
+
catch (error) {
|
|
28
|
+
this.logger.error(`Error sending heartbeat: ${error}`);
|
|
29
|
+
// Continue execution - a failed heartbeat shouldn't stop processing
|
|
30
|
+
}
|
|
31
|
+
try {
|
|
32
|
+
await this.batchProcessor.processBatch();
|
|
33
|
+
}
|
|
34
|
+
catch (error) {
|
|
35
|
+
this.logger.error(`Error processing batch: ${error}`);
|
|
36
|
+
// Continue to next iteration - failed batch shouldn't stop the worker
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
catch (error) {
|
|
41
|
+
this.logger.error(`Error in worker main loop: ${error}`);
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
async stop() {
|
|
46
|
+
// If the worker is already stopping or stopped, do nothing
|
|
47
|
+
if (this.lifecycle.isStopping || this.lifecycle.isStopped) {
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
this.lifecycle.transitionToStopping();
|
|
51
|
+
try {
|
|
52
|
+
this.logger.info('-> Stopped accepting new messages');
|
|
53
|
+
this.abortController.abort();
|
|
54
|
+
this.logger.info('-> Waiting for pending tasks to complete...');
|
|
55
|
+
await this.batchProcessor.awaitCompletion();
|
|
56
|
+
this.logger.info('-> Pending tasks completed!');
|
|
57
|
+
this.lifecycle.acknowledgeStop();
|
|
58
|
+
this.logger.info('-> Closing SQL connection...');
|
|
59
|
+
await this.sql.end();
|
|
60
|
+
this.logger.info('-> SQL connection closed!');
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
this.logger.info(`Error during worker stop: ${error}`);
|
|
64
|
+
throw error;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
get edgeFunctionName() {
|
|
68
|
+
return this.lifecycle.edgeFunctionName;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Returns true if worker state is Running and worker was not stopped
|
|
72
|
+
*/
|
|
73
|
+
get isMainLoopActive() {
|
|
74
|
+
return this.lifecycle.isRunning && !this.isAborted;
|
|
75
|
+
}
|
|
76
|
+
get isAborted() {
|
|
77
|
+
return this.abortController.signal.aborted;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { Queries } from './Queries.js';
|
|
2
|
+
import type { Queue } from '../queue/Queue.js';
|
|
3
|
+
import type { ILifecycle, Json, WorkerBootstrap } from './types.js';
|
|
4
|
+
import type { Logger } from '../platform/types.js';
|
|
5
|
+
export interface LifecycleConfig {
|
|
6
|
+
queueName: string;
|
|
7
|
+
}
|
|
8
|
+
export declare class WorkerLifecycle<IMessage extends Json> implements ILifecycle {
|
|
9
|
+
private workerState;
|
|
10
|
+
private heartbeat?;
|
|
11
|
+
private logger;
|
|
12
|
+
private queries;
|
|
13
|
+
private queue;
|
|
14
|
+
private workerRow?;
|
|
15
|
+
constructor(queries: Queries, queue: Queue<IMessage>, logger: Logger);
|
|
16
|
+
acknowledgeStart(workerBootstrap: WorkerBootstrap): Promise<void>;
|
|
17
|
+
acknowledgeStop(): void;
|
|
18
|
+
get edgeFunctionName(): string | undefined;
|
|
19
|
+
get queueName(): string;
|
|
20
|
+
sendHeartbeat(): Promise<void>;
|
|
21
|
+
get isRunning(): boolean;
|
|
22
|
+
get isStopping(): boolean;
|
|
23
|
+
get isStopped(): boolean;
|
|
24
|
+
transitionToStopping(): void;
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=WorkerLifecycle.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WorkerLifecycle.d.ts","sourceRoot":"","sources":["../../src/core/WorkerLifecycle.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,eAAe,EAAa,MAAM,YAAY,CAAC;AAE/E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,eAAe,CAAC,QAAQ,SAAS,IAAI,CAAE,YAAW,UAAU;IACvE,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,SAAS,CAAC,CAAY;IAC9B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,SAAS,CAAC,CAAY;gBAElB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,MAAM;IAO9D,gBAAgB,CAAC,eAAe,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBvE,eAAe;IAyBf,IAAI,gBAAgB,uBAEnB;IAED,IAAI,SAAS,WAEZ;IAEK,aAAa;IAInB,IAAI,SAAS,YAEZ;IAED,IAAI,UAAU,YAEb;IAED,IAAI,SAAS,YAEZ;IAED,oBAAoB;CAGrB"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { Heartbeat } from './Heartbeat.js';
|
|
2
|
+
import { States, WorkerState } from './WorkerState.js';
|
|
3
|
+
export class WorkerLifecycle {
|
|
4
|
+
workerState;
|
|
5
|
+
heartbeat;
|
|
6
|
+
logger;
|
|
7
|
+
queries;
|
|
8
|
+
queue;
|
|
9
|
+
workerRow;
|
|
10
|
+
constructor(queries, queue, logger) {
|
|
11
|
+
this.queries = queries;
|
|
12
|
+
this.queue = queue;
|
|
13
|
+
this.logger = logger;
|
|
14
|
+
this.workerState = new WorkerState(logger);
|
|
15
|
+
}
|
|
16
|
+
async acknowledgeStart(workerBootstrap) {
|
|
17
|
+
this.workerState.transitionTo(States.Starting);
|
|
18
|
+
this.logger.info(`Ensuring queue '${this.queue.queueName}' exists...`);
|
|
19
|
+
await this.queue.safeCreate();
|
|
20
|
+
this.workerRow = await this.queries.onWorkerStarted({
|
|
21
|
+
queueName: this.queueName,
|
|
22
|
+
...workerBootstrap,
|
|
23
|
+
});
|
|
24
|
+
this.heartbeat = new Heartbeat(5000, this.queries, this.workerRow, this.logger);
|
|
25
|
+
this.workerState.transitionTo(States.Running);
|
|
26
|
+
}
|
|
27
|
+
acknowledgeStop() {
|
|
28
|
+
this.workerState.transitionTo(States.Stopping);
|
|
29
|
+
if (!this.workerRow) {
|
|
30
|
+
throw new Error('Cannot stop worker: workerRow not set');
|
|
31
|
+
}
|
|
32
|
+
try {
|
|
33
|
+
this.logger.debug('Acknowledging worker stop...');
|
|
34
|
+
// TODO: commented out because we can live without this
|
|
35
|
+
// but it is causing problems with DbHandler - workes does not have
|
|
36
|
+
// enough time to fire this query before hard-terimnated
|
|
37
|
+
// We can always check the heartbeat to see if it is still running
|
|
38
|
+
//
|
|
39
|
+
// await this.queries.onWorkerStopped(this.workerRow);
|
|
40
|
+
this.workerState.transitionTo(States.Stopped);
|
|
41
|
+
this.logger.debug('Worker stop acknowledged');
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
this.logger.debug(`Error acknowledging worker stop: ${error}`);
|
|
45
|
+
throw error;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
get edgeFunctionName() {
|
|
49
|
+
return this.workerRow?.function_name;
|
|
50
|
+
}
|
|
51
|
+
get queueName() {
|
|
52
|
+
return this.queue.queueName;
|
|
53
|
+
}
|
|
54
|
+
async sendHeartbeat() {
|
|
55
|
+
await this.heartbeat?.send();
|
|
56
|
+
}
|
|
57
|
+
get isRunning() {
|
|
58
|
+
return this.workerState.isRunning;
|
|
59
|
+
}
|
|
60
|
+
get isStopping() {
|
|
61
|
+
return this.workerState.isStopping;
|
|
62
|
+
}
|
|
63
|
+
get isStopped() {
|
|
64
|
+
return this.workerState.isStopped;
|
|
65
|
+
}
|
|
66
|
+
transitionToStopping() {
|
|
67
|
+
this.workerState.transitionTo(States.Stopping);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { Logger } from '../platform/types.js';
|
|
2
|
+
export declare enum States {
|
|
3
|
+
/** The worker has been created but has not yet started. */
|
|
4
|
+
Created = "created",
|
|
5
|
+
/** The worker is starting but has not yet started processing messages. */
|
|
6
|
+
Starting = "starting",
|
|
7
|
+
/** The worker is processing messages. */
|
|
8
|
+
Running = "running",
|
|
9
|
+
/** The worker stopped processing messages but is still releasing resources. */
|
|
10
|
+
Stopping = "stopping",
|
|
11
|
+
/** The worker has stopped processing messages and released resources
|
|
12
|
+
* and can be discarded. */
|
|
13
|
+
Stopped = "stopped"
|
|
14
|
+
}
|
|
15
|
+
export declare const Transitions: Record<States, States[]>;
|
|
16
|
+
export declare class TransitionError extends Error {
|
|
17
|
+
constructor(options: {
|
|
18
|
+
from: States;
|
|
19
|
+
to: States;
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Represents the state of a worker and exposes method for doing allowed transitions
|
|
24
|
+
*/
|
|
25
|
+
export declare class WorkerState {
|
|
26
|
+
private logger;
|
|
27
|
+
private state;
|
|
28
|
+
constructor(logger: Logger);
|
|
29
|
+
get current(): States;
|
|
30
|
+
get isCreated(): boolean;
|
|
31
|
+
get isStarting(): boolean;
|
|
32
|
+
get isRunning(): boolean;
|
|
33
|
+
get isStopping(): boolean;
|
|
34
|
+
get isStopped(): boolean;
|
|
35
|
+
transitionTo(state: States): void;
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=WorkerState.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WorkerState.d.ts","sourceRoot":"","sources":["../../src/core/WorkerState.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,oBAAY,MAAM;IAChB,2DAA2D;IAC3D,OAAO,YAAY;IAEnB,0EAA0E;IAC1E,QAAQ,aAAa;IAErB,yCAAyC;IACzC,OAAO,YAAY;IAEnB,+EAA+E;IAC/E,QAAQ,aAAa;IAErB;+BAC2B;IAC3B,OAAO,YAAY;CACpB;AAED,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAMhD,CAAC;AAEF,qBAAa,eAAgB,SAAQ,KAAK;gBAC5B,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE;CAGlD;AAED;;GAEG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAA0B;gBAE3B,MAAM,EAAE,MAAM;IAI1B,IAAI,OAAO,WAEV;IAED,IAAI,SAAS,YAEZ;IAED,IAAI,UAAU,YAEb;IAED,IAAI,SAAS,YAEZ;IAED,IAAI,UAAU,YAEb;IAED,IAAI,SAAS,YAEZ;IAED,YAAY,CAAC,KAAK,EAAE,MAAM;CAmB3B"}
|