@pgflow/edge-worker 0.1.13 → 0.1.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/package.json +2 -2
- package/package.json +4 -4
- package/dist/CHANGELOG.md +0 -260
- package/dist/EdgeWorker.d.ts +0 -121
- package/dist/EdgeWorker.d.ts.map +0 -1
- package/dist/EdgeWorker.js +0 -160
- package/dist/LICENSE.md +0 -660
- package/dist/README.md +0 -46
- package/dist/core/BatchProcessor.d.ts +0 -13
- package/dist/core/BatchProcessor.d.ts.map +0 -1
- package/dist/core/BatchProcessor.js +0 -29
- package/dist/core/ExecutionController.d.ts +0 -15
- package/dist/core/ExecutionController.d.ts.map +0 -1
- package/dist/core/ExecutionController.js +0 -34
- package/dist/core/Heartbeat.d.ts +0 -13
- package/dist/core/Heartbeat.d.ts.map +0 -1
- package/dist/core/Heartbeat.js +0 -21
- package/dist/core/Queries.d.ts +0 -14
- package/dist/core/Queries.d.ts.map +0 -1
- package/dist/core/Queries.js +0 -31
- package/dist/core/Worker.d.ts +0 -21
- package/dist/core/Worker.d.ts.map +0 -1
- package/dist/core/Worker.js +0 -79
- package/dist/core/WorkerLifecycle.d.ts +0 -26
- package/dist/core/WorkerLifecycle.d.ts.map +0 -1
- package/dist/core/WorkerLifecycle.js +0 -69
- package/dist/core/WorkerState.d.ts +0 -37
- package/dist/core/WorkerState.d.ts.map +0 -1
- package/dist/core/WorkerState.js +0 -70
- package/dist/core/types.d.ts +0 -39
- package/dist/core/types.d.ts.map +0 -1
- package/dist/core/types.js +0 -1
- package/dist/flow/FlowWorkerLifecycle.d.ts +0 -26
- package/dist/flow/FlowWorkerLifecycle.d.ts.map +0 -1
- package/dist/flow/FlowWorkerLifecycle.js +0 -64
- package/dist/flow/StepTaskExecutor.d.ts +0 -28
- package/dist/flow/StepTaskExecutor.d.ts.map +0 -1
- package/dist/flow/StepTaskExecutor.js +0 -71
- package/dist/flow/StepTaskPoller.d.ts +0 -21
- package/dist/flow/StepTaskPoller.d.ts.map +0 -1
- package/dist/flow/StepTaskPoller.js +0 -34
- package/dist/flow/createFlowWorker.d.ts +0 -42
- package/dist/flow/createFlowWorker.d.ts.map +0 -1
- package/dist/flow/createFlowWorker.js +0 -56
- package/dist/flow/types.d.ts +0 -2
- package/dist/flow/types.d.ts.map +0 -1
- package/dist/flow/types.js +0 -1
- package/dist/index.d.ts +0 -10
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -8
- package/dist/platform/DenoAdapter.d.ts +0 -39
- package/dist/platform/DenoAdapter.d.ts.map +0 -1
- package/dist/platform/DenoAdapter.js +0 -126
- package/dist/platform/createAdapter.d.ts +0 -6
- package/dist/platform/createAdapter.d.ts.map +0 -1
- package/dist/platform/createAdapter.js +0 -16
- package/dist/platform/index.d.ts +0 -4
- package/dist/platform/index.d.ts.map +0 -1
- package/dist/platform/index.js +0 -3
- package/dist/platform/logging.d.ts +0 -10
- package/dist/platform/logging.d.ts.map +0 -1
- package/dist/platform/logging.js +0 -68
- package/dist/platform/types.d.ts +0 -37
- package/dist/platform/types.d.ts.map +0 -1
- package/dist/platform/types.js +0 -1
- package/dist/queue/MessageExecutor.d.ts +0 -43
- package/dist/queue/MessageExecutor.d.ts.map +0 -1
- package/dist/queue/MessageExecutor.js +0 -95
- package/dist/queue/Queue.d.ts +0 -35
- package/dist/queue/Queue.d.ts.map +0 -1
- package/dist/queue/Queue.js +0 -87
- package/dist/queue/ReadWithPollPoller.d.ts +0 -20
- package/dist/queue/ReadWithPollPoller.d.ts.map +0 -1
- package/dist/queue/ReadWithPollPoller.js +0 -25
- package/dist/queue/createQueueWorker.d.ts +0 -75
- package/dist/queue/createQueueWorker.d.ts.map +0 -1
- package/dist/queue/createQueueWorker.js +0 -47
- package/dist/queue/types.d.ts +0 -17
- package/dist/queue/types.d.ts.map +0 -1
- package/dist/queue/types.js +0 -1
package/dist/platform/logging.js
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Creates a logging factory with dynamic workerId support
|
|
3
|
-
*/
|
|
4
|
-
export function createLoggingFactory() {
|
|
5
|
-
// Shared state for all loggers
|
|
6
|
-
let sharedWorkerId = 'unknown';
|
|
7
|
-
let logLevel = 'info';
|
|
8
|
-
// All created logger instances - using Map for efficient lookup
|
|
9
|
-
const loggers = new Map();
|
|
10
|
-
// Simple level filtering
|
|
11
|
-
const levels = { error: 0, warn: 1, info: 2, debug: 3 };
|
|
12
|
-
/**
|
|
13
|
-
* Creates a new logger for a specific module
|
|
14
|
-
*/
|
|
15
|
-
const createLogger = (module) => {
|
|
16
|
-
console.log('--- createLoggingFactory CALLED ---'); // See how many times this appears
|
|
17
|
-
// Create a logger that directly references the shared state
|
|
18
|
-
const logger = {
|
|
19
|
-
debug: (message, ...args) => {
|
|
20
|
-
const levelValue = levels[logLevel] ?? levels.info;
|
|
21
|
-
if (levelValue >= levels.debug) {
|
|
22
|
-
// Use console.log for debug messages since console.debug isn't available in Supabase
|
|
23
|
-
console.log(`[DEBUG] worker_id=${sharedWorkerId} module=${module} ${message}`, ...args);
|
|
24
|
-
}
|
|
25
|
-
},
|
|
26
|
-
info: (message, ...args) => {
|
|
27
|
-
const levelValue = levels[logLevel] ?? levels.info;
|
|
28
|
-
if (levelValue >= levels.info) {
|
|
29
|
-
// Use console.log for info messages since console.info isn't available in Supabase
|
|
30
|
-
console.log(`[INFO] worker_id=${sharedWorkerId} module=${module} ${message}`, ...args);
|
|
31
|
-
}
|
|
32
|
-
},
|
|
33
|
-
warn: (message, ...args) => {
|
|
34
|
-
const levelValue = levels[logLevel] ?? levels.info;
|
|
35
|
-
if (levelValue >= levels.warn) {
|
|
36
|
-
console.warn(`worker_id=${sharedWorkerId} module=${module} ${message}`, ...args);
|
|
37
|
-
}
|
|
38
|
-
},
|
|
39
|
-
error: (message, ...args) => {
|
|
40
|
-
const levelValue = levels[logLevel] ?? levels.info;
|
|
41
|
-
if (levelValue >= levels.error) {
|
|
42
|
-
console.error(`worker_id=${sharedWorkerId} module=${module} ${message}`, ...args);
|
|
43
|
-
}
|
|
44
|
-
},
|
|
45
|
-
};
|
|
46
|
-
// Store the logger in our registry using module as key
|
|
47
|
-
loggers.set(module, logger);
|
|
48
|
-
// Return the logger
|
|
49
|
-
return logger;
|
|
50
|
-
};
|
|
51
|
-
/**
|
|
52
|
-
* Updates the workerId for all loggers
|
|
53
|
-
*/
|
|
54
|
-
const setWorkerId = (workerId) => {
|
|
55
|
-
sharedWorkerId = workerId;
|
|
56
|
-
};
|
|
57
|
-
/**
|
|
58
|
-
* Updates the log level for all loggers
|
|
59
|
-
*/
|
|
60
|
-
const setLogLevel = (newLogLevel) => {
|
|
61
|
-
logLevel = newLogLevel;
|
|
62
|
-
};
|
|
63
|
-
return {
|
|
64
|
-
createLogger,
|
|
65
|
-
setWorkerId,
|
|
66
|
-
setLogLevel,
|
|
67
|
-
};
|
|
68
|
-
}
|
package/dist/platform/types.d.ts
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import type { Worker } from '../core/Worker.js';
|
|
2
|
-
/**
|
|
3
|
-
* Basic logger interface used throughout the application
|
|
4
|
-
*/
|
|
5
|
-
export interface Logger {
|
|
6
|
-
debug(message: string, ...args: any[]): void;
|
|
7
|
-
info(message: string, ...args: any[]): void;
|
|
8
|
-
warn(message: string, ...args: any[]): void;
|
|
9
|
-
error(message: string, ...args: any[]): void;
|
|
10
|
-
}
|
|
11
|
-
/**
|
|
12
|
-
* Logger factory function
|
|
13
|
-
*/
|
|
14
|
-
export type CreateLoggerFn = (module: string) => Logger;
|
|
15
|
-
/**
|
|
16
|
-
* Logger factory function
|
|
17
|
-
*/
|
|
18
|
-
export type CreateWorkerFn = (createLoggerFn: CreateLoggerFn) => Worker;
|
|
19
|
-
/**
|
|
20
|
-
* Common interface for all platform adapters
|
|
21
|
-
*/
|
|
22
|
-
export interface PlatformAdapter {
|
|
23
|
-
/**
|
|
24
|
-
* startWorker the platform adapter with a worker factory function
|
|
25
|
-
* @param createWorkerFn Function that creates a worker instance when called with a logger
|
|
26
|
-
*/
|
|
27
|
-
startWorker(createWorkerFn: CreateWorkerFn): Promise<void>;
|
|
28
|
-
/**
|
|
29
|
-
* Clean up resources when shutting down
|
|
30
|
-
*/
|
|
31
|
-
stopWorker(): Promise<void>;
|
|
32
|
-
/**
|
|
33
|
-
* Get the connection string for the database
|
|
34
|
-
*/
|
|
35
|
-
getConnectionString(): string;
|
|
36
|
-
}
|
|
37
|
-
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/platform/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAChD;;GAEG;AACH,MAAM,WAAW,MAAM;IACrB,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC7C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC5C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC5C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;CAC9C;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,CAAC;AAExD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,cAAc,EAAE,cAAc,KAAK,MAAM,CAAC;AAExE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,WAAW,CAAC,cAAc,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3D;;OAEG;IACH,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B;;OAEG;IACH,mBAAmB,IAAI,MAAM,CAAC;CAC/B"}
|
package/dist/platform/types.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
import type { Json } from '../core/types.js';
|
|
2
|
-
import type { MessageHandlerFn, PgmqMessageRecord } from './types.js';
|
|
3
|
-
import type { Queue } from './Queue.js';
|
|
4
|
-
import type { Logger } from '../platform/types.js';
|
|
5
|
-
/**
|
|
6
|
-
* A class that executes a message handler.
|
|
7
|
-
*
|
|
8
|
-
* It handles the execution of the message handler and retries or archives the message
|
|
9
|
-
* based on the retry limit and delay.
|
|
10
|
-
*
|
|
11
|
-
* It also handles the abort signal and logs the error.
|
|
12
|
-
*/
|
|
13
|
-
export declare class MessageExecutor<TPayload extends Json> {
|
|
14
|
-
private readonly queue;
|
|
15
|
-
private readonly record;
|
|
16
|
-
private readonly messageHandler;
|
|
17
|
-
private readonly signal;
|
|
18
|
-
private readonly retryLimit;
|
|
19
|
-
private readonly retryDelay;
|
|
20
|
-
private logger;
|
|
21
|
-
constructor(queue: Queue<TPayload>, record: PgmqMessageRecord<TPayload>, messageHandler: MessageHandlerFn<TPayload>, signal: AbortSignal, retryLimit: number, retryDelay: number, logger: Logger);
|
|
22
|
-
get msgId(): number;
|
|
23
|
-
execute(): Promise<void>;
|
|
24
|
-
/**
|
|
25
|
-
* Handles the error that occurred during execution.
|
|
26
|
-
*
|
|
27
|
-
* If the error is an AbortError, it means that the worker was aborted and stopping,
|
|
28
|
-
* the message will reappear after the visibility timeout and be picked up by another worker.
|
|
29
|
-
*
|
|
30
|
-
* Otherwise, it proceeds with retry or archiving forever.
|
|
31
|
-
*/
|
|
32
|
-
private handleExecutionError;
|
|
33
|
-
/**
|
|
34
|
-
* Retries the message if it is available.
|
|
35
|
-
* Otherwise, archives the message forever and stops processing it.
|
|
36
|
-
*/
|
|
37
|
-
private retryOrArchive;
|
|
38
|
-
/**
|
|
39
|
-
* Returns true if the message can be retried.
|
|
40
|
-
*/
|
|
41
|
-
private get retryAvailable();
|
|
42
|
-
}
|
|
43
|
-
//# sourceMappingURL=MessageExecutor.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"MessageExecutor.d.ts","sourceRoot":"","sources":["../../src/queue/MessageExecutor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AASnD;;;;;;;GAOG;AACH,qBAAa,eAAe,CAAC,QAAQ,SAAS,IAAI;IAI9C,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,cAAc;IAC/B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAR7B,OAAO,CAAC,MAAM,CAAS;gBAGJ,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,EACtB,MAAM,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EACnC,cAAc,EAAE,gBAAgB,CAAC,QAAQ,CAAC,EAC1C,MAAM,EAAE,WAAW,EACnB,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EACnC,MAAM,EAAE,MAAM;IAKhB,IAAI,KAAK,WAER;IAEK,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAsB9B;;;;;;;OAOG;YACW,oBAAoB;IAYlC;;;OAGG;YACW,cAAc;IAY5B;;OAEG;IACH,OAAO,KAAK,cAAc,GAIzB;CACF"}
|
|
@@ -1,95 +0,0 @@
|
|
|
1
|
-
class AbortError extends Error {
|
|
2
|
-
constructor() {
|
|
3
|
-
super('Operation aborted');
|
|
4
|
-
this.name = 'AbortError';
|
|
5
|
-
}
|
|
6
|
-
}
|
|
7
|
-
/**
|
|
8
|
-
* A class that executes a message handler.
|
|
9
|
-
*
|
|
10
|
-
* It handles the execution of the message handler and retries or archives the message
|
|
11
|
-
* based on the retry limit and delay.
|
|
12
|
-
*
|
|
13
|
-
* It also handles the abort signal and logs the error.
|
|
14
|
-
*/
|
|
15
|
-
export class MessageExecutor {
|
|
16
|
-
queue;
|
|
17
|
-
record;
|
|
18
|
-
messageHandler;
|
|
19
|
-
signal;
|
|
20
|
-
retryLimit;
|
|
21
|
-
retryDelay;
|
|
22
|
-
logger;
|
|
23
|
-
constructor(queue, record, messageHandler, signal, retryLimit, retryDelay, logger) {
|
|
24
|
-
this.queue = queue;
|
|
25
|
-
this.record = record;
|
|
26
|
-
this.messageHandler = messageHandler;
|
|
27
|
-
this.signal = signal;
|
|
28
|
-
this.retryLimit = retryLimit;
|
|
29
|
-
this.retryDelay = retryDelay;
|
|
30
|
-
this.logger = logger;
|
|
31
|
-
}
|
|
32
|
-
get msgId() {
|
|
33
|
-
return this.record.msg_id;
|
|
34
|
-
}
|
|
35
|
-
async execute() {
|
|
36
|
-
try {
|
|
37
|
-
if (this.signal.aborted) {
|
|
38
|
-
throw new AbortError();
|
|
39
|
-
}
|
|
40
|
-
// Check if already aborted before starting
|
|
41
|
-
this.signal.throwIfAborted();
|
|
42
|
-
this.logger.debug(`Executing task ${this.msgId}...`);
|
|
43
|
-
await this.messageHandler(this.record.message);
|
|
44
|
-
this.logger.debug(`Task ${this.msgId} completed successfully, archiving...`);
|
|
45
|
-
await this.queue.archive(this.msgId);
|
|
46
|
-
this.logger.debug(`Archived task ${this.msgId} successfully`);
|
|
47
|
-
}
|
|
48
|
-
catch (error) {
|
|
49
|
-
await this.handleExecutionError(error);
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
/**
|
|
53
|
-
* Handles the error that occurred during execution.
|
|
54
|
-
*
|
|
55
|
-
* If the error is an AbortError, it means that the worker was aborted and stopping,
|
|
56
|
-
* the message will reappear after the visibility timeout and be picked up by another worker.
|
|
57
|
-
*
|
|
58
|
-
* Otherwise, it proceeds with retry or archiving forever.
|
|
59
|
-
*/
|
|
60
|
-
async handleExecutionError(error) {
|
|
61
|
-
if (error instanceof Error && error.name === 'AbortError') {
|
|
62
|
-
this.logger.debug(`Aborted execution for ${this.msgId}`);
|
|
63
|
-
// Do not throw - the worker was aborted and stopping,
|
|
64
|
-
// the message will reappear after the visibility timeout
|
|
65
|
-
// and be picked up by another worker
|
|
66
|
-
}
|
|
67
|
-
else {
|
|
68
|
-
this.logger.debug(`Task ${this.msgId} failed with error: ${error}`);
|
|
69
|
-
await this.retryOrArchive();
|
|
70
|
-
}
|
|
71
|
-
}
|
|
72
|
-
/**
|
|
73
|
-
* Retries the message if it is available.
|
|
74
|
-
* Otherwise, archives the message forever and stops processing it.
|
|
75
|
-
*/
|
|
76
|
-
async retryOrArchive() {
|
|
77
|
-
if (this.retryAvailable) {
|
|
78
|
-
// adjust visibility timeout for message to appear after retryDelay
|
|
79
|
-
this.logger.debug(`Retrying ${this.msgId} in ${this.retryDelay} seconds`);
|
|
80
|
-
await this.queue.setVt(this.msgId, this.retryDelay);
|
|
81
|
-
}
|
|
82
|
-
else {
|
|
83
|
-
// archive message forever and stop processing it
|
|
84
|
-
this.logger.debug(`Archiving ${this.msgId} forever`);
|
|
85
|
-
await this.queue.archive(this.msgId);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Returns true if the message can be retried.
|
|
90
|
-
*/
|
|
91
|
-
get retryAvailable() {
|
|
92
|
-
const readCountLimit = this.retryLimit + 1; // initial read also counts
|
|
93
|
-
return this.record.read_ct < readCountLimit;
|
|
94
|
-
}
|
|
95
|
-
}
|
package/dist/queue/Queue.d.ts
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
import type postgres from 'postgres';
|
|
2
|
-
import type { PgmqMessageRecord } from './types.js';
|
|
3
|
-
import type { Json } from '../core/types.js';
|
|
4
|
-
import type { Logger } from '../platform/types.js';
|
|
5
|
-
export declare class Queue<TPayload extends Json> {
|
|
6
|
-
private readonly sql;
|
|
7
|
-
readonly queueName: string;
|
|
8
|
-
private logger;
|
|
9
|
-
constructor(sql: postgres.Sql, queueName: string, logger: Logger);
|
|
10
|
-
/**
|
|
11
|
-
* Creates a queue if it doesn't exist.
|
|
12
|
-
* If the queue already exists, this method does nothing.
|
|
13
|
-
*/
|
|
14
|
-
safeCreate(): Promise<postgres.RowList<postgres.Row[]>>;
|
|
15
|
-
/**
|
|
16
|
-
* Drops a queue if it exists.
|
|
17
|
-
* If the queue doesn't exist, this method does nothing.
|
|
18
|
-
*/
|
|
19
|
-
safeDrop(): Promise<postgres.RowList<postgres.Row[]>>;
|
|
20
|
-
archive(msgId: number): Promise<void>;
|
|
21
|
-
archiveBatch(msgIds: number[]): Promise<void>;
|
|
22
|
-
send(message: TPayload): Promise<void>;
|
|
23
|
-
readWithPoll(batchSize?: number, visibilityTimeout?: number, maxPollSeconds?: number, pollIntervalMs?: number): Promise<postgres.RowList<PgmqMessageRecord<TPayload>[]>>;
|
|
24
|
-
/**
|
|
25
|
-
* Sets the visibility timeout of a message to the current time plus the given offset.
|
|
26
|
-
*
|
|
27
|
-
* This is an inlined version of the pgmq.set_vt in order to fix the bug.
|
|
28
|
-
* The original uses now() instead of clock_timestamp() which is problematic in transactions.
|
|
29
|
-
* See more details here: https://github.com/tembo-io/pgmq/issues/367
|
|
30
|
-
*
|
|
31
|
-
* The only change made is now() replaced with clock_timestamp().
|
|
32
|
-
*/
|
|
33
|
-
setVt(msgId: number, vtOffsetSeconds: number): Promise<PgmqMessageRecord<TPayload>>;
|
|
34
|
-
}
|
|
35
|
-
//# sourceMappingURL=Queue.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"Queue.d.ts","sourceRoot":"","sources":["../../src/queue/Queue.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,UAAU,CAAC;AACrC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,qBAAa,KAAK,CAAC,QAAQ,SAAS,IAAI;IAIpC,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,QAAQ,CAAC,SAAS,EAAE,MAAM;IAJ5B,OAAO,CAAC,MAAM,CAAS;gBAGJ,GAAG,EAAE,QAAQ,CAAC,GAAG,EACzB,SAAS,EAAE,MAAM,EAC1B,MAAM,EAAE,MAAM;IAKhB;;;OAGG;IACG,UAAU;IAUhB;;;OAGG;IACG,QAAQ;IAUR,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOrC,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7C,IAAI,CAAC,OAAO,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtC,YAAY,CAChB,SAAS,SAAK,EACd,iBAAiB,SAAI,EACrB,cAAc,SAAI,EAClB,cAAc,SAAM;IAetB;;;;;;;;OAQG;IACG,KAAK,CACT,KAAK,EAAE,MAAM,EACb,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;CAUxC"}
|
package/dist/queue/Queue.js
DELETED
|
@@ -1,87 +0,0 @@
|
|
|
1
|
-
export class Queue {
|
|
2
|
-
sql;
|
|
3
|
-
queueName;
|
|
4
|
-
logger;
|
|
5
|
-
constructor(sql, queueName, logger) {
|
|
6
|
-
this.sql = sql;
|
|
7
|
-
this.queueName = queueName;
|
|
8
|
-
this.logger = logger;
|
|
9
|
-
}
|
|
10
|
-
/**
|
|
11
|
-
* Creates a queue if it doesn't exist.
|
|
12
|
-
* If the queue already exists, this method does nothing.
|
|
13
|
-
*/
|
|
14
|
-
async safeCreate() {
|
|
15
|
-
this.logger.debug(`Creating queue '${this.queueName}' if it doesn't exist`);
|
|
16
|
-
return await this.sql `
|
|
17
|
-
select * from pgmq.create(${this.queueName})
|
|
18
|
-
where not exists (
|
|
19
|
-
select 1 from pgmq.list_queues() where queue_name = ${this.queueName}
|
|
20
|
-
);
|
|
21
|
-
`;
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Drops a queue if it exists.
|
|
25
|
-
* If the queue doesn't exist, this method does nothing.
|
|
26
|
-
*/
|
|
27
|
-
async safeDrop() {
|
|
28
|
-
this.logger.debug(`Dropping queue '${this.queueName}' if it exists`);
|
|
29
|
-
return await this.sql `
|
|
30
|
-
select * from pgmq.drop_queue(${this.queueName})
|
|
31
|
-
where exists (
|
|
32
|
-
select 1 from pgmq.list_queues() where queue_name = ${this.queueName}
|
|
33
|
-
);
|
|
34
|
-
`;
|
|
35
|
-
}
|
|
36
|
-
async archive(msgId) {
|
|
37
|
-
this.logger.debug(`Archiving message ${msgId} from queue '${this.queueName}'`);
|
|
38
|
-
await this.sql `
|
|
39
|
-
SELECT pgmq.archive(queue_name => ${this.queueName}, msg_id => ${msgId}::bigint);
|
|
40
|
-
`;
|
|
41
|
-
}
|
|
42
|
-
async archiveBatch(msgIds) {
|
|
43
|
-
this.logger.debug(`Archiving ${msgIds.length} messages from queue '${this.queueName}'`);
|
|
44
|
-
await this.sql `
|
|
45
|
-
SELECT pgmq.archive(queue_name => ${this.queueName}, msg_ids => ${msgIds}::bigint[]);
|
|
46
|
-
`;
|
|
47
|
-
}
|
|
48
|
-
async send(message) {
|
|
49
|
-
this.logger.debug(`Sending message to queue '${this.queueName}'`);
|
|
50
|
-
const msgJson = JSON.stringify(message);
|
|
51
|
-
await this.sql `
|
|
52
|
-
SELECT pgmq.send(queue_name => ${this.queueName}, msg => ${msgJson}::jsonb)
|
|
53
|
-
`;
|
|
54
|
-
}
|
|
55
|
-
async readWithPoll(batchSize = 20, visibilityTimeout = 2, maxPollSeconds = 5, pollIntervalMs = 200) {
|
|
56
|
-
this.logger.debug(`Reading messages from queue '${this.queueName}' with poll`);
|
|
57
|
-
return await this.sql `
|
|
58
|
-
SELECT *
|
|
59
|
-
FROM edge_worker.read_with_poll(
|
|
60
|
-
queue_name => ${this.queueName},
|
|
61
|
-
vt => ${visibilityTimeout},
|
|
62
|
-
qty => ${batchSize},
|
|
63
|
-
max_poll_seconds => ${maxPollSeconds},
|
|
64
|
-
poll_interval_ms => ${pollIntervalMs}
|
|
65
|
-
);
|
|
66
|
-
`;
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Sets the visibility timeout of a message to the current time plus the given offset.
|
|
70
|
-
*
|
|
71
|
-
* This is an inlined version of the pgmq.set_vt in order to fix the bug.
|
|
72
|
-
* The original uses now() instead of clock_timestamp() which is problematic in transactions.
|
|
73
|
-
* See more details here: https://github.com/tembo-io/pgmq/issues/367
|
|
74
|
-
*
|
|
75
|
-
* The only change made is now() replaced with clock_timestamp().
|
|
76
|
-
*/
|
|
77
|
-
async setVt(msgId, vtOffsetSeconds) {
|
|
78
|
-
this.logger.debug(`Setting visibility timeout for message ${msgId} to ${vtOffsetSeconds} seconds`);
|
|
79
|
-
const records = await this.sql `
|
|
80
|
-
UPDATE ${this.sql('pgmq.q_' + this.queueName)}
|
|
81
|
-
SET vt = (clock_timestamp() + make_interval(secs => ${vtOffsetSeconds}))
|
|
82
|
-
WHERE msg_id = ${msgId}::bigint
|
|
83
|
-
RETURNING *;
|
|
84
|
-
`;
|
|
85
|
-
return records[0];
|
|
86
|
-
}
|
|
87
|
-
}
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import type { Queue } from './Queue.js';
|
|
2
|
-
import type { PgmqMessageRecord } from './types.js';
|
|
3
|
-
import type { Json } from '../core/types.js';
|
|
4
|
-
import type { Logger } from '../platform/types.js';
|
|
5
|
-
export interface PollerConfig {
|
|
6
|
-
batchSize: number;
|
|
7
|
-
maxPollSeconds: number;
|
|
8
|
-
pollIntervalMs: number;
|
|
9
|
-
visibilityTimeout: number;
|
|
10
|
-
}
|
|
11
|
-
export declare class ReadWithPollPoller<TPayload extends Json> {
|
|
12
|
-
protected readonly queue: Queue<TPayload>;
|
|
13
|
-
protected readonly signal: AbortSignal;
|
|
14
|
-
protected readonly config: PollerConfig;
|
|
15
|
-
private logger;
|
|
16
|
-
constructor(queue: Queue<TPayload>, signal: AbortSignal, config: PollerConfig, logger: Logger);
|
|
17
|
-
poll(): Promise<PgmqMessageRecord<TPayload>[]>;
|
|
18
|
-
private isAborted;
|
|
19
|
-
}
|
|
20
|
-
//# sourceMappingURL=ReadWithPollPoller.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ReadWithPollPoller.d.ts","sourceRoot":"","sources":["../../src/queue/ReadWithPollPoller.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACxC,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;IACvB,cAAc,EAAE,MAAM,CAAC;IACvB,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,qBAAa,kBAAkB,CAAC,QAAQ,SAAS,IAAI;IAIjD,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC;IACzC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,WAAW;IACtC,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY;IALzC,OAAO,CAAC,MAAM,CAAS;gBAGF,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,EACtB,MAAM,EAAE,WAAW,EACnB,MAAM,EAAE,YAAY,EACvC,MAAM,EAAE,MAAM;IAKV,IAAI,IAAI,OAAO,CAAC,iBAAiB,CAAC,QAAQ,CAAC,EAAE,CAAC;IAkBpD,OAAO,CAAC,SAAS;CAGlB"}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
export class ReadWithPollPoller {
|
|
2
|
-
queue;
|
|
3
|
-
signal;
|
|
4
|
-
config;
|
|
5
|
-
logger;
|
|
6
|
-
constructor(queue, signal, config, logger) {
|
|
7
|
-
this.queue = queue;
|
|
8
|
-
this.signal = signal;
|
|
9
|
-
this.config = config;
|
|
10
|
-
this.logger = logger;
|
|
11
|
-
}
|
|
12
|
-
async poll() {
|
|
13
|
-
if (this.isAborted()) {
|
|
14
|
-
this.logger.debug('Polling aborted, returning empty array');
|
|
15
|
-
return [];
|
|
16
|
-
}
|
|
17
|
-
this.logger.debug(`Polling queue '${this.queue.queueName}' with batch size ${this.config.batchSize}`);
|
|
18
|
-
const messages = await this.queue.readWithPoll(this.config.batchSize, this.config.visibilityTimeout, this.config.maxPollSeconds, this.config.pollIntervalMs);
|
|
19
|
-
this.logger.debug(`Received ${messages.length} messages from queue '${this.queue.queueName}'`);
|
|
20
|
-
return messages;
|
|
21
|
-
}
|
|
22
|
-
isAborted() {
|
|
23
|
-
return this.signal.aborted;
|
|
24
|
-
}
|
|
25
|
-
}
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
import type { Json } from '../core/types.js';
|
|
2
|
-
import type { MessageHandlerFn } from './types.js';
|
|
3
|
-
import { Worker } from '../core/Worker.js';
|
|
4
|
-
import postgres from 'postgres';
|
|
5
|
-
import type { Logger } from '../platform/types.js';
|
|
6
|
-
/**
|
|
7
|
-
* Configuration for the queue worker
|
|
8
|
-
*/
|
|
9
|
-
export type QueueWorkerConfig = {
|
|
10
|
-
/**
|
|
11
|
-
* PostgreSQL connection string.
|
|
12
|
-
* If not provided, it will be read from the EDGE_WORKER_DB_URL environment variable.
|
|
13
|
-
*/
|
|
14
|
-
connectionString?: string;
|
|
15
|
-
/**
|
|
16
|
-
* Name of the queue to poll for messages
|
|
17
|
-
* @default 'tasks'
|
|
18
|
-
*/
|
|
19
|
-
queueName?: string;
|
|
20
|
-
/**
|
|
21
|
-
* How many tasks are processed at the same time
|
|
22
|
-
* @default 10
|
|
23
|
-
*/
|
|
24
|
-
maxConcurrent?: number;
|
|
25
|
-
/**
|
|
26
|
-
* How many connections to the database are opened
|
|
27
|
-
* @default 4
|
|
28
|
-
*/
|
|
29
|
-
maxPgConnections?: number;
|
|
30
|
-
/**
|
|
31
|
-
* In-worker polling interval in seconds
|
|
32
|
-
* @default 5
|
|
33
|
-
*/
|
|
34
|
-
maxPollSeconds?: number;
|
|
35
|
-
/**
|
|
36
|
-
* In-database polling interval in milliseconds
|
|
37
|
-
* @default 200
|
|
38
|
-
*/
|
|
39
|
-
pollIntervalMs?: number;
|
|
40
|
-
/**
|
|
41
|
-
* How long to wait before retrying a failed job in seconds
|
|
42
|
-
* @default 5
|
|
43
|
-
*/
|
|
44
|
-
retryDelay?: number;
|
|
45
|
-
/**
|
|
46
|
-
* How many times to retry a failed job
|
|
47
|
-
* @default 5
|
|
48
|
-
*/
|
|
49
|
-
retryLimit?: number;
|
|
50
|
-
/**
|
|
51
|
-
* How long a job is invisible after reading in seconds.
|
|
52
|
-
* If not successful, will reappear after this time.
|
|
53
|
-
* @default 3
|
|
54
|
-
*/
|
|
55
|
-
visibilityTimeout?: number;
|
|
56
|
-
/**
|
|
57
|
-
* Batch size for polling messages
|
|
58
|
-
* @default 10
|
|
59
|
-
*/
|
|
60
|
-
batchSize?: number;
|
|
61
|
-
/**
|
|
62
|
-
* Optional SQL client instance
|
|
63
|
-
*/
|
|
64
|
-
sql?: postgres.Sql;
|
|
65
|
-
};
|
|
66
|
-
/**
|
|
67
|
-
* Creates a new Worker instance for processing queue messages.
|
|
68
|
-
*
|
|
69
|
-
* @param handler - The message handler function that processes each message from the queue
|
|
70
|
-
* @param config - Configuration options for the worker
|
|
71
|
-
* @param createLogger - Function to create loggers for different components
|
|
72
|
-
* @returns A configured Worker instance ready to be started
|
|
73
|
-
*/
|
|
74
|
-
export declare function createQueueWorker<TPayload extends Json>(handler: MessageHandlerFn<TPayload>, config: QueueWorkerConfig, createLogger: (module: string) => Logger): Worker;
|
|
75
|
-
//# sourceMappingURL=createQueueWorker.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"createQueueWorker.d.ts","sourceRoot":"","sources":["../../src/queue/createQueueWorker.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,KAAK,EAAqB,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,QAAQ,MAAM,UAAU,CAAC;AAGhC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAEnD;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAE3B;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;OAEG;IACH,GAAG,CAAC,EAAE,QAAQ,CAAC,GAAG,CAAC;CACpB,CAAC;AAEF;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,SAAS,IAAI,EACrD,OAAO,EAAE,gBAAgB,CAAC,QAAQ,CAAC,EACnC,MAAM,EAAE,iBAAiB,EACzB,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,GACvC,MAAM,CAyER"}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import { ExecutionController } from '../core/ExecutionController.js';
|
|
2
|
-
import { MessageExecutor } from './MessageExecutor.js';
|
|
3
|
-
import { Queries } from '../core/Queries.js';
|
|
4
|
-
import { Queue } from './Queue.js';
|
|
5
|
-
import { ReadWithPollPoller } from './ReadWithPollPoller.js';
|
|
6
|
-
import { Worker } from '../core/Worker.js';
|
|
7
|
-
import postgres from 'postgres';
|
|
8
|
-
import { WorkerLifecycle } from '../core/WorkerLifecycle.js';
|
|
9
|
-
import { BatchProcessor } from '../core/BatchProcessor.js';
|
|
10
|
-
/**
|
|
11
|
-
* Creates a new Worker instance for processing queue messages.
|
|
12
|
-
*
|
|
13
|
-
* @param handler - The message handler function that processes each message from the queue
|
|
14
|
-
* @param config - Configuration options for the worker
|
|
15
|
-
* @param createLogger - Function to create loggers for different components
|
|
16
|
-
* @returns A configured Worker instance ready to be started
|
|
17
|
-
*/
|
|
18
|
-
export function createQueueWorker(handler, config, createLogger) {
|
|
19
|
-
// Create component-specific loggers
|
|
20
|
-
const logger = createLogger('QueueWorker');
|
|
21
|
-
logger.info(`Creating queue worker for ${config.queueName || 'tasks'}`);
|
|
22
|
-
const abortController = new AbortController();
|
|
23
|
-
const abortSignal = abortController.signal;
|
|
24
|
-
// Use provided SQL connection if available, otherwise create one from connection string
|
|
25
|
-
const sql = config.sql ||
|
|
26
|
-
postgres(config.connectionString || '', {
|
|
27
|
-
max: config.maxPgConnections,
|
|
28
|
-
prepare: false,
|
|
29
|
-
});
|
|
30
|
-
const queue = new Queue(sql, config.queueName || 'tasks', createLogger('Queue'));
|
|
31
|
-
const queries = new Queries(sql);
|
|
32
|
-
const lifecycle = new WorkerLifecycle(queries, queue, createLogger('WorkerLifecycle'));
|
|
33
|
-
const executorFactory = (record, signal) => {
|
|
34
|
-
return new MessageExecutor(queue, record, handler, signal, config.retryLimit || 5, config.retryDelay || 3, createLogger('MessageExecutor'));
|
|
35
|
-
};
|
|
36
|
-
const poller = new ReadWithPollPoller(queue, abortSignal, {
|
|
37
|
-
batchSize: config.batchSize || config.maxConcurrent || 10,
|
|
38
|
-
maxPollSeconds: config.maxPollSeconds || 5,
|
|
39
|
-
pollIntervalMs: config.pollIntervalMs || 200,
|
|
40
|
-
visibilityTimeout: config.visibilityTimeout || 3,
|
|
41
|
-
}, createLogger('ReadWithPollPoller'));
|
|
42
|
-
const executionController = new ExecutionController(executorFactory, abortSignal, {
|
|
43
|
-
maxConcurrent: config.maxConcurrent || 10,
|
|
44
|
-
}, createLogger('ExecutionController'));
|
|
45
|
-
const batchProcessor = new BatchProcessor(executionController, poller, abortSignal, createLogger('BatchProcessor'));
|
|
46
|
-
return new Worker(batchProcessor, lifecycle, sql, createLogger('Worker'));
|
|
47
|
-
}
|
package/dist/queue/types.d.ts
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import type { Json, IMessage } from '../core/types.js';
|
|
2
|
-
/**
|
|
3
|
-
* Fields are nullable because types in postgres does not allow NOT NULL,
|
|
4
|
-
* but all those values except `message` come from queue table columns,
|
|
5
|
-
* which are explicitely marked as NOT NULL.
|
|
6
|
-
*/
|
|
7
|
-
export interface PgmqMessageRecord<TPayload extends Json | null = Json> extends IMessage {
|
|
8
|
-
read_ct: number;
|
|
9
|
-
enqueued_at: string;
|
|
10
|
-
vt: string;
|
|
11
|
-
message: TPayload;
|
|
12
|
-
}
|
|
13
|
-
/**
|
|
14
|
-
* User-provided handler function, called for each message in the queue
|
|
15
|
-
*/
|
|
16
|
-
export type MessageHandlerFn<TPayload> = (message: TPayload) => Promise<void> | void;
|
|
17
|
-
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/queue/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAEvD;;;;GAIG;AACH,MAAM,WAAW,iBAAiB,CAAC,QAAQ,SAAS,IAAI,GAAG,IAAI,GAAG,IAAI,CACpE,SAAQ,QAAQ;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,QAAQ,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,CAAC,QAAQ,IAAI,CACvC,OAAO,EAAE,QAAQ,KACd,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC"}
|
package/dist/queue/types.js
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|