duron 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +7 -0
- package/README.md +140 -0
- package/dist/action-job.d.ts +24 -0
- package/dist/action-job.d.ts.map +1 -0
- package/dist/action-job.js +108 -0
- package/dist/action-manager.d.ts +21 -0
- package/dist/action-manager.d.ts.map +1 -0
- package/dist/action-manager.js +78 -0
- package/dist/action.d.ts +129 -0
- package/dist/action.d.ts.map +1 -0
- package/dist/action.js +87 -0
- package/dist/adapters/adapter.d.ts +92 -0
- package/dist/adapters/adapter.d.ts.map +1 -0
- package/dist/adapters/adapter.js +424 -0
- package/dist/adapters/postgres/drizzle.config.d.ts +3 -0
- package/dist/adapters/postgres/drizzle.config.d.ts.map +1 -0
- package/dist/adapters/postgres/drizzle.config.js +10 -0
- package/dist/adapters/postgres/pglite.d.ts +13 -0
- package/dist/adapters/postgres/pglite.d.ts.map +1 -0
- package/dist/adapters/postgres/pglite.js +36 -0
- package/dist/adapters/postgres/postgres.d.ts +51 -0
- package/dist/adapters/postgres/postgres.d.ts.map +1 -0
- package/dist/adapters/postgres/postgres.js +867 -0
- package/dist/adapters/postgres/schema.d.ts +581 -0
- package/dist/adapters/postgres/schema.d.ts.map +1 -0
- package/dist/adapters/postgres/schema.default.d.ts +577 -0
- package/dist/adapters/postgres/schema.default.d.ts.map +1 -0
- package/dist/adapters/postgres/schema.default.js +3 -0
- package/dist/adapters/postgres/schema.js +87 -0
- package/dist/adapters/schemas.d.ts +516 -0
- package/dist/adapters/schemas.d.ts.map +1 -0
- package/dist/adapters/schemas.js +184 -0
- package/dist/client.d.ts +85 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +416 -0
- package/dist/constants.d.ts +14 -0
- package/dist/constants.d.ts.map +1 -0
- package/dist/constants.js +22 -0
- package/dist/errors.d.ts +43 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +75 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +6 -0
- package/dist/server.d.ts +1193 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +516 -0
- package/dist/step-manager.d.ts +46 -0
- package/dist/step-manager.d.ts.map +1 -0
- package/dist/step-manager.js +216 -0
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.d.ts.map +1 -0
- package/dist/utils/checksum.js +6 -0
- package/dist/utils/p-retry.d.ts +19 -0
- package/dist/utils/p-retry.d.ts.map +1 -0
- package/dist/utils/p-retry.js +130 -0
- package/dist/utils/wait-for-abort.d.ts +5 -0
- package/dist/utils/wait-for-abort.d.ts.map +1 -0
- package/dist/utils/wait-for-abort.js +32 -0
- package/migrations/postgres/0000_lethal_speed_demon.sql +64 -0
- package/migrations/postgres/meta/0000_snapshot.json +606 -0
- package/migrations/postgres/meta/_journal.json +13 -0
- package/package.json +88 -0
- package/src/action-job.ts +201 -0
- package/src/action-manager.ts +166 -0
- package/src/action.ts +247 -0
- package/src/adapters/adapter.ts +969 -0
- package/src/adapters/postgres/drizzle.config.ts +11 -0
- package/src/adapters/postgres/pglite.ts +86 -0
- package/src/adapters/postgres/postgres.ts +1346 -0
- package/src/adapters/postgres/schema.default.ts +5 -0
- package/src/adapters/postgres/schema.ts +119 -0
- package/src/adapters/schemas.ts +320 -0
- package/src/client.ts +859 -0
- package/src/constants.ts +37 -0
- package/src/errors.ts +205 -0
- package/src/index.ts +14 -0
- package/src/server.ts +718 -0
- package/src/step-manager.ts +471 -0
- package/src/utils/checksum.ts +7 -0
- package/src/utils/p-retry.ts +213 -0
- package/src/utils/wait-for-abort.ts +40 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
Copyright 2025 Geut
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
|
|
4
|
+
|
|
5
|
+
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
|
|
6
|
+
|
|
7
|
+
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# Duron
|
|
2
|
+
|
|
3
|
+
A powerful, type-safe job queue system for Node.js and Bun.js. Duron provides a robust foundation for executing asynchronous tasks with built-in retry logic, concurrency control, step-based execution, and comprehensive observability.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
# Using bun
|
|
9
|
+
bun add duron
|
|
10
|
+
|
|
11
|
+
# Using npm
|
|
12
|
+
npm install duron
|
|
13
|
+
|
|
14
|
+
# Using pnpm
|
|
15
|
+
pnpm add duron
|
|
16
|
+
|
|
17
|
+
# Using yarn
|
|
18
|
+
yarn add duron
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Quick Start
|
|
22
|
+
|
|
23
|
+
### 1. Define an Action
|
|
24
|
+
|
|
25
|
+
Actions are the building blocks of Duron. They define what work needs to be done:
|
|
26
|
+
|
|
27
|
+
```typescript
|
|
28
|
+
import { defineAction } from 'duron'
|
|
29
|
+
import { z } from 'zod'
|
|
30
|
+
|
|
31
|
+
const sendEmail = defineAction()({
|
|
32
|
+
name: 'send-email',
|
|
33
|
+
input: z.object({
|
|
34
|
+
to: z.string().email(),
|
|
35
|
+
subject: z.string(),
|
|
36
|
+
body: z.string(),
|
|
37
|
+
}),
|
|
38
|
+
output: z.object({
|
|
39
|
+
success: z.boolean(),
|
|
40
|
+
}),
|
|
41
|
+
handler: async (ctx) => {
|
|
42
|
+
const { to, subject, body } = ctx.input
|
|
43
|
+
|
|
44
|
+
// Use steps to break down work into retryable units
|
|
45
|
+
const result = await ctx.step('send-email', async ({ signal }) => {
|
|
46
|
+
const response = await fetch('https://api.email.com/send', {
|
|
47
|
+
method: 'POST',
|
|
48
|
+
headers: { 'Content-Type': 'application/json' },
|
|
49
|
+
body: JSON.stringify({ to, subject, body }),
|
|
50
|
+
signal, // Pass signal to enable cancellation
|
|
51
|
+
})
|
|
52
|
+
|
|
53
|
+
if (!response.ok) {
|
|
54
|
+
throw new Error('Failed to send email')
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return await response.json()
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
return { success: result.success ?? true }
|
|
61
|
+
},
|
|
62
|
+
})
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### 2. Create a Client
|
|
66
|
+
|
|
67
|
+
```typescript
|
|
68
|
+
import { duron } from 'duron'
|
|
69
|
+
import { postgresAdapter } from 'duron/adapters/postgres'
|
|
70
|
+
|
|
71
|
+
const client = duron({
|
|
72
|
+
database: postgresAdapter({
|
|
73
|
+
connection: process.env.DATABASE_URL,
|
|
74
|
+
}),
|
|
75
|
+
actions: {
|
|
76
|
+
sendEmail,
|
|
77
|
+
},
|
|
78
|
+
logger: 'info',
|
|
79
|
+
})
|
|
80
|
+
|
|
81
|
+
await client.start()
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### 3. Run Actions
|
|
85
|
+
|
|
86
|
+
```typescript
|
|
87
|
+
// Run an action
|
|
88
|
+
const jobId = await client.runAction('send-email', {
|
|
89
|
+
to: 'user@example.com',
|
|
90
|
+
subject: 'Hello',
|
|
91
|
+
body: 'Welcome!',
|
|
92
|
+
})
|
|
93
|
+
|
|
94
|
+
// Wait for the job to complete
|
|
95
|
+
const job = await client.waitForJob(jobId)
|
|
96
|
+
console.log('Job completed:', job?.output)
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
## Key Features
|
|
100
|
+
|
|
101
|
+
- **Type-Safe** - Full TypeScript support with Zod validation
|
|
102
|
+
- **Step-Based Execution** - Break down complex workflows into manageable, retryable steps
|
|
103
|
+
- **Intelligent Retry Logic** - Configurable exponential backoff with per-action and per-step options
|
|
104
|
+
- **Flexible Sync Patterns** - Pull, push, hybrid, or manual job fetching
|
|
105
|
+
- **Advanced Concurrency Control** - Per-action, per-group, and dynamic concurrency limits
|
|
106
|
+
- **Reliability & Recovery** - Automatic job recovery, multi-process coordination, and stuck job detection
|
|
107
|
+
- **Database Adapters** - PostgreSQL (production) and PGLite (development/testing)
|
|
108
|
+
- **REST API Server** - Built-in Elysia-based API with advanced filtering and pagination
|
|
109
|
+
|
|
110
|
+
## Documentation
|
|
111
|
+
|
|
112
|
+
- [Getting Started](https://duron.dev/docs/getting-started)
|
|
113
|
+
- [Actions](https://duron.dev/docs/actions)
|
|
114
|
+
- [Jobs and Steps](https://duron.dev/docs/jobs-and-steps)
|
|
115
|
+
- [Client API](https://duron.dev/docs/client-api)
|
|
116
|
+
- [Server API](https://duron.dev/docs/server-api)
|
|
117
|
+
- [Adapters](https://duron.dev/docs/adapters)
|
|
118
|
+
- [Retries](https://duron.dev/docs/retries)
|
|
119
|
+
- [Error Handling](https://duron.dev/docs/error-handling)
|
|
120
|
+
- [Examples](https://duron.dev/docs/examples)
|
|
121
|
+
|
|
122
|
+
## Development
|
|
123
|
+
|
|
124
|
+
```bash
|
|
125
|
+
# Install dependencies
|
|
126
|
+
bun install
|
|
127
|
+
|
|
128
|
+
# Run tests
|
|
129
|
+
bun test
|
|
130
|
+
|
|
131
|
+
# Build
|
|
132
|
+
bun run build
|
|
133
|
+
|
|
134
|
+
# Type check
|
|
135
|
+
bun run typecheck
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
## License
|
|
139
|
+
|
|
140
|
+
MIT
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Logger } from 'pino';
|
|
2
|
+
import type { Action } from './action.js';
|
|
3
|
+
import type { Adapter } from './adapters/adapter.js';
|
|
4
|
+
export interface ActionJobOptions<TAction extends Action<any, any, any>> {
|
|
5
|
+
job: {
|
|
6
|
+
id: string;
|
|
7
|
+
input: any;
|
|
8
|
+
groupKey: string;
|
|
9
|
+
timeoutMs: number;
|
|
10
|
+
actionName: string;
|
|
11
|
+
};
|
|
12
|
+
action: TAction;
|
|
13
|
+
database: Adapter;
|
|
14
|
+
variables: Record<string, unknown>;
|
|
15
|
+
logger: Logger;
|
|
16
|
+
}
|
|
17
|
+
export declare class ActionJob<TAction extends Action<any, any, any>> {
|
|
18
|
+
#private;
|
|
19
|
+
constructor(options: ActionJobOptions<TAction>);
|
|
20
|
+
execute(): Promise<any>;
|
|
21
|
+
waitForDone(): Promise<void>;
|
|
22
|
+
cancel(): void;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=action-job.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"action-job.d.ts","sourceRoot":"","sources":["../src/action-job.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAElC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,uBAAuB,CAAA;AAKpD,MAAM,WAAW,gBAAgB,CAAC,OAAO,SAAS,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;IACrE,GAAG,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,GAAG,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IACxF,MAAM,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,OAAO,CAAA;IACjB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAClC,MAAM,EAAE,MAAM,CAAA;CACf;AAQD,qBAAa,SAAS,CAAC,OAAO,SAAS,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;;gBAqB9C,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC;IAoCxC,OAAO;IA8Fb,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B,MAAM;CAmBP"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { ActionCancelError, ActionTimeoutError, isCancelError, StepTimeoutError, serializeError } from './errors.js';
|
|
2
|
+
import { StepManager } from './step-manager.js';
|
|
3
|
+
import waitForAbort from './utils/wait-for-abort.js';
|
|
4
|
+
export class ActionJob {
|
|
5
|
+
#job;
|
|
6
|
+
#action;
|
|
7
|
+
#database;
|
|
8
|
+
#variables;
|
|
9
|
+
#logger;
|
|
10
|
+
#stepManager;
|
|
11
|
+
#abortController;
|
|
12
|
+
#timeoutId = null;
|
|
13
|
+
#done;
|
|
14
|
+
#resolve = null;
|
|
15
|
+
constructor(options) {
|
|
16
|
+
this.#job = options.job;
|
|
17
|
+
this.#action = options.action;
|
|
18
|
+
this.#database = options.database;
|
|
19
|
+
this.#variables = options.variables;
|
|
20
|
+
this.#logger = options.logger;
|
|
21
|
+
this.#abortController = new AbortController();
|
|
22
|
+
this.#stepManager = new StepManager({
|
|
23
|
+
jobId: options.job.id,
|
|
24
|
+
actionName: options.job.actionName,
|
|
25
|
+
adapter: options.database,
|
|
26
|
+
logger: options.logger,
|
|
27
|
+
concurrencyLimit: options.action.concurrency,
|
|
28
|
+
});
|
|
29
|
+
this.#done = new Promise((resolve) => {
|
|
30
|
+
this.#resolve = resolve;
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
async execute() {
|
|
34
|
+
try {
|
|
35
|
+
const jobLogger = this.#logger.child({
|
|
36
|
+
jobId: this.#job.id,
|
|
37
|
+
actionName: this.#action.name,
|
|
38
|
+
});
|
|
39
|
+
const ctx = this.#stepManager.createActionContext(this.#job, this.#action, this.#variables, this.#abortController.signal, jobLogger);
|
|
40
|
+
this.#timeoutId = setTimeout(() => {
|
|
41
|
+
const timeoutError = new ActionTimeoutError(this.#action.name, this.#job.timeoutMs);
|
|
42
|
+
this.#abortController.abort(timeoutError);
|
|
43
|
+
}, this.#job.timeoutMs);
|
|
44
|
+
this.#timeoutId?.unref?.();
|
|
45
|
+
const abortWaiter = waitForAbort(this.#abortController.signal);
|
|
46
|
+
let result = null;
|
|
47
|
+
await Promise.race([
|
|
48
|
+
this.#action
|
|
49
|
+
.handler(ctx)
|
|
50
|
+
.then((res) => {
|
|
51
|
+
if (res !== undefined) {
|
|
52
|
+
result = res;
|
|
53
|
+
}
|
|
54
|
+
})
|
|
55
|
+
.finally(() => {
|
|
56
|
+
abortWaiter.release();
|
|
57
|
+
}),
|
|
58
|
+
abortWaiter.promise,
|
|
59
|
+
]);
|
|
60
|
+
if (this.#action.output) {
|
|
61
|
+
result = this.#action.output.parse(result, {
|
|
62
|
+
error: () => 'Error parsing action output',
|
|
63
|
+
reportInput: true,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
const completed = await this.#database.completeJob({ jobId: this.#job.id, output: result });
|
|
67
|
+
if (!completed) {
|
|
68
|
+
throw new Error('Job not completed');
|
|
69
|
+
}
|
|
70
|
+
this.#logger.debug({ jobId: this.#job.id, actionName: this.#action.name }, '[ActionJob] Action finished executing');
|
|
71
|
+
return result;
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
if (isCancelError(error) ||
|
|
75
|
+
(error instanceof Error && error.name === 'AbortError' && isCancelError(error.cause))) {
|
|
76
|
+
this.#logger.warn({ jobId: this.#job.id, actionName: this.#action.name }, '[ActionJob] Job cancelled');
|
|
77
|
+
await this.#database.cancelJob({ jobId: this.#job.id });
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
const message = error instanceof ActionTimeoutError
|
|
81
|
+
? '[ActionJob] Job timed out'
|
|
82
|
+
: error instanceof StepTimeoutError
|
|
83
|
+
? '[ActionJob] Step timed out'
|
|
84
|
+
: '[ActionJob] Job failed';
|
|
85
|
+
this.#logger.error({ jobId: this.#job.id, actionName: this.#action.name }, message);
|
|
86
|
+
await this.#database.failJob({ jobId: this.#job.id, error: serializeError(error) });
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
this.#clear();
|
|
91
|
+
this.#resolve?.();
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
waitForDone() {
|
|
95
|
+
return this.#done;
|
|
96
|
+
}
|
|
97
|
+
cancel() {
|
|
98
|
+
this.#clear();
|
|
99
|
+
const cancelError = new ActionCancelError(this.#action.name, this.#job.id);
|
|
100
|
+
this.#abortController.abort(cancelError);
|
|
101
|
+
}
|
|
102
|
+
#clear() {
|
|
103
|
+
if (this.#timeoutId) {
|
|
104
|
+
clearTimeout(this.#timeoutId);
|
|
105
|
+
this.#timeoutId = null;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Logger } from 'pino';
|
|
2
|
+
import type { Action } from './action.js';
|
|
3
|
+
import type { Adapter, Job } from './adapters/adapter.js';
|
|
4
|
+
export interface ActionManagerOptions<TAction extends Action<any, any, any>> {
|
|
5
|
+
action: TAction;
|
|
6
|
+
database: Adapter;
|
|
7
|
+
variables: Record<string, unknown>;
|
|
8
|
+
logger: Logger;
|
|
9
|
+
concurrencyLimit: number;
|
|
10
|
+
}
|
|
11
|
+
export declare class ActionManager<TAction extends Action<any, any, any>> {
|
|
12
|
+
#private;
|
|
13
|
+
constructor(options: ActionManagerOptions<TAction>);
|
|
14
|
+
push(job: Job): Promise<void>;
|
|
15
|
+
cancelJob(jobId: string): boolean;
|
|
16
|
+
abortAll(): void;
|
|
17
|
+
idle(): Promise<boolean>;
|
|
18
|
+
drain(): Promise<void>;
|
|
19
|
+
stop(): Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=action-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"action-manager.d.ts","sourceRoot":"","sources":["../src/action-manager.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAElC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AAEzC,OAAO,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,uBAAuB,CAAA;AAEzD,MAAM,WAAW,oBAAoB,CAAC,OAAO,SAAS,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;IACzE,MAAM,EAAE,OAAO,CAAA;IACf,QAAQ,EAAE,OAAO,CAAA;IACjB,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAClC,MAAM,EAAE,MAAM,CAAA;IACd,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAQD,qBAAa,aAAa,CAAC,OAAO,SAAS,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;;gBAmBlD,OAAO,EAAE,oBAAoB,CAAC,OAAO,CAAC;IA2B5C,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC;IAUnC,SAAS,CAAC,KAAK,EAAE,MAAM;IAYvB,QAAQ,IAAI,IAAI;IAWV,IAAI,IAAI,OAAO,CAAC,OAAO,CAAC;IASxB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAUtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;CA8C5B"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import fastq from 'fastq';
|
|
2
|
+
import { ActionJob } from './action-job.js';
|
|
3
|
+
export class ActionManager {
|
|
4
|
+
#action;
|
|
5
|
+
#database;
|
|
6
|
+
#variables;
|
|
7
|
+
#logger;
|
|
8
|
+
#queue;
|
|
9
|
+
#concurrencyLimit;
|
|
10
|
+
#activeJobs = new Map();
|
|
11
|
+
#stopped = false;
|
|
12
|
+
constructor(options) {
|
|
13
|
+
this.#action = options.action;
|
|
14
|
+
this.#database = options.database;
|
|
15
|
+
this.#variables = options.variables;
|
|
16
|
+
this.#logger = options.logger;
|
|
17
|
+
this.#concurrencyLimit = options.concurrencyLimit;
|
|
18
|
+
this.#queue = fastq.promise(async (job) => {
|
|
19
|
+
if (this.#stopped) {
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
await this.#executeJob(job);
|
|
23
|
+
}, this.#concurrencyLimit);
|
|
24
|
+
}
|
|
25
|
+
async push(job) {
|
|
26
|
+
return this.#queue.push(job);
|
|
27
|
+
}
|
|
28
|
+
cancelJob(jobId) {
|
|
29
|
+
const actionJob = this.#activeJobs.get(jobId);
|
|
30
|
+
if (actionJob) {
|
|
31
|
+
actionJob.cancel();
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
abortAll() {
|
|
37
|
+
for (const actionJob of this.#activeJobs.values()) {
|
|
38
|
+
actionJob.cancel();
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
async idle() {
|
|
42
|
+
return this.#queue.idle();
|
|
43
|
+
}
|
|
44
|
+
async drain() {
|
|
45
|
+
return this.#queue.drain();
|
|
46
|
+
}
|
|
47
|
+
async stop() {
|
|
48
|
+
if (this.#stopped) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
this.#stopped = true;
|
|
52
|
+
this.abortAll();
|
|
53
|
+
await this.#queue.killAndDrain();
|
|
54
|
+
await Promise.all(Array.from(this.#activeJobs.values()).map((actionJob) => actionJob.waitForDone()));
|
|
55
|
+
}
|
|
56
|
+
async #executeJob(job) {
|
|
57
|
+
const actionJob = new ActionJob({
|
|
58
|
+
job: {
|
|
59
|
+
id: job.id,
|
|
60
|
+
input: job.input,
|
|
61
|
+
groupKey: job.groupKey,
|
|
62
|
+
timeoutMs: job.timeoutMs,
|
|
63
|
+
actionName: job.actionName,
|
|
64
|
+
},
|
|
65
|
+
action: this.#action,
|
|
66
|
+
database: this.#database,
|
|
67
|
+
variables: this.#variables,
|
|
68
|
+
logger: this.#logger,
|
|
69
|
+
});
|
|
70
|
+
this.#activeJobs.set(job.id, actionJob);
|
|
71
|
+
try {
|
|
72
|
+
await actionJob.execute();
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
this.#activeJobs.delete(job.id);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
package/dist/action.d.ts
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import type { Logger } from 'pino';
|
|
2
|
+
import * as z from 'zod';
|
|
3
|
+
export type RetryOptions = z.infer<typeof RetryOptionsSchema>;
|
|
4
|
+
export type StepOptions = z.infer<typeof StepOptionsSchema>;
|
|
5
|
+
export interface ActionHandlerContext<TInput extends z.ZodObject, TVariables = Record<string, unknown>> {
|
|
6
|
+
input: z.infer<TInput>;
|
|
7
|
+
jobId: string;
|
|
8
|
+
groupKey: string;
|
|
9
|
+
var: TVariables;
|
|
10
|
+
logger: Logger;
|
|
11
|
+
step: <TResult>(name: string, cb: (ctx: StepHandlerContext) => Promise<TResult>, options?: z.input<typeof StepOptionsSchema>) => Promise<TResult>;
|
|
12
|
+
}
|
|
13
|
+
export interface StepHandlerContext {
|
|
14
|
+
signal: AbortSignal;
|
|
15
|
+
}
|
|
16
|
+
export interface ConcurrencyHandlerContext<TInput extends z.ZodObject, TVariables = Record<string, unknown>> {
|
|
17
|
+
input: z.infer<TInput>;
|
|
18
|
+
var: TVariables;
|
|
19
|
+
}
|
|
20
|
+
export type ActionDefinition<TInput extends z.ZodObject, TOutput extends z.ZodObject, TVariables = Record<string, unknown>> = z.input<ReturnType<typeof createActionDefinitionSchema<TInput, TOutput, TVariables>>>;
|
|
21
|
+
export type Action<TInput extends z.ZodObject, TOutput extends z.ZodObject, TVariables = Record<string, unknown>> = z.infer<ReturnType<typeof createActionDefinitionSchema<TInput, TOutput, TVariables>>>;
|
|
22
|
+
export declare const RetryOptionsSchema: z.ZodDefault<z.ZodObject<{
|
|
23
|
+
limit: z.ZodDefault<z.ZodNumber>;
|
|
24
|
+
factor: z.ZodDefault<z.ZodNumber>;
|
|
25
|
+
minTimeout: z.ZodDefault<z.ZodNumber>;
|
|
26
|
+
maxTimeout: z.ZodDefault<z.ZodNumber>;
|
|
27
|
+
}, z.core.$strip>>;
|
|
28
|
+
export declare const StepOptionsSchema: z.ZodObject<{
|
|
29
|
+
retry: z.ZodDefault<z.ZodObject<{
|
|
30
|
+
limit: z.ZodDefault<z.ZodNumber>;
|
|
31
|
+
factor: z.ZodDefault<z.ZodNumber>;
|
|
32
|
+
minTimeout: z.ZodDefault<z.ZodNumber>;
|
|
33
|
+
maxTimeout: z.ZodDefault<z.ZodNumber>;
|
|
34
|
+
}, z.core.$strip>>;
|
|
35
|
+
expire: z.ZodDefault<z.ZodNumber>;
|
|
36
|
+
}, z.core.$strip>;
|
|
37
|
+
export declare function createActionDefinitionSchema<TInput extends z.ZodObject, TOutput extends z.ZodObject, TVariables = Record<string, unknown>>(): z.ZodPipe<z.ZodObject<{
|
|
38
|
+
name: z.ZodString;
|
|
39
|
+
version: z.ZodOptional<z.ZodString>;
|
|
40
|
+
input: z.ZodOptional<z.ZodCustom<TInput, TInput>>;
|
|
41
|
+
output: z.ZodOptional<z.ZodCustom<TOutput, TOutput>>;
|
|
42
|
+
groups: z.ZodOptional<z.ZodObject<{
|
|
43
|
+
groupKey: z.ZodOptional<z.ZodCustom<(ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<string>, (ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<string>>>;
|
|
44
|
+
concurrency: z.ZodOptional<z.ZodCustom<(ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<number>, (ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<number>>>;
|
|
45
|
+
}, z.core.$strip>>;
|
|
46
|
+
steps: z.ZodDefault<z.ZodObject<{
|
|
47
|
+
concurrency: z.ZodDefault<z.ZodNumber>;
|
|
48
|
+
retry: z.ZodDefault<z.ZodObject<{
|
|
49
|
+
limit: z.ZodDefault<z.ZodNumber>;
|
|
50
|
+
factor: z.ZodDefault<z.ZodNumber>;
|
|
51
|
+
minTimeout: z.ZodDefault<z.ZodNumber>;
|
|
52
|
+
maxTimeout: z.ZodDefault<z.ZodNumber>;
|
|
53
|
+
}, z.core.$strip>>;
|
|
54
|
+
expire: z.ZodDefault<z.ZodNumber>;
|
|
55
|
+
}, z.core.$strip>>;
|
|
56
|
+
concurrency: z.ZodDefault<z.ZodNumber>;
|
|
57
|
+
expire: z.ZodDefault<z.ZodNumber>;
|
|
58
|
+
handler: z.ZodCustom<(ctx: ActionHandlerContext<TInput, TVariables>) => Promise<z.infer<TOutput>>, (ctx: ActionHandlerContext<TInput, TVariables>) => Promise<z.infer<TOutput>>>;
|
|
59
|
+
}, z.core.$strip>, z.ZodTransform<{
|
|
60
|
+
checksum: string;
|
|
61
|
+
name: string;
|
|
62
|
+
steps: {
|
|
63
|
+
concurrency: number;
|
|
64
|
+
retry: {
|
|
65
|
+
limit: number;
|
|
66
|
+
factor: number;
|
|
67
|
+
minTimeout: number;
|
|
68
|
+
maxTimeout: number;
|
|
69
|
+
};
|
|
70
|
+
expire: number;
|
|
71
|
+
};
|
|
72
|
+
concurrency: number;
|
|
73
|
+
expire: number;
|
|
74
|
+
handler: (ctx: ActionHandlerContext<TInput, TVariables>) => Promise<z.infer<TOutput>>;
|
|
75
|
+
version?: string | undefined;
|
|
76
|
+
input?: TInput | undefined;
|
|
77
|
+
output?: TOutput | undefined;
|
|
78
|
+
groups?: {
|
|
79
|
+
groupKey?: ((ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<string>) | undefined;
|
|
80
|
+
concurrency?: ((ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<number>) | undefined;
|
|
81
|
+
} | undefined;
|
|
82
|
+
}, {
|
|
83
|
+
name: string;
|
|
84
|
+
steps: {
|
|
85
|
+
concurrency: number;
|
|
86
|
+
retry: {
|
|
87
|
+
limit: number;
|
|
88
|
+
factor: number;
|
|
89
|
+
minTimeout: number;
|
|
90
|
+
maxTimeout: number;
|
|
91
|
+
};
|
|
92
|
+
expire: number;
|
|
93
|
+
};
|
|
94
|
+
concurrency: number;
|
|
95
|
+
expire: number;
|
|
96
|
+
handler: (ctx: ActionHandlerContext<TInput, TVariables>) => Promise<z.infer<TOutput>>;
|
|
97
|
+
version?: string | undefined;
|
|
98
|
+
input?: TInput | undefined;
|
|
99
|
+
output?: TOutput | undefined;
|
|
100
|
+
groups?: {
|
|
101
|
+
groupKey?: ((ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<string>) | undefined;
|
|
102
|
+
concurrency?: ((ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<number>) | undefined;
|
|
103
|
+
} | undefined;
|
|
104
|
+
}>>;
|
|
105
|
+
export declare const defineAction: <TVariables = Record<string, unknown>>() => <TInput extends z.ZodObject, TOutput extends z.ZodObject>(def: ActionDefinition<TInput, TOutput, TVariables>) => {
|
|
106
|
+
checksum: string;
|
|
107
|
+
name: string;
|
|
108
|
+
steps: {
|
|
109
|
+
concurrency: number;
|
|
110
|
+
retry: {
|
|
111
|
+
limit: number;
|
|
112
|
+
factor: number;
|
|
113
|
+
minTimeout: number;
|
|
114
|
+
maxTimeout: number;
|
|
115
|
+
};
|
|
116
|
+
expire: number;
|
|
117
|
+
};
|
|
118
|
+
concurrency: number;
|
|
119
|
+
expire: number;
|
|
120
|
+
handler: (ctx: ActionHandlerContext<TInput, TVariables>) => Promise<z.core.output<TOutput>>;
|
|
121
|
+
version?: string | undefined;
|
|
122
|
+
input?: TInput | undefined;
|
|
123
|
+
output?: TOutput | undefined;
|
|
124
|
+
groups?: {
|
|
125
|
+
groupKey?: ((ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<string>) | undefined;
|
|
126
|
+
concurrency?: ((ctx: ConcurrencyHandlerContext<TInput, TVariables>) => Promise<number>) | undefined;
|
|
127
|
+
} | undefined;
|
|
128
|
+
};
|
|
129
|
+
//# sourceMappingURL=action.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"action.d.ts","sourceRoot":"","sources":["../src/action.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,MAAM,CAAA;AAClC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAA;AAIxB,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAA;AAE7D,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAA;AAE3D,MAAM,WAAW,oBAAoB,CAAC,MAAM,SAAS,CAAC,CAAC,SAAS,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACpG,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,GAAG,EAAE,UAAU,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,CAAC,OAAO,EACZ,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,CAAC,GAAG,EAAE,kBAAkB,KAAK,OAAO,CAAC,OAAO,CAAC,EACjD,OAAO,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,KACxC,OAAO,CAAC,OAAO,CAAC,CAAA;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,MAAM,EAAE,WAAW,CAAA;CACpB;AAED,MAAM,WAAW,yBAAyB,CAAC,MAAM,SAAS,CAAC,CAAC,SAAS,EAAE,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IACzG,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IACtB,GAAG,EAAE,UAAU,CAAA;CAChB;AAED,MAAM,MAAM,gBAAgB,CAC1B,MAAM,SAAS,CAAC,CAAC,SAAS,EAC1B,OAAO,SAAS,CAAC,CAAC,SAAS,EAC3B,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAClC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,4BAA4B,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;AAEzF,MAAM,MAAM,MAAM,CAChB,MAAM,SAAS,CAAC,CAAC,SAAS,EAC1B,OAAO,SAAS,CAAC,CAAC,SAAS,EAC3B,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,IAClC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,4BAA4B,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC,CAAC,CAAC,CAAA;AAKzF,eAAO,MAAM,kBAAkB;;;;;kBAiCC,CAAA;AAKhC,eAAO,MAAM,iBAAiB;;;;;;;;iBAiB5B,CAAA;AAUF,wBAAgB,4BAA4B,CAC1C,MAAM,SAAS,CAAC,CAAC,SAAS,EAC1B,OAAO,SAAS,CAAC,CAAC,SAAS,EAC3B,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;;;;;;kDAkDZ,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,QAAjE,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC;qDAejE,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,QAAjE,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC;;;;;;;;;;;;;;+BA6CrE,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAtE,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;;;;;;;;;;;;;;;;mBAAtE,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;;;;;0BA5DlE,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC;6BAejE,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC;;;;;;;;;;;;;;;;mBA6CrE,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;;;;;0BA5DlE,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC;6BAejE,yBAAyB,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC;;IAyD1F;AAED,eAAO,MAAM,YAAY,GAAI,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,QACvD,MAAM,SAAS,CAAC,CAAC,SAAS,EAAE,OAAO,SAAS,CAAC,CAAC,SAAS,EAC7D,KAAK,gBAAgB,CAAC,MAAM,EAAE,OAAO,EAAE,UAAU,CAAC;;;;;;;;;;;;;;;;;;;;4EA5EsB,OAAO,CAAC,MAAM,CAAC;+EAef,OAAO,CAAC,MAAM,CAAC;;CAmE1F,CAAA"}
|
package/dist/action.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import * as z from 'zod';
|
|
2
|
+
import generateChecksum from './utils/checksum.js';
|
|
3
|
+
export const RetryOptionsSchema = z
|
|
4
|
+
.object({
|
|
5
|
+
limit: z.number().default(4),
|
|
6
|
+
factor: z.number().default(2),
|
|
7
|
+
minTimeout: z.number().default(1000),
|
|
8
|
+
maxTimeout: z.number().default(30000),
|
|
9
|
+
})
|
|
10
|
+
.default({ limit: 4, factor: 2, minTimeout: 1000, maxTimeout: 30000 })
|
|
11
|
+
.describe('The retry options');
|
|
12
|
+
export const StepOptionsSchema = z.object({
|
|
13
|
+
retry: RetryOptionsSchema,
|
|
14
|
+
expire: z
|
|
15
|
+
.number()
|
|
16
|
+
.default(5 * 60 * 1000)
|
|
17
|
+
.describe('The expire time for the step (milliseconds)'),
|
|
18
|
+
});
|
|
19
|
+
export function createActionDefinitionSchema() {
|
|
20
|
+
return z
|
|
21
|
+
.object({
|
|
22
|
+
name: z.string().describe('The name of the action'),
|
|
23
|
+
version: z.string().describe('The version of the action').optional(),
|
|
24
|
+
input: z
|
|
25
|
+
.custom((val) => {
|
|
26
|
+
return !val || ('_zod' in val && 'type' in val && val.type === 'object');
|
|
27
|
+
})
|
|
28
|
+
.optional(),
|
|
29
|
+
output: z
|
|
30
|
+
.custom((val) => {
|
|
31
|
+
return !val || ('_zod' in val && 'type' in val && val.type === 'object');
|
|
32
|
+
})
|
|
33
|
+
.optional(),
|
|
34
|
+
groups: z
|
|
35
|
+
.object({
|
|
36
|
+
groupKey: z
|
|
37
|
+
.custom((val) => {
|
|
38
|
+
return !val || val instanceof Function;
|
|
39
|
+
})
|
|
40
|
+
.optional(),
|
|
41
|
+
concurrency: z
|
|
42
|
+
.custom((val) => {
|
|
43
|
+
return !val || val instanceof Function;
|
|
44
|
+
})
|
|
45
|
+
.optional(),
|
|
46
|
+
})
|
|
47
|
+
.optional(),
|
|
48
|
+
steps: z
|
|
49
|
+
.object({
|
|
50
|
+
concurrency: z.number().default(10).describe('How many steps can run concurrently for this action'),
|
|
51
|
+
retry: RetryOptionsSchema.describe('How to retry on failure for the steps of this action'),
|
|
52
|
+
expire: z
|
|
53
|
+
.number()
|
|
54
|
+
.default(5 * 60 * 1000)
|
|
55
|
+
.describe('How long a step can run for (milliseconds)'),
|
|
56
|
+
})
|
|
57
|
+
.default({
|
|
58
|
+
concurrency: 10,
|
|
59
|
+
retry: { limit: 4, factor: 2, minTimeout: 1000, maxTimeout: 30000 },
|
|
60
|
+
expire: 5 * 60 * 1000,
|
|
61
|
+
}),
|
|
62
|
+
concurrency: z.number().default(100).describe('How many jobs can run concurrently for this action'),
|
|
63
|
+
expire: z
|
|
64
|
+
.number()
|
|
65
|
+
.default(15 * 60 * 1000)
|
|
66
|
+
.describe('How long a job can run for (milliseconds)'),
|
|
67
|
+
handler: z
|
|
68
|
+
.custom((val) => {
|
|
69
|
+
return val instanceof Function;
|
|
70
|
+
})
|
|
71
|
+
.describe('The handler for the action'),
|
|
72
|
+
})
|
|
73
|
+
.transform((def) => {
|
|
74
|
+
const checksum = [def.name, def.version, def.handler.toString()].filter(Boolean).join(':');
|
|
75
|
+
return {
|
|
76
|
+
...def,
|
|
77
|
+
checksum: generateChecksum(checksum),
|
|
78
|
+
};
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
export const defineAction = () => {
|
|
82
|
+
return (def) => {
|
|
83
|
+
return createActionDefinitionSchema().parse(def, {
|
|
84
|
+
reportInput: true,
|
|
85
|
+
});
|
|
86
|
+
};
|
|
87
|
+
};
|