@auriclabs/jobs-infra 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +25 -0
- package/README.md +168 -0
- package/dist/index.cjs +59 -0
- package/dist/index.d.cts +30 -0
- package/dist/index.d.cts.map +1 -0
- package/dist/index.d.mts +30 -0
- package/dist/index.d.mts.map +1 -0
- package/dist/index.mjs +59 -0
- package/dist/index.mjs.map +1 -0
- package/package.json +49 -0
- package/src/index.ts +2 -0
- package/src/job-resources.ts +88 -0
- package/src/job-table.ts +21 -0
- package/tsconfig.build.json +19 -0
- package/tsconfig.json +4 -0
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
|
|
2
|
+
> @auriclabs/jobs-infra@0.1.0 build /home/runner/work/packages/packages/packages/jobs-infra
|
|
3
|
+
> tsdown src/index.ts --format cjs,esm --dts --tsconfig tsconfig.build.json --no-hash
|
|
4
|
+
|
|
5
|
+
[33m[tsdown] Node.js v20.20.1 is deprecated. Support will be removed in the next minor release. Please upgrade to Node.js 22.18.0 or later.[39m
|
|
6
|
+
[34mℹ[39m tsdown [2mv0.21.4[22m powered by rolldown [2mv1.0.0-rc.9[22m
|
|
7
|
+
[34mℹ[39m entry: [34msrc/index.ts[39m
|
|
8
|
+
[34mℹ[39m tsconfig: [34mtsconfig.build.json[39m
|
|
9
|
+
[34mℹ[39m Build start
|
|
10
|
+
[34mℹ[39m [33m[CJS][39m [2mdist/[22m[1mindex.cjs[22m [2m1.67 kB[22m [2m│ gzip: 0.76 kB[22m
|
|
11
|
+
[34mℹ[39m [33m[CJS][39m 1 files, total: 1.67 kB
|
|
12
|
+
[34mℹ[39m [33m[CJS][39m [2mdist/[22mindex.d.cts.map [2m0.55 kB[22m [2m│ gzip: 0.27 kB[22m
|
|
13
|
+
[34mℹ[39m [33m[CJS][39m [2mdist/[22m[32m[1mindex.d.cts[22m[39m [2m0.88 kB[22m [2m│ gzip: 0.39 kB[22m
|
|
14
|
+
[34mℹ[39m [33m[CJS][39m 2 files, total: 1.43 kB
|
|
15
|
+
[33m[PLUGIN_TIMINGS] Warning:[0m Your build spent significant time in plugin `rolldown-plugin-dts:generate`. See https://rolldown.rs/options/checks#plugintimings for more details.
|
|
16
|
+
|
|
17
|
+
[32m✔[39m Build complete in [32m4865ms[39m
|
|
18
|
+
[34mℹ[39m [34m[ESM][39m [2mdist/[22m[1mindex.mjs[22m [2m1.58 kB[22m [2m│ gzip: 0.73 kB[22m
|
|
19
|
+
[34mℹ[39m [34m[ESM][39m [2mdist/[22mindex.mjs.map [2m3.73 kB[22m [2m│ gzip: 1.35 kB[22m
|
|
20
|
+
[34mℹ[39m [34m[ESM][39m [2mdist/[22mindex.d.mts.map [2m0.55 kB[22m [2m│ gzip: 0.27 kB[22m
|
|
21
|
+
[34mℹ[39m [34m[ESM][39m [2mdist/[22m[32m[1mindex.d.mts[22m[39m [2m0.88 kB[22m [2m│ gzip: 0.38 kB[22m
|
|
22
|
+
[34mℹ[39m [34m[ESM][39m 4 files, total: 6.74 kB
|
|
23
|
+
[33m[PLUGIN_TIMINGS] Warning:[0m Your build spent significant time in plugin `rolldown-plugin-dts:generate`. See https://rolldown.rs/options/checks#plugintimings for more details.
|
|
24
|
+
|
|
25
|
+
[32m✔[39m Build complete in [32m4867ms[39m
|
package/README.md
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# @auriclabs/jobs-infra
|
|
2
|
+
|
|
3
|
+
SST infrastructure helpers for provisioning DynamoDB job tables, SQS queues, and Lambda executor subscriptions.
|
|
4
|
+
|
|
5
|
+
## Setup
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pnpm add @auriclabs/jobs-infra
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
### Peer dependencies
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pnpm add sst @auriclabs/sst-types @auriclabs/sst-utils
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## API Reference
|
|
18
|
+
|
|
19
|
+
### `createJobTable(name)`
|
|
20
|
+
|
|
21
|
+
Creates a DynamoDB table with the standard job queue schema: primary index, number index, GSI1, and DynamoDB Streams enabled.
|
|
22
|
+
|
|
23
|
+
```typescript
|
|
24
|
+
import { createJobTable } from '@auriclabs/jobs-infra';
|
|
25
|
+
|
|
26
|
+
export const table = createJobTable('JobTable');
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Returns: `sst.aws.Dynamo`
|
|
30
|
+
|
|
31
|
+
Table schema:
|
|
32
|
+
| Field | Type | Index |
|
|
33
|
+
|-------|------|-------|
|
|
34
|
+
| `pk` | string | Primary hash key |
|
|
35
|
+
| `sk` | string | Primary range key |
|
|
36
|
+
| `numberIndexPk` | string | numberIndex hash key |
|
|
37
|
+
| `numberIndexSk` | number | numberIndex range key |
|
|
38
|
+
| `gsi1pk` | string | gsi1 hash key |
|
|
39
|
+
| `gsi1sk` | string | gsi1 range key |
|
|
40
|
+
|
|
41
|
+
### `registerJobResources(config)`
|
|
42
|
+
|
|
43
|
+
Registers job resources: subscribes the DynamoDB stream handler and sets up Lambda executor queue subscriptions.
|
|
44
|
+
|
|
45
|
+
```typescript
|
|
46
|
+
import { registerJobResources } from '@auriclabs/jobs-infra';
|
|
47
|
+
|
|
48
|
+
registerJobResources({
|
|
49
|
+
table: jobTable,
|
|
50
|
+
resources: [
|
|
51
|
+
{
|
|
52
|
+
id: 'lambda',
|
|
53
|
+
executor: 'lambda',
|
|
54
|
+
queue: lambdaJobQueue,
|
|
55
|
+
fns: [workerFn1, workerFn2],
|
|
56
|
+
},
|
|
57
|
+
{
|
|
58
|
+
id: 'worker',
|
|
59
|
+
queue: workerQueue,
|
|
60
|
+
// No executor — custom processing
|
|
61
|
+
},
|
|
62
|
+
],
|
|
63
|
+
handlerPaths: {
|
|
64
|
+
stream: 'services/job/handlers/job-table-stream.handler',
|
|
65
|
+
executor: 'services/job/handlers/lambda-executor.handler',
|
|
66
|
+
},
|
|
67
|
+
});
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Config:
|
|
71
|
+
|
|
72
|
+
```typescript
|
|
73
|
+
interface RegisterJobResourcesConfig {
|
|
74
|
+
table: sst.aws.Dynamo;
|
|
75
|
+
resources: JobResource[];
|
|
76
|
+
handlerPaths: {
|
|
77
|
+
stream: string; // DynamoDB stream handler path
|
|
78
|
+
executor: string; // Lambda executor handler path
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
### Resource Types
|
|
84
|
+
|
|
85
|
+
```typescript
|
|
86
|
+
// Lambda-executed jobs: SQS → Lambda executor → target Lambda
|
|
87
|
+
interface LambdaJobResource {
|
|
88
|
+
id: string;
|
|
89
|
+
executor: 'lambda';
|
|
90
|
+
queue: sst.aws.Queue;
|
|
91
|
+
fns: FunctionWithName[]; // From @auriclabs/sst-utils
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Worker jobs: SQS → custom processing (no executor subscription)
|
|
95
|
+
interface WorkerJobResource {
|
|
96
|
+
id: string;
|
|
97
|
+
executor?: never;
|
|
98
|
+
queue: sst.aws.Queue;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
type JobResource = LambdaJobResource | WorkerJobResource;
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### What `registerJobResources` sets up
|
|
105
|
+
|
|
106
|
+
1. **DynamoDB stream subscription** on the job table
|
|
107
|
+
- Filters for `job-attempt` entity records only (ElectroDB `__edb_e__` field)
|
|
108
|
+
- Links the table and all queues
|
|
109
|
+
- Sets `QUEUE_URL_LIST` env var (maps queue IDs to URLs)
|
|
110
|
+
|
|
111
|
+
2. **Lambda executor subscriptions** for each `LambdaJobResource`
|
|
112
|
+
- Subscribes queue to executor handler
|
|
113
|
+
- Links table, queue, and target Lambda functions
|
|
114
|
+
- Sets `LAMBDA_FUNCTION_LIST` env var (maps function names to ARNs)
|
|
115
|
+
- Batch config: 10 items, 3 second window, partial responses enabled
|
|
116
|
+
|
|
117
|
+
## Full Example
|
|
118
|
+
|
|
119
|
+
```typescript
|
|
120
|
+
// infra/job.ts
|
|
121
|
+
import { createJobTable, registerJobResources } from '@auriclabs/jobs-infra';
|
|
122
|
+
|
|
123
|
+
export const table = createJobTable('JobTable');
|
|
124
|
+
|
|
125
|
+
export const lambdaJobDeadLetterQueue = new sst.aws.Queue('LambdaJobDeadLetterQueue');
|
|
126
|
+
export const lambdaJobQueue = new sst.aws.Queue('LambdaJobQueue', {
|
|
127
|
+
delay: '0 seconds',
|
|
128
|
+
visibilityTimeout: '10 minutes',
|
|
129
|
+
dlq: {
|
|
130
|
+
retry: 3,
|
|
131
|
+
queue: lambdaJobDeadLetterQueue.arn,
|
|
132
|
+
},
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
registerJobResources({
|
|
136
|
+
table,
|
|
137
|
+
resources: [
|
|
138
|
+
{
|
|
139
|
+
id: 'lambda',
|
|
140
|
+
executor: 'lambda',
|
|
141
|
+
queue: lambdaJobQueue,
|
|
142
|
+
fns: [crawlerWorker, indexerWorker],
|
|
143
|
+
},
|
|
144
|
+
],
|
|
145
|
+
handlerPaths: {
|
|
146
|
+
stream: 'services/job/handlers/job-table-stream.handler',
|
|
147
|
+
executor: 'services/job/handlers/lambda-executor.handler',
|
|
148
|
+
},
|
|
149
|
+
});
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
The handlers should use factories from `@auriclabs/jobs`:
|
|
153
|
+
|
|
154
|
+
```typescript
|
|
155
|
+
// services/job/handlers/job-table-stream.ts
|
|
156
|
+
import { createJobTableStreamHandler, initJobs } from '@auriclabs/jobs';
|
|
157
|
+
import { Resource } from 'sst';
|
|
158
|
+
|
|
159
|
+
initJobs({ tableName: Resource.JobTable.name });
|
|
160
|
+
export const handler = createJobTableStreamHandler();
|
|
161
|
+
|
|
162
|
+
// services/job/handlers/lambda-executor.ts
|
|
163
|
+
import { createLambdaExecutorHandler, initJobs } from '@auriclabs/jobs';
|
|
164
|
+
import { Resource } from 'sst';
|
|
165
|
+
|
|
166
|
+
initJobs({ tableName: Resource.JobTable.name });
|
|
167
|
+
export const handler = createLambdaExecutorHandler();
|
|
168
|
+
```
|
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
//#region src/job-table.ts
|
|
3
|
+
function createJobTable(name) {
|
|
4
|
+
return new sst.aws.Dynamo(name, {
|
|
5
|
+
fields: {
|
|
6
|
+
pk: "string",
|
|
7
|
+
sk: "string",
|
|
8
|
+
numberIndexPk: "string",
|
|
9
|
+
numberIndexSk: "number",
|
|
10
|
+
gsi1pk: "string",
|
|
11
|
+
gsi1sk: "string"
|
|
12
|
+
},
|
|
13
|
+
primaryIndex: {
|
|
14
|
+
hashKey: "pk",
|
|
15
|
+
rangeKey: "sk"
|
|
16
|
+
},
|
|
17
|
+
globalIndexes: {
|
|
18
|
+
numberIndex: {
|
|
19
|
+
hashKey: "numberIndexPk",
|
|
20
|
+
rangeKey: "numberIndexSk"
|
|
21
|
+
},
|
|
22
|
+
gsi1: {
|
|
23
|
+
hashKey: "gsi1pk",
|
|
24
|
+
rangeKey: "gsi1sk"
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
stream: "new-and-old-images"
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
//#endregion
|
|
31
|
+
//#region src/job-resources.ts
|
|
32
|
+
function registerJobResources(config) {
|
|
33
|
+
const { table, resources, handlerPaths } = config;
|
|
34
|
+
const QUEUE_URL_LIST = $jsonStringify(resources.map(({ id, queue }) => [id, queue.url]));
|
|
35
|
+
table.subscribe("JobTableStream", {
|
|
36
|
+
handler: handlerPaths.stream,
|
|
37
|
+
link: [table, ...resources.map(({ queue }) => queue)],
|
|
38
|
+
environment: { QUEUE_URL_LIST }
|
|
39
|
+
}, { filters: [{ dynamodb: { NewImage: { __edb_e__: { S: ["job-attempt"] } } } }, { dynamodb: { OldImage: { __edb_e__: { S: ["job-attempt"] } } } }] });
|
|
40
|
+
resources.forEach((resource) => {
|
|
41
|
+
if (!("executor" in resource)) return;
|
|
42
|
+
if (resource.executor === "lambda") resource.queue.subscribe({
|
|
43
|
+
handler: handlerPaths.executor,
|
|
44
|
+
link: [
|
|
45
|
+
table,
|
|
46
|
+
resource.queue,
|
|
47
|
+
...resource.fns
|
|
48
|
+
],
|
|
49
|
+
environment: { LAMBDA_FUNCTION_LIST: $jsonStringify(resource.fns.map((f) => [f.name, f.arn])) }
|
|
50
|
+
}, { batch: {
|
|
51
|
+
size: 10,
|
|
52
|
+
window: "3 seconds",
|
|
53
|
+
partialResponses: true
|
|
54
|
+
} });
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
//#endregion
|
|
58
|
+
exports.createJobTable = createJobTable;
|
|
59
|
+
exports.registerJobResources = registerJobResources;
|
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { FunctionWithName } from "@auriclabs/sst-utils";
|
|
2
|
+
|
|
3
|
+
//#region src/job-table.d.ts
|
|
4
|
+
declare function createJobTable(name: string): sst.aws.Dynamo;
|
|
5
|
+
//#endregion
|
|
6
|
+
//#region src/job-resources.d.ts
|
|
7
|
+
interface LambdaJobResource {
|
|
8
|
+
id: string;
|
|
9
|
+
executor: 'lambda';
|
|
10
|
+
queue: sst.aws.Queue;
|
|
11
|
+
fns: FunctionWithName[];
|
|
12
|
+
}
|
|
13
|
+
interface WorkerJobResource {
|
|
14
|
+
id: string;
|
|
15
|
+
executor?: never;
|
|
16
|
+
queue: sst.aws.Queue;
|
|
17
|
+
}
|
|
18
|
+
type JobResource = LambdaJobResource | WorkerJobResource;
|
|
19
|
+
interface RegisterJobResourcesConfig {
|
|
20
|
+
table: sst.aws.Dynamo;
|
|
21
|
+
resources: JobResource[];
|
|
22
|
+
handlerPaths: {
|
|
23
|
+
stream: string;
|
|
24
|
+
executor: string;
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
declare function registerJobResources(config: RegisterJobResourcesConfig): void;
|
|
28
|
+
//#endregion
|
|
29
|
+
export { JobResource, LambdaJobResource, RegisterJobResourcesConfig, WorkerJobResource, createJobTable, registerJobResources };
|
|
30
|
+
//# sourceMappingURL=index.d.cts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.cts","names":[],"sources":["../src/job-table.ts","../src/job-resources.ts"],"mappings":";;;iBAAgB,cAAA,CAAe,IAAA,WAAY,GAAA,CAAA,GAAA,CAAA,MAAA;;;UCE1B,iBAAA;EACf,EAAA;EACA,QAAA;EACA,KAAA,EAAO,GAAA,CAAI,GAAA,CAAI,KAAA;EACf,GAAA,EAAK,gBAAA;AAAA;AAAA,UAGU,iBAAA;EACf,EAAA;EACA,QAAA;EACA,KAAA,EAAO,GAAA,CAAI,GAAA,CAAI,KAAA;AAAA;AAAA,KAGL,WAAA,GAAc,iBAAA,GAAoB,iBAAA;AAAA,UAE7B,0BAAA;EACf,KAAA,EAAO,GAAA,CAAI,GAAA,CAAI,MAAA;EACf,SAAA,EAAW,WAAA;EACX,YAAA;IACE,MAAA;IACA,QAAA;EAAA;AAAA;AAAA,iBAIY,oBAAA,CAAqB,MAAA,EAAQ,0BAAA"}
|
package/dist/index.d.mts
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { FunctionWithName } from "@auriclabs/sst-utils";
|
|
2
|
+
|
|
3
|
+
//#region src/job-table.d.ts
|
|
4
|
+
declare function createJobTable(name: string): sst.aws.Dynamo;
|
|
5
|
+
//#endregion
|
|
6
|
+
//#region src/job-resources.d.ts
|
|
7
|
+
interface LambdaJobResource {
|
|
8
|
+
id: string;
|
|
9
|
+
executor: 'lambda';
|
|
10
|
+
queue: sst.aws.Queue;
|
|
11
|
+
fns: FunctionWithName[];
|
|
12
|
+
}
|
|
13
|
+
interface WorkerJobResource {
|
|
14
|
+
id: string;
|
|
15
|
+
executor?: never;
|
|
16
|
+
queue: sst.aws.Queue;
|
|
17
|
+
}
|
|
18
|
+
type JobResource = LambdaJobResource | WorkerJobResource;
|
|
19
|
+
interface RegisterJobResourcesConfig {
|
|
20
|
+
table: sst.aws.Dynamo;
|
|
21
|
+
resources: JobResource[];
|
|
22
|
+
handlerPaths: {
|
|
23
|
+
stream: string;
|
|
24
|
+
executor: string;
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
declare function registerJobResources(config: RegisterJobResourcesConfig): void;
|
|
28
|
+
//#endregion
|
|
29
|
+
export { JobResource, LambdaJobResource, RegisterJobResourcesConfig, WorkerJobResource, createJobTable, registerJobResources };
|
|
30
|
+
//# sourceMappingURL=index.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.mts","names":[],"sources":["../src/job-table.ts","../src/job-resources.ts"],"mappings":";;;iBAAgB,cAAA,CAAe,IAAA,WAAY,GAAA,CAAA,GAAA,CAAA,MAAA;;;UCE1B,iBAAA;EACf,EAAA;EACA,QAAA;EACA,KAAA,EAAO,GAAA,CAAI,GAAA,CAAI,KAAA;EACf,GAAA,EAAK,gBAAA;AAAA;AAAA,UAGU,iBAAA;EACf,EAAA;EACA,QAAA;EACA,KAAA,EAAO,GAAA,CAAI,GAAA,CAAI,KAAA;AAAA;AAAA,KAGL,WAAA,GAAc,iBAAA,GAAoB,iBAAA;AAAA,UAE7B,0BAAA;EACf,KAAA,EAAO,GAAA,CAAI,GAAA,CAAI,MAAA;EACf,SAAA,EAAW,WAAA;EACX,YAAA;IACE,MAAA;IACA,QAAA;EAAA;AAAA;AAAA,iBAIY,oBAAA,CAAqB,MAAA,EAAQ,0BAAA"}
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
//#region src/job-table.ts
|
|
2
|
+
function createJobTable(name) {
|
|
3
|
+
return new sst.aws.Dynamo(name, {
|
|
4
|
+
fields: {
|
|
5
|
+
pk: "string",
|
|
6
|
+
sk: "string",
|
|
7
|
+
numberIndexPk: "string",
|
|
8
|
+
numberIndexSk: "number",
|
|
9
|
+
gsi1pk: "string",
|
|
10
|
+
gsi1sk: "string"
|
|
11
|
+
},
|
|
12
|
+
primaryIndex: {
|
|
13
|
+
hashKey: "pk",
|
|
14
|
+
rangeKey: "sk"
|
|
15
|
+
},
|
|
16
|
+
globalIndexes: {
|
|
17
|
+
numberIndex: {
|
|
18
|
+
hashKey: "numberIndexPk",
|
|
19
|
+
rangeKey: "numberIndexSk"
|
|
20
|
+
},
|
|
21
|
+
gsi1: {
|
|
22
|
+
hashKey: "gsi1pk",
|
|
23
|
+
rangeKey: "gsi1sk"
|
|
24
|
+
}
|
|
25
|
+
},
|
|
26
|
+
stream: "new-and-old-images"
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
//#endregion
|
|
30
|
+
//#region src/job-resources.ts
|
|
31
|
+
function registerJobResources(config) {
|
|
32
|
+
const { table, resources, handlerPaths } = config;
|
|
33
|
+
const QUEUE_URL_LIST = $jsonStringify(resources.map(({ id, queue }) => [id, queue.url]));
|
|
34
|
+
table.subscribe("JobTableStream", {
|
|
35
|
+
handler: handlerPaths.stream,
|
|
36
|
+
link: [table, ...resources.map(({ queue }) => queue)],
|
|
37
|
+
environment: { QUEUE_URL_LIST }
|
|
38
|
+
}, { filters: [{ dynamodb: { NewImage: { __edb_e__: { S: ["job-attempt"] } } } }, { dynamodb: { OldImage: { __edb_e__: { S: ["job-attempt"] } } } }] });
|
|
39
|
+
resources.forEach((resource) => {
|
|
40
|
+
if (!("executor" in resource)) return;
|
|
41
|
+
if (resource.executor === "lambda") resource.queue.subscribe({
|
|
42
|
+
handler: handlerPaths.executor,
|
|
43
|
+
link: [
|
|
44
|
+
table,
|
|
45
|
+
resource.queue,
|
|
46
|
+
...resource.fns
|
|
47
|
+
],
|
|
48
|
+
environment: { LAMBDA_FUNCTION_LIST: $jsonStringify(resource.fns.map((f) => [f.name, f.arn])) }
|
|
49
|
+
}, { batch: {
|
|
50
|
+
size: 10,
|
|
51
|
+
window: "3 seconds",
|
|
52
|
+
partialResponses: true
|
|
53
|
+
} });
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
//#endregion
|
|
57
|
+
export { createJobTable, registerJobResources };
|
|
58
|
+
|
|
59
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.mjs","names":[],"sources":["../src/job-table.ts","../src/job-resources.ts"],"sourcesContent":["export function createJobTable(name: string) {\n return new sst.aws.Dynamo(name, {\n fields: {\n pk: 'string',\n sk: 'string',\n numberIndexPk: 'string',\n numberIndexSk: 'number',\n gsi1pk: 'string',\n gsi1sk: 'string',\n },\n primaryIndex: {\n hashKey: 'pk',\n rangeKey: 'sk',\n },\n globalIndexes: {\n numberIndex: { hashKey: 'numberIndexPk', rangeKey: 'numberIndexSk' },\n gsi1: { hashKey: 'gsi1pk', rangeKey: 'gsi1sk' },\n },\n stream: 'new-and-old-images',\n });\n}\n","import { FunctionWithName } from '@auriclabs/sst-utils';\n\nexport interface LambdaJobResource {\n id: string;\n executor: 'lambda';\n queue: sst.aws.Queue;\n fns: FunctionWithName[];\n}\n\nexport interface WorkerJobResource {\n id: string;\n executor?: never;\n queue: sst.aws.Queue;\n}\n\nexport type JobResource = LambdaJobResource | WorkerJobResource;\n\nexport interface RegisterJobResourcesConfig {\n table: sst.aws.Dynamo;\n resources: JobResource[];\n handlerPaths: {\n stream: string;\n executor: string;\n };\n}\n\nexport function registerJobResources(config: RegisterJobResourcesConfig) {\n const { table, resources, handlerPaths } = config;\n const QUEUE_URL_LIST = $jsonStringify(resources.map(({ id, queue }) => [id, queue.url]));\n\n table.subscribe(\n 'JobTableStream',\n {\n handler: handlerPaths.stream,\n link: [table, ...resources.map(({ queue }) => queue)],\n environment: {\n QUEUE_URL_LIST,\n },\n },\n {\n filters: [\n {\n dynamodb: {\n NewImage: {\n __edb_e__: {\n S: ['job-attempt'],\n },\n },\n },\n },\n {\n dynamodb: {\n OldImage: {\n __edb_e__: {\n S: ['job-attempt'],\n },\n },\n },\n },\n ],\n },\n );\n\n resources.forEach((resource) => {\n if (!('executor' in resource)) {\n return;\n }\n\n if (resource.executor === 'lambda') {\n resource.queue.subscribe(\n {\n handler: handlerPaths.executor,\n link: [table, resource.queue, ...resource.fns],\n environment: {\n LAMBDA_FUNCTION_LIST: $jsonStringify(resource.fns.map((f) => [f.name, f.arn])),\n },\n },\n {\n batch: {\n size: 10,\n window: '3 seconds',\n partialResponses: true,\n },\n },\n );\n }\n });\n}\n"],"mappings":";AAAA,SAAgB,eAAe,MAAc;AAC3C,QAAO,IAAI,IAAI,IAAI,OAAO,MAAM;EAC9B,QAAQ;GACN,IAAI;GACJ,IAAI;GACJ,eAAe;GACf,eAAe;GACf,QAAQ;GACR,QAAQ;GACT;EACD,cAAc;GACZ,SAAS;GACT,UAAU;GACX;EACD,eAAe;GACb,aAAa;IAAE,SAAS;IAAiB,UAAU;IAAiB;GACpE,MAAM;IAAE,SAAS;IAAU,UAAU;IAAU;GAChD;EACD,QAAQ;EACT,CAAC;;;;ACOJ,SAAgB,qBAAqB,QAAoC;CACvE,MAAM,EAAE,OAAO,WAAW,iBAAiB;CAC3C,MAAM,iBAAiB,eAAe,UAAU,KAAK,EAAE,IAAI,YAAY,CAAC,IAAI,MAAM,IAAI,CAAC,CAAC;AAExF,OAAM,UACJ,kBACA;EACE,SAAS,aAAa;EACtB,MAAM,CAAC,OAAO,GAAG,UAAU,KAAK,EAAE,YAAY,MAAM,CAAC;EACrD,aAAa,EACX,gBACD;EACF,EACD,EACE,SAAS,CACP,EACE,UAAU,EACR,UAAU,EACR,WAAW,EACT,GAAG,CAAC,cAAc,EACnB,EACF,EACF,EACF,EACD,EACE,UAAU,EACR,UAAU,EACR,WAAW,EACT,GAAG,CAAC,cAAc,EACnB,EACF,EACF,EACF,CACF,EACF,CACF;AAED,WAAU,SAAS,aAAa;AAC9B,MAAI,EAAE,cAAc,UAClB;AAGF,MAAI,SAAS,aAAa,SACxB,UAAS,MAAM,UACb;GACE,SAAS,aAAa;GACtB,MAAM;IAAC;IAAO,SAAS;IAAO,GAAG,SAAS;IAAI;GAC9C,aAAa,EACX,sBAAsB,eAAe,SAAS,IAAI,KAAK,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,EAC/E;GACF,EACD,EACE,OAAO;GACL,MAAM;GACN,QAAQ;GACR,kBAAkB;GACnB,EACF,CACF;GAEH"}
|
package/package.json
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@auriclabs/jobs-infra",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "SST infrastructure helpers for job queue tables and resources",
|
|
5
|
+
"prettier": "@auriclabs/prettier-config",
|
|
6
|
+
"main": "dist/index.cjs",
|
|
7
|
+
"module": "dist/index.mjs",
|
|
8
|
+
"types": "dist/index.d.mts",
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"types": "./dist/index.d.mts",
|
|
12
|
+
"import": "./dist/index.mjs",
|
|
13
|
+
"require": "./dist/index.cjs"
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
"keywords": [],
|
|
17
|
+
"author": "",
|
|
18
|
+
"type": "module",
|
|
19
|
+
"license": "ISC",
|
|
20
|
+
"dependencies": {},
|
|
21
|
+
"devDependencies": {
|
|
22
|
+
"sst": "^4.3.7",
|
|
23
|
+
"@auriclabs/sst-types": "0.1.0",
|
|
24
|
+
"@auriclabs/sst-utils": "1.0.0"
|
|
25
|
+
},
|
|
26
|
+
"peerDependencies": {
|
|
27
|
+
"@auriclabs/sst-types": "^0.1.0",
|
|
28
|
+
"@auriclabs/sst-utils": "^1.0.0",
|
|
29
|
+
"sst": "^4.3.7"
|
|
30
|
+
},
|
|
31
|
+
"publishConfig": {
|
|
32
|
+
"registry": "https://registry.npmjs.org/"
|
|
33
|
+
},
|
|
34
|
+
"repository": {
|
|
35
|
+
"type": "git",
|
|
36
|
+
"url": "https://github.com/auriclabs/packages.git",
|
|
37
|
+
"directory": "packages/jobs-infra"
|
|
38
|
+
},
|
|
39
|
+
"scripts": {
|
|
40
|
+
"build": "tsdown src/index.ts --format cjs,esm --dts --tsconfig tsconfig.build.json --no-hash",
|
|
41
|
+
"dev": "concurrently \"pnpm build --watch\" \"pnpm:y:watch\"",
|
|
42
|
+
"y:watch": "chokidar dist --initial --silent -c \"yalc publish --push\"",
|
|
43
|
+
"lint": "eslint .",
|
|
44
|
+
"lint:fix": "eslint . --fix",
|
|
45
|
+
"typecheck": "ts-config-typecheck",
|
|
46
|
+
"test": "vitest run --passWithNoTests",
|
|
47
|
+
"test:watch": "vitest"
|
|
48
|
+
}
|
|
49
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { FunctionWithName } from '@auriclabs/sst-utils';
|
|
2
|
+
|
|
3
|
+
export interface LambdaJobResource {
|
|
4
|
+
id: string;
|
|
5
|
+
executor: 'lambda';
|
|
6
|
+
queue: sst.aws.Queue;
|
|
7
|
+
fns: FunctionWithName[];
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface WorkerJobResource {
|
|
11
|
+
id: string;
|
|
12
|
+
executor?: never;
|
|
13
|
+
queue: sst.aws.Queue;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export type JobResource = LambdaJobResource | WorkerJobResource;
|
|
17
|
+
|
|
18
|
+
export interface RegisterJobResourcesConfig {
|
|
19
|
+
table: sst.aws.Dynamo;
|
|
20
|
+
resources: JobResource[];
|
|
21
|
+
handlerPaths: {
|
|
22
|
+
stream: string;
|
|
23
|
+
executor: string;
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function registerJobResources(config: RegisterJobResourcesConfig) {
|
|
28
|
+
const { table, resources, handlerPaths } = config;
|
|
29
|
+
const QUEUE_URL_LIST = $jsonStringify(resources.map(({ id, queue }) => [id, queue.url]));
|
|
30
|
+
|
|
31
|
+
table.subscribe(
|
|
32
|
+
'JobTableStream',
|
|
33
|
+
{
|
|
34
|
+
handler: handlerPaths.stream,
|
|
35
|
+
link: [table, ...resources.map(({ queue }) => queue)],
|
|
36
|
+
environment: {
|
|
37
|
+
QUEUE_URL_LIST,
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
filters: [
|
|
42
|
+
{
|
|
43
|
+
dynamodb: {
|
|
44
|
+
NewImage: {
|
|
45
|
+
__edb_e__: {
|
|
46
|
+
S: ['job-attempt'],
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
dynamodb: {
|
|
53
|
+
OldImage: {
|
|
54
|
+
__edb_e__: {
|
|
55
|
+
S: ['job-attempt'],
|
|
56
|
+
},
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
],
|
|
61
|
+
},
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
resources.forEach((resource) => {
|
|
65
|
+
if (!('executor' in resource)) {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (resource.executor === 'lambda') {
|
|
70
|
+
resource.queue.subscribe(
|
|
71
|
+
{
|
|
72
|
+
handler: handlerPaths.executor,
|
|
73
|
+
link: [table, resource.queue, ...resource.fns],
|
|
74
|
+
environment: {
|
|
75
|
+
LAMBDA_FUNCTION_LIST: $jsonStringify(resource.fns.map((f) => [f.name, f.arn])),
|
|
76
|
+
},
|
|
77
|
+
},
|
|
78
|
+
{
|
|
79
|
+
batch: {
|
|
80
|
+
size: 10,
|
|
81
|
+
window: '3 seconds',
|
|
82
|
+
partialResponses: true,
|
|
83
|
+
},
|
|
84
|
+
},
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
}
|
package/src/job-table.ts
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export function createJobTable(name: string) {
|
|
2
|
+
return new sst.aws.Dynamo(name, {
|
|
3
|
+
fields: {
|
|
4
|
+
pk: 'string',
|
|
5
|
+
sk: 'string',
|
|
6
|
+
numberIndexPk: 'string',
|
|
7
|
+
numberIndexSk: 'number',
|
|
8
|
+
gsi1pk: 'string',
|
|
9
|
+
gsi1sk: 'string',
|
|
10
|
+
},
|
|
11
|
+
primaryIndex: {
|
|
12
|
+
hashKey: 'pk',
|
|
13
|
+
rangeKey: 'sk',
|
|
14
|
+
},
|
|
15
|
+
globalIndexes: {
|
|
16
|
+
numberIndex: { hashKey: 'numberIndexPk', rangeKey: 'numberIndexSk' },
|
|
17
|
+
gsi1: { hashKey: 'gsi1pk', rangeKey: 'gsi1sk' },
|
|
18
|
+
},
|
|
19
|
+
stream: 'new-and-old-images',
|
|
20
|
+
});
|
|
21
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
{
|
|
2
|
+
"extends": "@auriclabs/ts-config/node-package",
|
|
3
|
+
"compilerOptions": {
|
|
4
|
+
"rootDir": ".",
|
|
5
|
+
"outDir": "dist",
|
|
6
|
+
"composite": true,
|
|
7
|
+
"types": ["@auriclabs/sst-types"]
|
|
8
|
+
},
|
|
9
|
+
"include": ["src/**/*"],
|
|
10
|
+
"exclude": [
|
|
11
|
+
"dist",
|
|
12
|
+
"node_modules",
|
|
13
|
+
"**/*.test.ts",
|
|
14
|
+
"**/*.spec.ts",
|
|
15
|
+
"**/__tests__/**",
|
|
16
|
+
"**/__mocks__/**",
|
|
17
|
+
"vitest.config.ts"
|
|
18
|
+
]
|
|
19
|
+
}
|
package/tsconfig.json
ADDED