@venturekit/data 0.0.32 → 0.0.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/files/index.d.ts +9 -0
- package/dist/files/index.d.ts.map +1 -0
- package/dist/files/index.js +8 -0
- package/dist/files/index.js.map +1 -0
- package/dist/files/postgres.d.ts +150 -0
- package/dist/files/postgres.d.ts.map +1 -0
- package/dist/files/postgres.js +194 -0
- package/dist/files/postgres.js.map +1 -0
- package/dist/idempotency/index.d.ts +9 -0
- package/dist/idempotency/index.d.ts.map +1 -0
- package/dist/idempotency/index.js +8 -0
- package/dist/idempotency/index.js.map +1 -0
- package/dist/idempotency/postgres.d.ts +107 -0
- package/dist/idempotency/postgres.d.ts.map +1 -0
- package/dist/idempotency/postgres.js +145 -0
- package/dist/idempotency/postgres.js.map +1 -0
- package/dist/internal/identifier.d.ts +16 -0
- package/dist/internal/identifier.d.ts.map +1 -0
- package/dist/internal/identifier.js +23 -0
- package/dist/internal/identifier.js.map +1 -0
- package/dist/jobs/index.d.ts +9 -0
- package/dist/jobs/index.d.ts.map +1 -0
- package/dist/jobs/index.js +8 -0
- package/dist/jobs/index.js.map +1 -0
- package/dist/jobs/postgres.d.ts +197 -0
- package/dist/jobs/postgres.d.ts.map +1 -0
- package/dist/jobs/postgres.js +270 -0
- package/dist/jobs/postgres.js.map +1 -0
- package/dist/outbox/index.d.ts +9 -0
- package/dist/outbox/index.d.ts.map +1 -0
- package/dist/outbox/index.js +8 -0
- package/dist/outbox/index.js.map +1 -0
- package/dist/outbox/postgres.d.ts +124 -0
- package/dist/outbox/postgres.d.ts.map +1 -0
- package/dist/outbox/postgres.js +177 -0
- package/dist/outbox/postgres.js.map +1 -0
- package/dist/query/index.d.ts.map +1 -1
- package/dist/query/index.js.map +1 -1
- package/dist/query/secret.d.ts.map +1 -1
- package/dist/query/secret.js +1 -1
- package/dist/query/secret.js.map +1 -1
- package/package.json +18 -2
- package/src/sql/{vk_data_001_tenancy_foundation.sql → 0000_vk_data_foundation.sql} +305 -278
- package/src/sql/vk_data_001_idempotency.sql +48 -0
- package/src/sql/vk_data_002_outbox.sql +99 -0
- package/src/sql/vk_data_003_jobs.sql +114 -0
- package/src/sql/vk_data_004_file_object.sql +112 -0
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @venturekit/data — Postgres idempotency store.
|
|
3
|
+
*
|
|
4
|
+
* `@venturekit/runtime`'s `idempotencyMiddleware` takes a pluggable
|
|
5
|
+
* `IdempotencyStore`; the package ships an in-memory one (per-process, so wrong
|
|
6
|
+
* for Lambda) and a DynamoDB one, and its own docblock has always advertised
|
|
7
|
+
* "DynamoDB, Postgres, in-memory, etc." without the middle one existing. A
|
|
8
|
+
* project whose only datastore is Postgres therefore had to add a DynamoDB
|
|
9
|
+
* table for four columns, or write this adapter itself.
|
|
10
|
+
*
|
|
11
|
+
* This is that adapter. It is deliberately typed structurally rather than
|
|
12
|
+
* against `@venturekit/runtime`'s interface: `runtime` depends on `data`, so
|
|
13
|
+
* importing the type here would invert the dependency. The shape is asserted by
|
|
14
|
+
* the tests instead, and `createPostgresIdempotencyStore()` is assignable to
|
|
15
|
+
* `IdempotencyStore` wherever the middleware wants one.
|
|
16
|
+
*
|
|
17
|
+
* ### Schema
|
|
18
|
+
*
|
|
19
|
+
* `vk_idempotency_record`, from this package's `vk_data_001_idempotency.sql`.
|
|
20
|
+
*
|
|
21
|
+
* ### One thing Postgres does not do for you
|
|
22
|
+
*
|
|
23
|
+
* DynamoDB expires rows itself via its TTL attribute. Postgres has no
|
|
24
|
+
* equivalent, so expired rows accumulate until something deletes them:
|
|
25
|
+
* {@link purgeExpiredIdempotencyRecords} is that something, and it wants a
|
|
26
|
+
* schedule. Reads are already correct without it — `get()` filters on
|
|
27
|
+
* `expires_at` — so a project that forgets pays in disk, not in behaviour.
|
|
28
|
+
*/
|
|
29
|
+
import { query } from '../query/index.js';
|
|
30
|
+
import { assertSqlIdentifier } from '../internal/identifier.js';
|
|
31
|
+
/** Default table name. Overridable for projects that prefix or schema-qualify. */
|
|
32
|
+
export const DEFAULT_IDEMPOTENCY_TABLE = 'vk_idempotency_record';
|
|
33
|
+
/** Structurally compatible with `runtime`'s `IdempotencyConflictError`. */
|
|
34
|
+
class ConflictError extends Error {
|
|
35
|
+
code = 'IDEMPOTENCY_CONFLICT';
|
|
36
|
+
constructor(key) {
|
|
37
|
+
super(`A request with idempotency key '${key}' is already in progress`);
|
|
38
|
+
this.name = 'IdempotencyConflictError';
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
const toMillis = (v) => v instanceof Date ? v.getTime() : new Date(v).getTime();
|
|
42
|
+
/**
|
|
43
|
+
* Create a Postgres-backed idempotency store.
|
|
44
|
+
*
|
|
45
|
+
* @example
|
|
46
|
+
* ```ts
|
|
47
|
+
* import { idempotencyMiddleware } from '@venturekit/runtime';
|
|
48
|
+
* import { createPostgresIdempotencyStore } from '@venturekit/data/idempotency';
|
|
49
|
+
*
|
|
50
|
+
* const idempotency = idempotencyMiddleware({
|
|
51
|
+
* store: createPostgresIdempotencyStore({ scope: () => getCurrentTenantId() }),
|
|
52
|
+
* });
|
|
53
|
+
* ```
|
|
54
|
+
*/
|
|
55
|
+
export function createPostgresIdempotencyStore(options = {}) {
|
|
56
|
+
const table = options.tableName ?? DEFAULT_IDEMPOTENCY_TABLE;
|
|
57
|
+
assertSqlIdentifier(table, 'PostgresIdempotencyStoreOptions.tableName');
|
|
58
|
+
const run = options.querier ?? query;
|
|
59
|
+
/* `<scope>:<key>`. The separator is a colon rather than nothing so that
|
|
60
|
+
scope `a` + key `bc` cannot collide with scope `ab` + key `c`. */
|
|
61
|
+
const scoped = (key) => {
|
|
62
|
+
const s = options.scope?.();
|
|
63
|
+
return s ? `${s}:${key}` : key;
|
|
64
|
+
};
|
|
65
|
+
return {
|
|
66
|
+
async get(key) {
|
|
67
|
+
/* Expiry is filtered in SQL, not in JS: an expired row must read as
|
|
68
|
+
absent even if a purge has not run, and `now()` on the server avoids
|
|
69
|
+
trusting a Lambda's clock. */
|
|
70
|
+
const rows = await run(`SELECT key, response, status, expires_at
|
|
71
|
+
FROM ${table}
|
|
72
|
+
WHERE key = $1 AND expires_at > now()`, [scoped(key)]);
|
|
73
|
+
const row = rows[0];
|
|
74
|
+
if (!row)
|
|
75
|
+
return null;
|
|
76
|
+
return {
|
|
77
|
+
/* the caller's key, not the scoped one it is stored under */
|
|
78
|
+
key,
|
|
79
|
+
response: row.response,
|
|
80
|
+
status: row.status,
|
|
81
|
+
expiresAt: toMillis(row.expires_at),
|
|
82
|
+
};
|
|
83
|
+
},
|
|
84
|
+
async save(record) {
|
|
85
|
+
/* One statement, so the check and the write cannot be raced.
|
|
86
|
+
`ON CONFLICT ... DO UPDATE ... WHERE` takes the row only when the
|
|
87
|
+
existing one is finished or stale; a live `pending` row matches
|
|
88
|
+
neither, updates nothing, and returns no rows — which is the conflict.
|
|
89
|
+
Doing this as SELECT-then-INSERT would let two concurrent requests
|
|
90
|
+
both see "absent" and both proceed. */
|
|
91
|
+
const rows = await run(`INSERT INTO ${table} (key, response, status, expires_at)
|
|
92
|
+
VALUES ($1, $2, $3, to_timestamp($4::double precision / 1000))
|
|
93
|
+
ON CONFLICT (key) DO UPDATE
|
|
94
|
+
SET response = EXCLUDED.response,
|
|
95
|
+
status = EXCLUDED.status,
|
|
96
|
+
expires_at = EXCLUDED.expires_at,
|
|
97
|
+
updated_at = now()
|
|
98
|
+
WHERE ${table}.status <> 'pending'
|
|
99
|
+
OR ${table}.expires_at <= now()
|
|
100
|
+
RETURNING key`, [scoped(record.key), record.response, record.status, record.expiresAt]);
|
|
101
|
+
if (rows.length === 0)
|
|
102
|
+
throw new ConflictError(record.key);
|
|
103
|
+
},
|
|
104
|
+
async update(key, updates) {
|
|
105
|
+
/* Built from whichever fields were supplied — the middleware sends
|
|
106
|
+
`{ status, response }` on success and nothing else. */
|
|
107
|
+
const sets = [];
|
|
108
|
+
const params = [scoped(key)];
|
|
109
|
+
if (updates.response !== undefined) {
|
|
110
|
+
params.push(updates.response);
|
|
111
|
+
sets.push(`response = $${params.length}`);
|
|
112
|
+
}
|
|
113
|
+
if (updates.status !== undefined) {
|
|
114
|
+
params.push(updates.status);
|
|
115
|
+
sets.push(`status = $${params.length}`);
|
|
116
|
+
}
|
|
117
|
+
if (updates.expiresAt !== undefined) {
|
|
118
|
+
params.push(updates.expiresAt);
|
|
119
|
+
sets.push(`expires_at = to_timestamp($${params.length}::double precision / 1000)`);
|
|
120
|
+
}
|
|
121
|
+
if (sets.length === 0)
|
|
122
|
+
return;
|
|
123
|
+
await run(`UPDATE ${table} SET ${sets.join(', ')}, updated_at = now() WHERE key = $1`, params);
|
|
124
|
+
},
|
|
125
|
+
async delete(key) {
|
|
126
|
+
await run(`DELETE FROM ${table} WHERE key = $1`, [scoped(key)]);
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Delete expired records. Returns how many went.
|
|
132
|
+
*
|
|
133
|
+
* Postgres has no TTL, so this is the counterpart to DynamoDB's automatic
|
|
134
|
+
* expiry and it needs a caller — a `schedules` intent running nightly is
|
|
135
|
+
* plenty. Nothing reads an expired row before it is deleted (`get()` filters
|
|
136
|
+
* on `expires_at`), so the only cost of never calling this is disk.
|
|
137
|
+
*/
|
|
138
|
+
export async function purgeExpiredIdempotencyRecords(options = {}) {
|
|
139
|
+
const table = options.tableName ?? DEFAULT_IDEMPOTENCY_TABLE;
|
|
140
|
+
assertSqlIdentifier(table, 'purgeExpiredIdempotencyRecords.tableName');
|
|
141
|
+
const run = options.querier ?? query;
|
|
142
|
+
const rows = await run(`DELETE FROM ${table} WHERE expires_at <= now() RETURNING key`);
|
|
143
|
+
return rows.length;
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=postgres.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postgres.js","sourceRoot":"","sources":["../../src/idempotency/postgres.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAE1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEhE,kFAAkF;AAClF,MAAM,CAAC,MAAM,yBAAyB,GAAG,uBAAuB,CAAC;AAiDjE,2EAA2E;AAC3E,MAAM,aAAc,SAAQ,KAAK;IACf,IAAI,GAAG,sBAAsB,CAAC;IAC9C,YAAY,GAAW;QACrB,KAAK,CAAC,mCAAmC,GAAG,0BAA0B,CAAC,CAAC;QACxE,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;IACzC,CAAC;CACF;AAiBD,MAAM,QAAQ,GAAG,CAAC,CAAgB,EAAU,EAAE,CAC5C,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC;AAE1D;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,8BAA8B,CAC5C,UAA2C,EAAE;IAE7C,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,IAAI,yBAAyB,CAAC;IAC7D,mBAAmB,CAAC,KAAK,EAAE,2CAA2C,CAAC,CAAC;IACxE,MAAM,GAAG,GAAY,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAE9C;wEACoE;IACpE,MAAM,MAAM,GAAG,CAAC,GAAW,EAAU,EAAE;QACrC,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;QAC5B,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IACjC,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,CAAC,GAAG,CAAC,GAAG;YACX;;4CAEgC;YAChC,MAAM,IAAI,GAAG,MAAM,GAAG,CACpB;kBACU,KAAK;gDACyB,EACxC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CACd,CAAC;YACF,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAC;YACtB,OAAO;gBACL,6DAA6D;gBAC7D,GAAG;gBACH,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC;aACpC,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,MAAM;YACf;;;;;qDAKyC;YACzC,MAAM,IAAI,GAAG,MAAM,GAAG,CACpB,eAAe,KAAK;;;;;;;sBAON,KAAK;sBACL,KAAK;yBACF,EACjB,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CACvE,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,MAAM,IAAI,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7D,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO;YACvB;qEACyD;YACzD,MAAM,IAAI,GAAa,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAc,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;YACxC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAC9B,IAAI,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC5B,IAAI,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC1C,CAAC;YACD,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAC/B,IAAI,CAAC,IAAI,CAAC,8BAA8B,MAAM,CAAC,MAAM,4BAA4B,CAAC,CAAC;YACrF,CAAC;YACD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAC9B,MAAM,GAAG,CACP,UAAU,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,qCAAqC,EAC3E,MAAM,CACP,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,GAAG;YACd,MAAM,GAAG,CAAC,eAAe,KAAK,iBAAiB,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClE,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAClD,UAA0E,EAAE;IAE5E,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,IAAI,yBAAyB,CAAC;IAC7D,mBAAmB,CAAC,KAAK,EAAE,0CAA0C,CAAC,CAAC;IACvE,MAAM,GAAG,GAAY,OAAO,CAAC,OAAO,IAAI,KAAK,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,GAAG,CACpB,eAAe,KAAK,0CAA0C,CAC/D,CAAC;IACF,OAAO,IAAI,CAAC,MAAM,CAAC;AACrB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @venturekit/data — SQL identifier validation.
|
|
3
|
+
*
|
|
4
|
+
* Postgres accepts parameter placeholders for *values* only, so a table or
|
|
5
|
+
* column name that comes from configuration has to be interpolated into the
|
|
6
|
+
* statement text. Restricting those to a conservative ASCII whitelist makes
|
|
7
|
+
* injection structurally impossible rather than a matter of trusting the caller.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Throw unless `name` is safe to interpolate into a statement as an identifier.
|
|
11
|
+
*
|
|
12
|
+
* `context` names the option the value came from, so the error tells the
|
|
13
|
+
* operator which setting to fix rather than just that something was rejected.
|
|
14
|
+
*/
|
|
15
|
+
export declare function assertSqlIdentifier(name: string, context: string): void;
|
|
16
|
+
//# sourceMappingURL=identifier.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"identifier.d.ts","sourceRoot":"","sources":["../../src/internal/identifier.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,CAOvE"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @venturekit/data — SQL identifier validation.
|
|
3
|
+
*
|
|
4
|
+
* Postgres accepts parameter placeholders for *values* only, so a table or
|
|
5
|
+
* column name that comes from configuration has to be interpolated into the
|
|
6
|
+
* statement text. Restricting those to a conservative ASCII whitelist makes
|
|
7
|
+
* injection structurally impossible rather than a matter of trusting the caller.
|
|
8
|
+
*/
|
|
9
|
+
/** Unquoted-identifier shape: a letter or underscore, then word characters. */
|
|
10
|
+
const SQL_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
11
|
+
/**
|
|
12
|
+
* Throw unless `name` is safe to interpolate into a statement as an identifier.
|
|
13
|
+
*
|
|
14
|
+
* `context` names the option the value came from, so the error tells the
|
|
15
|
+
* operator which setting to fix rather than just that something was rejected.
|
|
16
|
+
*/
|
|
17
|
+
export function assertSqlIdentifier(name, context) {
|
|
18
|
+
if (!SQL_IDENTIFIER.test(name)) {
|
|
19
|
+
throw new Error(`[venturekit/data] invalid SQL identifier for ${context}: '${name}'. ` +
|
|
20
|
+
`Must match ${String(SQL_IDENTIFIER)}.`);
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=identifier.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"identifier.js","sourceRoot":"","sources":["../../src/internal/identifier.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,+EAA+E;AAC/E,MAAM,cAAc,GAAG,0BAA0B,CAAC;AAElD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,OAAe;IAC/D,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACb,gDAAgD,OAAO,MAAM,IAAI,KAAK;YACpE,cAAc,MAAM,CAAC,cAAc,CAAC,GAAG,CAC1C,CAAC;IACJ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @venturekit/data — durable background jobs in Postgres.
|
|
3
|
+
*
|
|
4
|
+
* For work the product talks about, where the SQS `queues` intent is for work it
|
|
5
|
+
* does not. See `./postgres.ts`.
|
|
6
|
+
*/
|
|
7
|
+
export { enqueueJob, claimJobs, completeJob, failJob, releaseJob, reclaimStuckJobs, listJobs, runJobsOnce, DEFAULT_JOB_TABLE, } from './postgres.js';
|
|
8
|
+
export type { Job, JobStatus, JobHandler, EnqueueJobInput, ClaimJobsOptions, FailJobOptions, ReclaimOptions, ListJobsOptions, RunJobsResult, JobTableOption, JobQuerierOption, } from './postgres.js';
|
|
9
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/jobs/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,UAAU,EACV,SAAS,EACT,WAAW,EACX,OAAO,EACP,UAAU,EACV,gBAAgB,EAChB,QAAQ,EACR,WAAW,EACX,iBAAiB,GAClB,MAAM,eAAe,CAAC;AAEvB,YAAY,EACV,GAAG,EACH,SAAS,EACT,UAAU,EACV,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,eAAe,EACf,aAAa,EACb,cAAc,EACd,gBAAgB,GACjB,MAAM,eAAe,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @venturekit/data — durable background jobs in Postgres.
|
|
3
|
+
*
|
|
4
|
+
* For work the product talks about, where the SQS `queues` intent is for work it
|
|
5
|
+
* does not. See `./postgres.ts`.
|
|
6
|
+
*/
|
|
7
|
+
export { enqueueJob, claimJobs, completeJob, failJob, releaseJob, reclaimStuckJobs, listJobs, runJobsOnce, DEFAULT_JOB_TABLE, } from './postgres.js';
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/jobs/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,UAAU,EACV,SAAS,EACT,WAAW,EACX,OAAO,EACP,UAAU,EACV,gBAAgB,EAChB,QAAQ,EACR,WAAW,EACX,iBAAiB,GAClB,MAAM,eAAe,CAAC"}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @venturekit/data — durable background jobs in Postgres.
|
|
3
|
+
*
|
|
4
|
+
* The `queues` intent (SQS) stays the default for opaque, high-throughput
|
|
5
|
+
* fan-out. This is for the other kind of work: the kind the product talks about.
|
|
6
|
+
* "Which imports are still geocoding for this tenant" is a screen, and a queue
|
|
7
|
+
* has no query surface — messages are invisible until received and then
|
|
8
|
+
* invisible to everyone else. Rebuilding that visibility over SQS means a second
|
|
9
|
+
* table tracking what you enqueued, at which point the queue is the redundant
|
|
10
|
+
* half, because the table can be claimed from directly.
|
|
11
|
+
*
|
|
12
|
+
* Unlike the outbox, there is no port and no adapter here. The whole value is
|
|
13
|
+
* that the work lives in queryable rows, and the claim — `FOR UPDATE SKIP
|
|
14
|
+
* LOCKED` — is SQL rather than portable logic. An abstraction over it would have
|
|
15
|
+
* exactly one implementation and would hide the one feature worth having.
|
|
16
|
+
*
|
|
17
|
+
* ```ts
|
|
18
|
+
* // enqueue in the transaction that made the work necessary
|
|
19
|
+
* await withTransaction(async (tx) => {
|
|
20
|
+
* const batch = await stageImport(tx, rows);
|
|
21
|
+
* await enqueueJob(tx.query, { kind: 'geocode-import', tenantId, payload: { batch: batch.id } });
|
|
22
|
+
* });
|
|
23
|
+
*
|
|
24
|
+
* // drain in a `schedules` cron
|
|
25
|
+
* await runJobsOnce({
|
|
26
|
+
* 'geocode-import': async (job) => { await geocode(job.payload); },
|
|
27
|
+
* });
|
|
28
|
+
* ```
|
|
29
|
+
*/
|
|
30
|
+
import type { Querier } from '../query/index.js';
|
|
31
|
+
/** Default table name, created by `vk_data_003_jobs.sql`. */
|
|
32
|
+
export declare const DEFAULT_JOB_TABLE = "vk_job";
|
|
33
|
+
export type JobStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'cancelled';
|
|
34
|
+
export interface Job<TPayload = unknown> {
|
|
35
|
+
id: string;
|
|
36
|
+
tenantId: string | null;
|
|
37
|
+
kind: string;
|
|
38
|
+
payload: TPayload;
|
|
39
|
+
status: JobStatus;
|
|
40
|
+
runAfter: string;
|
|
41
|
+
attempts: number;
|
|
42
|
+
maxAttempts: number;
|
|
43
|
+
lockedAt: string | null;
|
|
44
|
+
lockedBy: string | null;
|
|
45
|
+
lastError: string | null;
|
|
46
|
+
createdAt: string;
|
|
47
|
+
finishedAt: string | null;
|
|
48
|
+
}
|
|
49
|
+
export interface EnqueueJobInput<TPayload = unknown> {
|
|
50
|
+
kind: string;
|
|
51
|
+
payload?: TPayload;
|
|
52
|
+
/** `null` (the default) is platform-wide work belonging to no tenant. */
|
|
53
|
+
tenantId?: string | null;
|
|
54
|
+
/** Earliest run time. Omit for "as soon as a worker picks it up". */
|
|
55
|
+
runAfter?: Date | string;
|
|
56
|
+
/** Failures after which the job is left `failed` for an operator. Default 5. */
|
|
57
|
+
maxAttempts?: number;
|
|
58
|
+
}
|
|
59
|
+
export interface JobTableOption {
|
|
60
|
+
/** Table name. Validated as an identifier, since it is interpolated. */
|
|
61
|
+
tableName?: string;
|
|
62
|
+
}
|
|
63
|
+
export interface JobQuerierOption extends JobTableOption {
|
|
64
|
+
/** Runs the statements. Defaults to the pooled `query`. */
|
|
65
|
+
querier?: Querier;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* Enqueue a job **on the caller's transaction**.
|
|
69
|
+
*
|
|
70
|
+
* `querier` is required and not defaulted, for the same reason as
|
|
71
|
+
* `appendToOutbox`: a job enqueued on a pooled connection can commit while the
|
|
72
|
+
* transaction that needed it rolls back, leaving a worker to act on a state that
|
|
73
|
+
* does not exist. Pass the transaction's `query`.
|
|
74
|
+
*/
|
|
75
|
+
export declare function enqueueJob<TPayload = unknown>(querier: Querier, input: EnqueueJobInput<TPayload>, options?: JobTableOption): Promise<string>;
|
|
76
|
+
export interface ClaimJobsOptions extends JobQuerierOption {
|
|
77
|
+
/** How many to take. Default 10. */
|
|
78
|
+
limit?: number;
|
|
79
|
+
/** Only these kinds. Omit for any. */
|
|
80
|
+
kinds?: string[];
|
|
81
|
+
/**
|
|
82
|
+
* Identifies the claiming worker in `locked_by`. Default
|
|
83
|
+
* `AWS_LAMBDA_LOG_STREAM_NAME` or `'worker'` — a value that survives into the
|
|
84
|
+
* row so a stuck job can be traced to the process that abandoned it.
|
|
85
|
+
*/
|
|
86
|
+
workerId?: string;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Claim due jobs, marking them `running`.
|
|
90
|
+
*
|
|
91
|
+
* `FOR UPDATE SKIP LOCKED` inside the UPDATE's subquery is what makes several
|
|
92
|
+
* workers safe on one table: each takes rows nobody else holds and skips the
|
|
93
|
+
* rest rather than queueing behind them. Without SKIP LOCKED, a second worker
|
|
94
|
+
* blocks on the first's rows and the pool drains.
|
|
95
|
+
*/
|
|
96
|
+
export declare function claimJobs<TPayload = unknown>(options?: ClaimJobsOptions): Promise<Job<TPayload>[]>;
|
|
97
|
+
/** Mark a claimed job done. */
|
|
98
|
+
export declare function completeJob(id: string, options?: JobQuerierOption): Promise<void>;
|
|
99
|
+
export interface FailJobOptions extends JobQuerierOption {
|
|
100
|
+
/**
|
|
101
|
+
* Seconds before the retry becomes due. Omit for exponential backoff from the
|
|
102
|
+
* attempt count: 2^attempts seconds, capped at an hour.
|
|
103
|
+
*/
|
|
104
|
+
retryInSeconds?: number;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Record a failed attempt: re-queue it with a later `run_after`, or leave it
|
|
108
|
+
* `failed` once the attempts are spent.
|
|
109
|
+
*
|
|
110
|
+
* `attempts` was already incremented by the claim, so a crash between claim and
|
|
111
|
+
* this call still counts — otherwise a job that kills its worker every time
|
|
112
|
+
* would retry forever.
|
|
113
|
+
*/
|
|
114
|
+
export declare function failJob(id: string, error: unknown, options?: FailJobOptions): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Hand a claimed job back **without counting the attempt**.
|
|
117
|
+
*
|
|
118
|
+
* For "I cannot run this right now", as distinct from "I ran it and it failed":
|
|
119
|
+
* no handler is registered yet, a dependency is briefly unavailable, the worker
|
|
120
|
+
* is shutting down. The claim already incremented `attempts`, so this undoes
|
|
121
|
+
* that — otherwise a job nobody can run yet would exhaust its attempts while
|
|
122
|
+
* never actually being attempted, which is how a mid-deploy job ends up `failed`
|
|
123
|
+
* for no reason.
|
|
124
|
+
*/
|
|
125
|
+
export declare function releaseJob(id: string, reason: string, options?: JobQuerierOption & {
|
|
126
|
+
retryInSeconds?: number;
|
|
127
|
+
}): Promise<void>;
|
|
128
|
+
export interface ReclaimOptions extends JobQuerierOption {
|
|
129
|
+
/**
|
|
130
|
+
* How long a `running` job may hold its lock before it is presumed abandoned.
|
|
131
|
+
* Default 900 (15 minutes) — comfortably past any Lambda's own timeout, since
|
|
132
|
+
* reclaiming a job that is merely slow runs it twice.
|
|
133
|
+
*/
|
|
134
|
+
staleAfterSeconds?: number;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Re-queue jobs left `running` by a worker that died, and return how many.
|
|
138
|
+
*
|
|
139
|
+
* A crashed worker cannot clean up after itself, so nothing else will ever move
|
|
140
|
+
* these rows: `running` plus an old `locked_at` is indistinguishable from a dead
|
|
141
|
+
* process, and treating it as one is the only way the work resumes. Worth a
|
|
142
|
+
* schedule of its own, or a call at the top of each worker pass.
|
|
143
|
+
*
|
|
144
|
+
* Jobs whose attempts are spent go to `failed` rather than back to `queued` —
|
|
145
|
+
* a job that reliably kills its worker must not do so forever.
|
|
146
|
+
*/
|
|
147
|
+
export declare function reclaimStuckJobs(options?: ReclaimOptions): Promise<number>;
|
|
148
|
+
export interface ListJobsOptions extends JobQuerierOption {
|
|
149
|
+
kind?: string;
|
|
150
|
+
status?: JobStatus;
|
|
151
|
+
limit?: number;
|
|
152
|
+
/**
|
|
153
|
+
* Confine the result to the request's tenant scope. Default `true`, which is
|
|
154
|
+
* the safe default for anything serving a request: this table carries no RLS
|
|
155
|
+
* policy (a worker must see every tenant's work, and platform rows have no
|
|
156
|
+
* tenant at all), so the scope predicate has to be applied here instead.
|
|
157
|
+
*
|
|
158
|
+
* Set `false` only in a worker or an operator tool that is meant to see
|
|
159
|
+
* everything.
|
|
160
|
+
*/
|
|
161
|
+
tenantScoped?: boolean;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* The query this table exists to make possible: what work is outstanding.
|
|
165
|
+
*
|
|
166
|
+
* With `tenantScoped` (the default) the predicate is
|
|
167
|
+
* `tenant_id = ANY (vk_tenant_scope())` — the same predicate an RLS policy would
|
|
168
|
+
* apply, so a caller who goes through this helper cannot see another tenant's
|
|
169
|
+
* jobs. Platform rows (`tenant_id IS NULL`) fall out naturally, since
|
|
170
|
+
* `NULL = ANY (...)` is never true.
|
|
171
|
+
*/
|
|
172
|
+
export declare function listJobs<TPayload = unknown>(options?: ListJobsOptions): Promise<Job<TPayload>[]>;
|
|
173
|
+
/** A handler for one `kind`. Throwing marks the attempt failed. */
|
|
174
|
+
export type JobHandler<TPayload = unknown> = (job: Job<TPayload>) => Promise<void>;
|
|
175
|
+
export interface RunJobsResult {
|
|
176
|
+
claimed: number;
|
|
177
|
+
succeeded: number;
|
|
178
|
+
failed: number;
|
|
179
|
+
/** Claimed jobs whose `kind` had no handler. Re-queued, not failed. */
|
|
180
|
+
unhandled: number;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* One worker pass: claim, dispatch, settle. The job-queue analogue of the
|
|
184
|
+
* outbox's `relayOnce`.
|
|
185
|
+
*
|
|
186
|
+
* Returns counts rather than throwing on a handler failure — a failing job is
|
|
187
|
+
* an expected operating condition and the caller's job is to schedule the next
|
|
188
|
+
* pass. A failing *store* propagates, for the same reason as in the relay: a
|
|
189
|
+
* worker that cannot record outcomes must stop rather than keep claiming.
|
|
190
|
+
*
|
|
191
|
+
* A `kind` with no handler is re-queued with a short delay rather than failed.
|
|
192
|
+
* A deploy in progress is the usual cause — one version enqueues a kind the
|
|
193
|
+
* still-running version does not know — and burning an attempt on that would
|
|
194
|
+
* exhaust a perfectly good job during a rollout.
|
|
195
|
+
*/
|
|
196
|
+
export declare function runJobsOnce(handlers: Record<string, JobHandler<never>>, options?: ClaimJobsOptions): Promise<RunJobsResult>;
|
|
197
|
+
//# sourceMappingURL=postgres.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"postgres.d.ts","sourceRoot":"","sources":["../../src/jobs/postgres.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAGH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAGjD,6DAA6D;AAC7D,eAAO,MAAM,iBAAiB,WAAW,CAAC;AAE1C,MAAM,MAAM,SAAS,GACjB,QAAQ,GACR,SAAS,GACT,WAAW,GACX,QAAQ,GACR,WAAW,CAAC;AAEhB,MAAM,WAAW,GAAG,CAAC,QAAQ,GAAG,OAAO;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,QAAQ,CAAC;IAClB,MAAM,EAAE,SAAS,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe,CAAC,QAAQ,GAAG,OAAO;IACjD,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,QAAQ,CAAC;IACnB,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,qEAAqE;IACrE,QAAQ,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC;IACzB,gFAAgF;IAChF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,gBAAiB,SAAQ,cAAc;IACtD,2DAA2D;IAC3D,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAmDD;;;;;;;GAOG;AACH,wBAAsB,UAAU,CAAC,QAAQ,GAAG,OAAO,EACjD,OAAO,EAAE,OAAO,EAChB,KAAK,EAAE,eAAe,CAAC,QAAQ,CAAC,EAChC,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,MAAM,CAAC,CAiBjB;AAED,MAAM,WAAW,gBAAiB,SAAQ,gBAAgB;IACxD,oCAAoC;IACpC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;GAOG;AACH,wBAAsB,SAAS,CAAC,QAAQ,GAAG,OAAO,EAChD,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CA2B1B;AAED,+BAA+B;AAC/B,wBAAsB,WAAW,CAC/B,EAAE,EAAE,MAAM,EACV,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,MAAM,WAAW,cAAe,SAAQ,gBAAgB;IACtD;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;GAOG;AACH,wBAAsB,OAAO,CAC3B,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,OAAO,EACd,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,IAAI,CAAC,CAqBf;AAED;;;;;;;;;GASG;AACH,wBAAsB,UAAU,CAC9B,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,gBAAgB,GAAG;IAAE,cAAc,CAAC,EAAE,MAAM,CAAA;CAAO,GAC3D,OAAO,CAAC,IAAI,CAAC,CAef;AAED,MAAM,WAAW,cAAe,SAAQ,gBAAgB;IACtD;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,GAAE,cAAmB,GAC3B,OAAO,CAAC,MAAM,CAAC,CAkBjB;AAED,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACvD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,SAAS,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf;;;;;;;;OAQG;IACH,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAED;;;;;;;;GAQG;AACH,wBAAsB,QAAQ,CAAC,QAAQ,GAAG,OAAO,EAC/C,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAe1B;AAED,mEAAmE;AACnE,MAAM,MAAM,UAAU,CAAC,QAAQ,GAAG,OAAO,IAAI,CAC3C,GAAG,EAAE,GAAG,CAAC,QAAQ,CAAC,KACf,OAAO,CAAC,IAAI,CAAC,CAAC;AAEnB,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,uEAAuE;IACvE,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAC3C,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,aAAa,CAAC,CAqCxB"}
|