@ultimat3/jobs 29.3.2 → 30.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +68 -89
- package/README.md +304 -10
- package/package.json +18 -6
- package/src/backfill-definition.ts +184 -0
- package/src/backfill-errors.ts +18 -2
- package/src/backfill-pass.ts +43 -40
- package/src/backfill-pending.ts +40 -17
- package/src/backfill-reader.ts +94 -0
- package/src/backfill-registry.ts +5 -0
- package/src/backfill-scope.ts +1 -2
- package/src/backfill-source.ts +113 -0
- package/src/backfill.ts +54 -135
- package/src/consumer-errors.ts +42 -0
- package/src/consumer-guard.ts +19 -0
- package/src/consumer-loop.ts +171 -0
- package/src/consumer-registry.ts +103 -0
- package/src/consumer-types.ts +81 -0
- package/src/consumer.ts +147 -0
- package/src/driver-memory-operator.ts +25 -2
- package/src/driver-memory.ts +35 -1
- package/src/driver-pg-claim-sql.ts +128 -0
- package/src/driver-pg-ddl.ts +11 -2
- package/src/driver-pg-demand-sql.ts +27 -0
- package/src/driver-pg-enqueue-many.ts +153 -0
- package/src/driver-pg-event-wake-sql.ts +51 -0
- package/src/driver-pg-operator-sql.ts +34 -6
- package/src/driver-pg-operator.ts +36 -2
- package/src/driver-pg-outbox-sql.ts +34 -0
- package/src/driver-pg-sql.ts +24 -91
- package/src/driver-pg.ts +17 -1
- package/src/driver-redis-refusal.ts +25 -0
- package/src/driver-redis-scripts.ts +79 -30
- package/src/driver-redis.ts +70 -42
- package/src/driver.ts +38 -1
- package/src/enqueue-record.ts +137 -0
- package/src/error-codes.ts +233 -0
- package/src/errors-concurrency.ts +2 -2
- package/src/errors-declaration.ts +15 -2
- package/src/errors-operator.ts +1 -1
- package/src/errors-requeue.ts +1 -1
- package/src/errors-saga.ts +87 -0
- package/src/errors-steps.ts +35 -0
- package/src/errors-tenant.ts +1 -1
- package/src/errors.ts +25 -165
- package/src/events-pg.ts +20 -8
- package/src/events.ts +14 -3
- package/src/execute-ctx.ts +51 -0
- package/src/execute-unwind.ts +191 -0
- package/src/execute.ts +50 -51
- package/src/export-errors.ts +1 -1
- package/src/failure-row.ts +43 -0
- package/src/import-errors.ts +102 -0
- package/src/import-format.ts +232 -0
- package/src/import-pass.ts +255 -0
- package/src/import-source.ts +49 -0
- package/src/import.ts +175 -0
- package/src/index.ts +163 -7
- package/src/inspect-operator.ts +24 -3
- package/src/inspect.ts +17 -4
- package/src/introspection.ts +24 -1
- package/src/job-enqueue-many.ts +108 -0
- package/src/job-types.ts +229 -0
- package/src/job.ts +19 -208
- package/src/metrics.ts +10 -0
- package/src/outbox-pg.ts +35 -0
- package/src/outbox-relay.ts +58 -19
- package/src/outbox.ts +65 -73
- package/src/purge-entities.ts +92 -0
- package/src/purge.ts +10 -2
- package/src/queue-wake.ts +1 -1
- package/src/refresh-view.ts +76 -0
- package/src/retry-classification.ts +27 -8
- package/src/retry.ts +26 -0
- package/src/rollup-errors.ts +27 -0
- package/src/rollup.ts +190 -0
- package/src/run-deadline.ts +11 -7
- package/src/scheduler.ts +2 -1
- package/src/settled.ts +56 -15
- package/src/step-resolve.ts +37 -0
- package/src/steps-api.ts +111 -0
- package/src/steps-core.ts +26 -0
- package/src/steps-output.ts +142 -0
- package/src/steps-store.ts +68 -0
- package/src/steps-suspension.ts +51 -0
- package/src/steps-timeout.ts +10 -5
- package/src/steps-unwind.ts +94 -0
- package/src/steps-wait.ts +163 -0
- package/src/steps.ts +174 -210
- package/src/subject-erase-pass.ts +134 -0
- package/src/subject-errors.ts +48 -0
- package/src/subject-export-pass.ts +228 -0
- package/src/subject-jobs.ts +121 -0
- package/src/subject-plan.ts +287 -0
- package/src/subject-select.ts +106 -0
- package/src/subject.ts +119 -0
- package/src/tenant.ts +69 -1
- package/src/type-pins.ts +90 -0
- package/src/webhook-errors.ts +1 -1
- package/src/webhook-receiver-errors.ts +59 -0
- package/src/webhook-receiver-route.ts +127 -0
- package/src/webhook-receiver-types.ts +97 -0
- package/src/webhook-receiver.ts +132 -0
- package/src/webhook-target.ts +20 -43
- package/src/webhook.ts +7 -4
- package/src/worker-queue-demand.ts +57 -0
- package/src/worker-queue-depth.ts +15 -3
- package/src/worker-registry.ts +4 -1
- package/src/worker-types.ts +2 -0
- package/src/worker.ts +2 -2
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// How one run finds the subject's rows: through the app's own repositories (a `database()` over
|
|
2
|
+
// the app's driver, so tenancy, the row observer and cache tags hold), a page at a time. A table's
|
|
3
|
+
// rows are those whose key is in a page of its parent's rows — never a statement unbounded by a page.
|
|
4
|
+
|
|
5
|
+
import { assert } from '@ultimat3/core';
|
|
6
|
+
import type { EntityCore, Table } from '@ultimat3/entity';
|
|
7
|
+
import { database, getEntity } from '@ultimat3/entity';
|
|
8
|
+
import type { Subject } from './subject';
|
|
9
|
+
import type { SubjectPlan, SubjectVia } from './subject-plan';
|
|
10
|
+
|
|
11
|
+
export type SubjectRow = Readonly<Record<string, unknown>>;
|
|
12
|
+
export type SubjectTable = Table<Record<string, unknown>>;
|
|
13
|
+
|
|
14
|
+
export interface SubjectReader {
|
|
15
|
+
readonly plan: SubjectPlan;
|
|
16
|
+
readonly key: string;
|
|
17
|
+
readonly size: number;
|
|
18
|
+
table(entity: string): SubjectTable;
|
|
19
|
+
core(entity: string): EntityCore;
|
|
20
|
+
/** The subject's row, read now — never checkpointed, since it is the person's data. */
|
|
21
|
+
rootRow(): Promise<SubjectRow | null>;
|
|
22
|
+
/** Every row of `entity` the plan reaches, a page at a time. A row two paths reach comes twice. */
|
|
23
|
+
pages(entity: string): AsyncGenerator<readonly SubjectRow[]>;
|
|
24
|
+
/** The values `via` matches on, a page of the parent's rows at a time. */
|
|
25
|
+
parentKeys(via: SubjectVia): AsyncGenerator<readonly unknown[]>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** The primary key of one row as one string: what tells two pages' copies of a row apart. */
|
|
29
|
+
export const rowIdentity = (core: EntityCore, row: SubjectRow): string =>
|
|
30
|
+
JSON.stringify(core.$primaryKey.map((property) => String(row[property])));
|
|
31
|
+
|
|
32
|
+
export function subjectReader<I>(
|
|
33
|
+
subject: Subject<I>,
|
|
34
|
+
plan: SubjectPlan,
|
|
35
|
+
key: string,
|
|
36
|
+
): SubjectReader {
|
|
37
|
+
const cores = new Map<string, EntityCore>();
|
|
38
|
+
for (const name of [plan.subject, ...plan.steps.map((step) => step.entity)]) {
|
|
39
|
+
const core = getEntity(name)?.core;
|
|
40
|
+
assert(
|
|
41
|
+
core !== undefined,
|
|
42
|
+
`the subject ${plan.subject} reaches ${name}, which registered no entity core`,
|
|
43
|
+
`x exercise --json # lists the subject's reach; declare ${name} with entity() before the subject runs`,
|
|
44
|
+
);
|
|
45
|
+
cores.set(name, core as EntityCore);
|
|
46
|
+
}
|
|
47
|
+
const db = database(Object.fromEntries(cores), { driver: subject.init.driver });
|
|
48
|
+
const tables = new Map<string, SubjectTable>(
|
|
49
|
+
Object.entries(db).map(([name, table]) => [name, table as unknown as SubjectTable]),
|
|
50
|
+
);
|
|
51
|
+
const steps = new Map(plan.steps.map((step) => [step.entity, step]));
|
|
52
|
+
const size = subject.batch;
|
|
53
|
+
|
|
54
|
+
const table = (entity: string): SubjectTable => {
|
|
55
|
+
const found = tables.get(entity);
|
|
56
|
+
assert(
|
|
57
|
+
found !== undefined,
|
|
58
|
+
`the subject ${plan.subject} asked for ${entity}, which its plan does not reach`,
|
|
59
|
+
`x exercise --json # prints the subject's plan`,
|
|
60
|
+
);
|
|
61
|
+
return found;
|
|
62
|
+
};
|
|
63
|
+
const rootRow = (): Promise<SubjectRow | null> =>
|
|
64
|
+
table(plan.subject)
|
|
65
|
+
.where({ [plan.key]: key })
|
|
66
|
+
.one();
|
|
67
|
+
|
|
68
|
+
async function* parentKeys(via: SubjectVia): AsyncGenerator<readonly unknown[]> {
|
|
69
|
+
if (via.parent === plan.subject) {
|
|
70
|
+
const row = await rootRow();
|
|
71
|
+
const value = row?.[via.parentKey];
|
|
72
|
+
if (value !== undefined && value !== null) yield [value];
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
for await (const page of pages(via.parent)) {
|
|
76
|
+
const values = new Set(page.map((row) => row[via.parentKey]));
|
|
77
|
+
values.delete(null);
|
|
78
|
+
values.delete(undefined);
|
|
79
|
+
if (values.size > 0) yield [...values];
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function* pages(entity: string): AsyncGenerator<readonly SubjectRow[]> {
|
|
84
|
+
if (entity === plan.subject) {
|
|
85
|
+
const row = await rootRow();
|
|
86
|
+
if (row !== null) yield [row];
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
for (const via of steps.get(entity)?.via ?? []) {
|
|
90
|
+
for await (const values of parentKeys(via)) {
|
|
91
|
+
yield* table(entity).andWhere(via.property, 'in', values).inBatches(size);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
plan,
|
|
98
|
+
key,
|
|
99
|
+
size,
|
|
100
|
+
table,
|
|
101
|
+
core: (entity) => cores.get(entity) as EntityCore,
|
|
102
|
+
rootRow,
|
|
103
|
+
pages,
|
|
104
|
+
parentKeys,
|
|
105
|
+
};
|
|
106
|
+
}
|
package/src/subject.ts
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
// `defineSubject()`: one person's data, declared once — which entity they are, how to find their
|
|
2
|
+
// row, and what happens to every table holding something of theirs. A plain value both factories
|
|
3
|
+
// take (`subjectExport`, `subjectErase`); the reach itself is derived from foreign keys
|
|
4
|
+
// (`subject-plan.ts`), so a table added next month is a build error until it has a treatment.
|
|
5
|
+
|
|
6
|
+
import { assert } from '@ultimat3/core';
|
|
7
|
+
import type { Driver, EntityCore } from '@ultimat3/entity';
|
|
8
|
+
import type { ExportValue } from './export-format';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* What happens to one table's rows of the subject. Nothing defaults (axiom 8): which law applies,
|
|
12
|
+
* what is kept and for how long are the app's, so every reached table states one of the three.
|
|
13
|
+
*/
|
|
14
|
+
export type SubjectTreatment =
|
|
15
|
+
| 'delete'
|
|
16
|
+
/** Overwrite these columns, keep the row. A key set to `null` unlinks it from the subject. */
|
|
17
|
+
| { readonly anonymize: Readonly<Record<string, unknown>> }
|
|
18
|
+
/** Keep the row as it is. The reason is required and printed in the export's index. */
|
|
19
|
+
| { readonly retain: string };
|
|
20
|
+
|
|
21
|
+
/** The subject as one run sees it: the key it was asked for and its row, read fresh per step. */
|
|
22
|
+
export interface SubjectRef {
|
|
23
|
+
readonly key: string;
|
|
24
|
+
readonly row: Readonly<Record<string, unknown>>;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** One page a target answers for an export: rows over `columns`, and where the next page starts. */
|
|
28
|
+
export interface SubjectTargetPage {
|
|
29
|
+
readonly rows: readonly Readonly<Record<string, ExportValue>>[];
|
|
30
|
+
readonly cursor: string | null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Data the framework holds outside an app's entities — `x_users`, `x_audit`, stored files —
|
|
35
|
+
* behind the narrowest seam, structural exactly as `PurgeTarget` is: `@ultimat3/auth`,
|
|
36
|
+
* `@ultimat3/action` and `@ultimat3/storage` each satisfy it without this package importing them.
|
|
37
|
+
*/
|
|
38
|
+
export interface SubjectTarget {
|
|
39
|
+
/** The step key, the export's object prefix and the report's row. Unique within one subject. */
|
|
40
|
+
readonly name: string;
|
|
41
|
+
/** The keys of every row `exportPage` answers, in order: the csv header, the ndjson key order. */
|
|
42
|
+
readonly columns: readonly string[];
|
|
43
|
+
exportPage(subject: SubjectRef, cursor: string | null, size: number): Promise<SubjectTargetPage>;
|
|
44
|
+
/** Remove up to `size` of the subject's records and answer how many went; called until short. */
|
|
45
|
+
erase(subject: SubjectRef, size: number): Promise<number>;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** A target the erase leaves alone: still exported, never erased, the reason printed. */
|
|
49
|
+
export interface RetainedTarget {
|
|
50
|
+
readonly target: SubjectTarget;
|
|
51
|
+
readonly retain: string;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export type SubjectTargetEntry = SubjectTarget | RetainedTarget;
|
|
55
|
+
|
|
56
|
+
export interface SubjectInit<I> {
|
|
57
|
+
/** The driver the app's own `database()` reads through, so tenancy and row observers hold. */
|
|
58
|
+
readonly driver: Driver;
|
|
59
|
+
/** The subject's primary key, from the job's input. */
|
|
60
|
+
readonly key: (input: I) => string;
|
|
61
|
+
/** By entity name: every table the subject reaches, the subject's own included. */
|
|
62
|
+
readonly tables: Readonly<Record<string, SubjectTreatment>>;
|
|
63
|
+
readonly targets?: readonly SubjectTargetEntry[];
|
|
64
|
+
/**
|
|
65
|
+
* By entity name: a column holding the subject's key with no foreign key to say so. A table the
|
|
66
|
+
* reference walk cannot see is invisible until it is listed here, and then it needs a treatment.
|
|
67
|
+
*/
|
|
68
|
+
readonly columns?: Readonly<Record<string, string>>;
|
|
69
|
+
/** Rows per statement and per step. Default `DEFAULT_SUBJECT_BATCH`. */
|
|
70
|
+
readonly batch?: number;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface Subject<I> {
|
|
74
|
+
/** The subject entity's name. */
|
|
75
|
+
readonly entity: string;
|
|
76
|
+
readonly init: SubjectInit<I>;
|
|
77
|
+
readonly batch: number;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** One statement's worth of one person's rows: small, since a run is one person, never a table. */
|
|
81
|
+
export const DEFAULT_SUBJECT_BATCH = 200;
|
|
82
|
+
|
|
83
|
+
const declared: Subject<never>[] = [];
|
|
84
|
+
|
|
85
|
+
export const isRetainedTarget = (entry: SubjectTargetEntry): entry is RetainedTarget =>
|
|
86
|
+
'retain' in entry && 'target' in entry;
|
|
87
|
+
|
|
88
|
+
export const targetOf = (entry: SubjectTargetEntry): SubjectTarget =>
|
|
89
|
+
isRetainedTarget(entry) ? entry.target : entry;
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Declare the subject. Checked here only for what needs no other entity — the reach is checked
|
|
93
|
+
* when a run plans it and by `x exercise`, once every entity module has loaded.
|
|
94
|
+
*/
|
|
95
|
+
export function defineSubject<I>(entity: EntityCore, init: SubjectInit<I>): Subject<I> {
|
|
96
|
+
const batch = init.batch ?? DEFAULT_SUBJECT_BATCH;
|
|
97
|
+
assert(
|
|
98
|
+
Number.isSafeInteger(batch) && batch >= 1,
|
|
99
|
+
`defineSubject(${entity.$name}) declares batch: ${String(batch)}`,
|
|
100
|
+
`batch: ${DEFAULT_SUBJECT_BATCH} # in defineSubject(${entity.$name}, …): rows per statement, at least one`,
|
|
101
|
+
);
|
|
102
|
+
const names = (init.targets ?? []).map((entry) => targetOf(entry).name);
|
|
103
|
+
assert(
|
|
104
|
+
new Set(names).size === names.length,
|
|
105
|
+
`defineSubject(${entity.$name}) lists targets with one name twice: ${names.join(', ')}`,
|
|
106
|
+
`targets: [ … ] # in defineSubject(${entity.$name}, …): one entry per target name — the name is the step key`,
|
|
107
|
+
);
|
|
108
|
+
const subject: Subject<I> = { entity: entity.$name, init, batch };
|
|
109
|
+
declared.push(subject as Subject<never>);
|
|
110
|
+
return subject;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Every subject declared in this process, in declaration order. Read by `x exercise`. */
|
|
114
|
+
export const registeredSubjects = (): readonly Subject<never>[] => [...declared];
|
|
115
|
+
|
|
116
|
+
/** Test-only: forget every declared subject. */
|
|
117
|
+
export function resetSubjects(): void {
|
|
118
|
+
declared.length = 0;
|
|
119
|
+
}
|
package/src/tenant.ts
CHANGED
|
@@ -4,8 +4,10 @@
|
|
|
4
4
|
// by every job is the cross-tenant read this declaration exists to make impossible.
|
|
5
5
|
|
|
6
6
|
import type { Actor } from '@ultimat3/core';
|
|
7
|
-
import { assert } from '@ultimat3/core';
|
|
7
|
+
import { assert, isUltimateError, renderFixShellArg, tryUseContext } from '@ultimat3/core';
|
|
8
|
+
import { assertRowTenant, ORG_COLUMN } from '@ultimat3/entity';
|
|
8
9
|
import { JobTenantRequiredError } from './errors';
|
|
10
|
+
import { JobTenantMismatchError } from './errors-tenant';
|
|
9
11
|
|
|
10
12
|
/** The literal a job spells when it belongs to no tenant. */
|
|
11
13
|
export const NO_JOB_TENANT = 'none';
|
|
@@ -59,3 +61,69 @@ export function jobRunActor(actor: Actor, orgId: string | undefined): Actor {
|
|
|
59
61
|
if (actor.orgId === orgId) return actor;
|
|
60
62
|
return Object.freeze({ ...actor, orgId });
|
|
61
63
|
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The enqueue's half of the rule: a job whose payload names org A is refused when the enqueue says
|
|
67
|
+
* org B — an explicit `tenantId`, the org `handle.as(actor, …)` stamped, or the AMBIENT actor's
|
|
68
|
+
* org when one is present. It used to queue, and the run then acted as A for a caller in B.
|
|
69
|
+
* `tenant: 'none'` declares no org, so there is nothing to disagree with. The ambient half alone
|
|
70
|
+
* has one escape, entity's: a proven `crossTenant(reason, fn)` (`crossTenantAdmits`).
|
|
71
|
+
*/
|
|
72
|
+
export function refuseEnqueueTenant(
|
|
73
|
+
job: string,
|
|
74
|
+
declared: string | undefined,
|
|
75
|
+
tenantId: string | undefined,
|
|
76
|
+
): void {
|
|
77
|
+
if (declared === undefined) return;
|
|
78
|
+
if (tenantId !== undefined && tenantId !== declared) {
|
|
79
|
+
throw new JobTenantMismatchError({
|
|
80
|
+
job,
|
|
81
|
+
reason: `was enqueued for org "${tenantId}" while its payload names org "${declared}", so its run would act for an org the enqueuer is not in`,
|
|
82
|
+
fix: `${job}.enqueue(input) # from the org the payload names — an actor of "${declared}" — or with input naming "${tenantId}"`,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
const ambient = tryUseContext()?.actor?.orgId;
|
|
86
|
+
if (ambient !== undefined && ambient !== declared && !crossTenantAdmits(job, declared)) {
|
|
87
|
+
throw new JobTenantMismatchError({
|
|
88
|
+
job,
|
|
89
|
+
reason: `was enqueued by an actor of org "${ambient}" while its payload names org "${declared}"`,
|
|
90
|
+
fix: `${job}.enqueue(input) # from an actor of "${declared}", with input naming "${ambient}", or inside crossTenant('<why>', () => ${job}.enqueue(input)) by an actor holding tenancy:cross`,
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Whether an enclosing `crossTenant(reason, fn)` admits a payload naming `declared` — asked of
|
|
97
|
+
* `@ultimat3/entity`'s own write guard, the one place that proof is read: the payload's org is a
|
|
98
|
+
* tenant value the ambient actor is writing. Inside a proven scope the guard re-proves
|
|
99
|
+
* `tenancy:cross` and passes (a swapped-in actor without it is `X_TENANCY_CROSS_DENIED`, thrown
|
|
100
|
+
* through); outside, its mismatch is the answer `false`. Called only once the ambient org is known
|
|
101
|
+
* to differ, so outside a scope it always disagrees. The explicit `tenantId` half is never admitted:
|
|
102
|
+
* that row would be queued for one org and refused at its run, which reads no proof.
|
|
103
|
+
*/
|
|
104
|
+
function crossTenantAdmits(job: string, declared: string): boolean {
|
|
105
|
+
try {
|
|
106
|
+
assertRowTenant(job, ORG_COLUMN, 'enqueue', { [ORG_COLUMN]: declared });
|
|
107
|
+
return true;
|
|
108
|
+
} catch (error) {
|
|
109
|
+
if (isUltimateError(error) && error.code === 'X_TENANCY_ACTOR_MISMATCH') return false;
|
|
110
|
+
throw error;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The run's half, for a row queued before the enqueue refused it: the org the row was queued for
|
|
116
|
+
* against the org its payload declares. Terminal — the same row disagrees on every attempt.
|
|
117
|
+
*/
|
|
118
|
+
export function refuseRunTenant(
|
|
119
|
+
job: string,
|
|
120
|
+
declared: string | undefined,
|
|
121
|
+
queuedFor: string | undefined,
|
|
122
|
+
): void {
|
|
123
|
+
if (declared === undefined || queuedFor === undefined || queuedFor === declared) return;
|
|
124
|
+
throw new JobTenantMismatchError({
|
|
125
|
+
job,
|
|
126
|
+
reason: `was queued for org "${queuedFor}" while its payload names org "${declared}"`,
|
|
127
|
+
fix: `x jobs list --name ${renderFixShellArg(job, '<job>')} --state dead --json # this row is dead: re-enqueue it from an actor of the org its payload names, never x jobs retry it`,
|
|
128
|
+
});
|
|
129
|
+
}
|
package/src/type-pins.ts
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// Compile-time pins for what a step answers. Source, not a `.test.ts`, on purpose: `tsconfig.json`
|
|
2
|
+
// excludes `src/**/*.test.ts`, so `tsc -b` never reads a test file and a claim written there can
|
|
3
|
+
// never fail. Nothing here is called — a regression is a build error (axiom 3).
|
|
4
|
+
|
|
5
|
+
import { t } from '@ultimat3/schema';
|
|
6
|
+
import type { StepApi, StepOutput } from './steps';
|
|
7
|
+
|
|
8
|
+
/** Fails to compile when `T` is anything but `true`. The whole mechanism. */
|
|
9
|
+
type Assert<T extends true> = T;
|
|
10
|
+
|
|
11
|
+
/** Exact, both ways; assignability alone cannot tell a `readonly` or an optional member apart. */
|
|
12
|
+
type Same<A, B> =
|
|
13
|
+
(<X>() => X extends A ? 1 : 2) extends <X>() => X extends B ? 1 : 2 ? true : false;
|
|
14
|
+
|
|
15
|
+
type UserId = string & { readonly __brand: 'UserId' };
|
|
16
|
+
|
|
17
|
+
interface Row {
|
|
18
|
+
readonly id: UserId;
|
|
19
|
+
readonly at: Date;
|
|
20
|
+
readonly maybeAt: Date | null;
|
|
21
|
+
nick?: string;
|
|
22
|
+
gone: string | undefined;
|
|
23
|
+
load(): void;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type StepOutputPins = [
|
|
27
|
+
// A `Date` read off a step result is a compile error at the read.
|
|
28
|
+
Assert<Same<StepOutput<Date>, string>>,
|
|
29
|
+
Assert<Same<StepOutput<{ toJSON(): { wire: true } }>, { wire: true }>>,
|
|
30
|
+
// A brand survives: it is a string on the wire and in the type.
|
|
31
|
+
Assert<Same<StepOutput<UserId>, UserId>>,
|
|
32
|
+
// `undefined` at the top is the `null` the store holds.
|
|
33
|
+
Assert<Same<StepOutput<string | undefined>, string | null>>,
|
|
34
|
+
Assert<Same<StepOutput<undefined>, null>>,
|
|
35
|
+
// A readonly array stays one, and an element undefined is `null`, as `JSON.stringify` writes it.
|
|
36
|
+
Assert<Same<StepOutput<readonly Date[]>, readonly string[]>>,
|
|
37
|
+
Assert<Same<StepOutput<(number | undefined)[]>, (number | null)[]>>,
|
|
38
|
+
Assert<Same<StepOutput<readonly [Date, number]>, readonly [string, number]>>,
|
|
39
|
+
// `void` stays `void`: nobody reads it, and an `async () => {}` step must keep compiling.
|
|
40
|
+
Assert<Same<StepOutput<void>, void>>,
|
|
41
|
+
// `unknown` promises nothing and stays that way.
|
|
42
|
+
Assert<Same<StepOutput<unknown>, unknown>>,
|
|
43
|
+
Assert<Same<StepOutput<{ a: unknown }>, { a: unknown }>>,
|
|
44
|
+
// Members: readonly and optional kept, a function dropped, a `T | undefined` one may be absent.
|
|
45
|
+
Assert<
|
|
46
|
+
Same<
|
|
47
|
+
StepOutput<Row>,
|
|
48
|
+
{
|
|
49
|
+
readonly id: UserId;
|
|
50
|
+
readonly at: string;
|
|
51
|
+
readonly maybeAt: string | null;
|
|
52
|
+
nick?: string;
|
|
53
|
+
gone?: string;
|
|
54
|
+
}
|
|
55
|
+
>
|
|
56
|
+
>,
|
|
57
|
+
// A `Map` writes `{}`, and its `size` is not on the wire.
|
|
58
|
+
Assert<Same<StepOutput<Map<string, number>>, Record<string, never>>>,
|
|
59
|
+
Assert<Same<StepOutput<RegExp>, Record<string, never>>>,
|
|
60
|
+
// `message`, `name` and `stack` are not own enumerable; only what a subclass adds is written.
|
|
61
|
+
Assert<Same<keyof StepOutput<Error>, never>>,
|
|
62
|
+
Assert<Same<StepOutput<Error & { readonly status: number }>, { readonly status: number }>>,
|
|
63
|
+
// An object keyed by index, with no `length` on the wire.
|
|
64
|
+
Assert<Same<StepOutput<Uint8Array>, Record<string, number>>>,
|
|
65
|
+
// `bigint` never arrives: X_STEP_OUTPUT_UNSERIALIZABLE.
|
|
66
|
+
Assert<Same<StepOutput<bigint>, never>>,
|
|
67
|
+
];
|
|
68
|
+
|
|
69
|
+
const Approval = t.object({ amount: t.number });
|
|
70
|
+
|
|
71
|
+
/** Never called: the point is the types of what it reads. */
|
|
72
|
+
export async function waitForEventPins(step: StepApi): Promise<void> {
|
|
73
|
+
const answer = await step.waitForEvent('approval', 'invoice.approved', { schema: Approval });
|
|
74
|
+
const pinned: Assert<Same<typeof answer, { amount: number } | undefined>> = true;
|
|
75
|
+
void pinned;
|
|
76
|
+
|
|
77
|
+
// @ts-expect-error `schema` is required: a waiter's own type would be a cast, not a fact.
|
|
78
|
+
await step.waitForEvent('approval', 'invoice.approved', { timeout: '1h' });
|
|
79
|
+
// @ts-expect-error the options are required, because `schema` is.
|
|
80
|
+
await step.waitForEvent('approval', 'invoice.approved');
|
|
81
|
+
// @ts-expect-error `<T>` is gone: the schema is what decides the type.
|
|
82
|
+
await step.waitForEvent<{ amount: number }>('approval', 'invoice.approved', { schema: Approval });
|
|
83
|
+
|
|
84
|
+
const stamp = await step.run('stamp', () => new Date(0));
|
|
85
|
+
// @ts-expect-error a `Date` is a string by the time a step answers it; `new Date(stamp)`.
|
|
86
|
+
void stamp.getTime();
|
|
87
|
+
const row = await step.run('load', () => Promise.resolve<{ id: string } | undefined>(undefined));
|
|
88
|
+
const missed: Assert<Same<typeof row, { id: string } | null>> = true;
|
|
89
|
+
void missed;
|
|
90
|
+
}
|
package/src/webhook-errors.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// The eight `X_WEBHOOK_*` codes an outbound delivery can end on, apart from `errors.ts` for the
|
|
2
|
-
// reason `backfill-errors.ts` is: one file, one job, and `
|
|
2
|
+
// reason `backfill-errors.ts` is: one file, one job, and `error-codes.ts` holds the registry. The codes,
|
|
3
3
|
// the titles and the single `registerErrorCodes()` call stay there; only the classes live here.
|
|
4
4
|
//
|
|
5
5
|
// THE RULE THIS FILE EXISTS TO KEEP: no secret and no signature reaches a `cause`, a `fix` or a
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
// The two codes an inbound `webhookReceiver()` owns: a declaration that cannot be served, and a
|
|
2
|
+
// verified delivery the queue would not take. Classes only; `error-codes.ts` registers the codes.
|
|
3
|
+
// No body, no header and no secret ever reaches a cause: a refusal is read by the log store.
|
|
4
|
+
|
|
5
|
+
import { renderThrowable, UltimateError } from '@ultimat3/core';
|
|
6
|
+
|
|
7
|
+
/** Seconds a vendor is told to wait before redelivering after the queue refused. */
|
|
8
|
+
export const WEBHOOK_RECEIVER_RETRY_AFTER_SECONDS = 30;
|
|
9
|
+
|
|
10
|
+
/** Refused at declaration, while modules load: no request is ever answered with it. */
|
|
11
|
+
export class WebhookReceiverInvalidError extends UltimateError {
|
|
12
|
+
constructor(input: { receiver: string; field: string; reason: string; fix: string }) {
|
|
13
|
+
super({
|
|
14
|
+
code: 'X_WEBHOOK_RECEIVER_INVALID',
|
|
15
|
+
cause: `webhookReceiver("${input.receiver}") ${input.field} ${input.reason}`,
|
|
16
|
+
fix: `webhookReceiver({ name: '${input.receiver}', ${input.fix}, … })`,
|
|
17
|
+
meta: { receiver: input.receiver, field: input.field },
|
|
18
|
+
});
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** What `verify` may answer as an event id: 1-256 printable ASCII characters, no space. */
|
|
23
|
+
export const WEBHOOK_EVENT_ID_PATTERN = /^[\x21-\x7e]{1,256}$/;
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* `verify` answered an id no job may be keyed by. A 4xx, never a 5xx: a vendor retries a 5xx
|
|
27
|
+
* forever and the answer would never change. The id itself is never echoed, only its length: an
|
|
28
|
+
* 8,000-character header is not a value to put in a log line or a response.
|
|
29
|
+
*/
|
|
30
|
+
export class WebhookEventIdInvalidError extends UltimateError {
|
|
31
|
+
constructor(input: { receiver: string; length: number }) {
|
|
32
|
+
super({
|
|
33
|
+
code: 'X_WEBHOOK_EVENT_ID_INVALID',
|
|
34
|
+
cause: `webhookReceiver("${input.receiver}") verify answered an eventId of ${input.length} characters that is not 1-256 printable ASCII characters`,
|
|
35
|
+
fix: `webhookReceiver({ name: '${input.receiver}', verify: (inbound) => ({ eventId }) }) — read eventId out of the signed bytes (verifyBodyHmac(...).text), 1-256 printable ASCII characters`,
|
|
36
|
+
meta: { receiver: input.receiver, length: input.length },
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* The enqueue failed, so the delivery is NOT acknowledged: 503 with `Retry-After` through the error
|
|
43
|
+
* map (`meta.retryAfterSeconds`), and the vendor redelivers. The inner failure goes through core's
|
|
44
|
+
* total renderer — it is a caught `unknown`.
|
|
45
|
+
*/
|
|
46
|
+
export class WebhookReceiverUnavailableError extends UltimateError {
|
|
47
|
+
constructor(input: { receiver: string; eventId: string; error: unknown }) {
|
|
48
|
+
super({
|
|
49
|
+
code: 'X_WEBHOOK_RECEIVER_UNAVAILABLE',
|
|
50
|
+
cause: `webhookReceiver("${input.receiver}") verified event ${input.eventId} and could not queue it: ${renderThrowable(input.error)}`,
|
|
51
|
+
fix: 'x doctor --json # then check DATABASE_URL on the web role; the vendor redelivers once the queue answers',
|
|
52
|
+
meta: {
|
|
53
|
+
receiver: input.receiver,
|
|
54
|
+
eventId: input.eventId,
|
|
55
|
+
retryAfterSeconds: WEBHOOK_RECEIVER_RETRY_AFTER_SECONDS,
|
|
56
|
+
},
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// The door `webhookReceiver()` mounts: read the bytes under `maxBytes`, ask `verify`, enqueue the
|
|
2
|
+
// payload it verified durably, and only then answer 200 `{ ok, eventId, deduped }`. Nothing is
|
|
3
|
+
// parsed before `verify` and no content type is required. An enqueue failure is 503 with
|
|
4
|
+
// `Retry-After`, so the vendor redelivers: an ack never precedes the durable row.
|
|
5
|
+
|
|
6
|
+
import { readWithinLimit } from '@ultimat3/core';
|
|
7
|
+
import type { Route, RouteHandler, RouteMeta, UltimateRequest } from '@ultimat3/http';
|
|
8
|
+
import { HttpError, jsonResponse } from '@ultimat3/http';
|
|
9
|
+
import type { JsonValue } from '@ultimat3/schema';
|
|
10
|
+
import type { JobHandle } from './job';
|
|
11
|
+
import {
|
|
12
|
+
WEBHOOK_EVENT_ID_PATTERN,
|
|
13
|
+
WebhookEventIdInvalidError,
|
|
14
|
+
WebhookReceiverUnavailableError,
|
|
15
|
+
} from './webhook-receiver-errors';
|
|
16
|
+
import type {
|
|
17
|
+
WebhookInbound,
|
|
18
|
+
WebhookReceiverDefinition,
|
|
19
|
+
WebhookReceiverInput,
|
|
20
|
+
} from './webhook-receiver-types';
|
|
21
|
+
|
|
22
|
+
interface RouteSpec<P extends JsonValue> {
|
|
23
|
+
readonly definition: WebhookReceiverDefinition<P>;
|
|
24
|
+
readonly maxBytes: number;
|
|
25
|
+
readonly handle: JobHandle<WebhookReceiverInput>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async function inboundOf(
|
|
29
|
+
request: UltimateRequest,
|
|
30
|
+
name: string,
|
|
31
|
+
maxBytes: number,
|
|
32
|
+
): Promise<WebhookInbound> {
|
|
33
|
+
// Core's counting reader, NOT `bodyBytes()`: that one caps at `http.bodyLimitBytes`, and a
|
|
34
|
+
// receiver's own `maxBytes` is the number this route answers to, in either direction.
|
|
35
|
+
const read =
|
|
36
|
+
request.method === 'GET'
|
|
37
|
+
? { bytes: new Uint8Array(0) }
|
|
38
|
+
: await readWithinLimit(request.raw.body, maxBytes);
|
|
39
|
+
if ('over' in read) {
|
|
40
|
+
throw new HttpError({
|
|
41
|
+
code: 'X_BODY_INVALID',
|
|
42
|
+
cause: `${request.pathname} body rejected: body is at least ${read.over} bytes, limit is ${maxBytes}`,
|
|
43
|
+
fix: `webhookReceiver({ name: '${name}', maxBytes: ${maxBytes * 2}, … }) — raise maxBytes to the largest delivery the vendor sends`,
|
|
44
|
+
meta: { maxBytes },
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
const { bytes } = read;
|
|
48
|
+
const raw = request.raw;
|
|
49
|
+
return {
|
|
50
|
+
bytes,
|
|
51
|
+
headers: raw.headers,
|
|
52
|
+
params: request.params,
|
|
53
|
+
url: request.url,
|
|
54
|
+
request: () =>
|
|
55
|
+
new Request(raw.url, {
|
|
56
|
+
method: raw.method,
|
|
57
|
+
headers: raw.headers,
|
|
58
|
+
...(request.method === 'GET' ? {} : { body: new Uint8Array(bytes) }),
|
|
59
|
+
}),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function receiverRoutes<P extends JsonValue>(spec: RouteSpec<P>): readonly Route[] {
|
|
64
|
+
const { definition, maxBytes, handle } = spec;
|
|
65
|
+
const rate = definition.rateLimit;
|
|
66
|
+
const meta: RouteMeta = {
|
|
67
|
+
name: `webhook.${definition.name}`,
|
|
68
|
+
// A vendor carries no session: the credential is the signature `verify` checks.
|
|
69
|
+
auth: 'public',
|
|
70
|
+
...(rate === undefined
|
|
71
|
+
? {}
|
|
72
|
+
: typeof rate === 'string'
|
|
73
|
+
? { rateLimit: rate }
|
|
74
|
+
: { rateLimit: `webhook.${definition.name}`, rateLimitBucket: rate }),
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const receive: RouteHandler = async (request) => {
|
|
78
|
+
const inbound = await inboundOf(request, definition.name, maxBytes);
|
|
79
|
+
const verified = await definition.verify(inbound);
|
|
80
|
+
if ('answer' in verified) return verified.answer;
|
|
81
|
+
// Bounded before it is a key, a response field or a cause: `typeof` first, because `verify`
|
|
82
|
+
// is app code and JS callers return what they like.
|
|
83
|
+
const eventId: unknown = verified.eventId;
|
|
84
|
+
if (typeof eventId !== 'string' || !WEBHOOK_EVENT_ID_PATTERN.test(eventId)) {
|
|
85
|
+
throw new WebhookEventIdInvalidError({
|
|
86
|
+
receiver: definition.name,
|
|
87
|
+
length: typeof eventId === 'string' ? eventId.length : 0,
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
const row: WebhookReceiverInput = {
|
|
91
|
+
eventId,
|
|
92
|
+
...(verified.orgId === undefined ? {} : { orgId: verified.orgId }),
|
|
93
|
+
...(verified.topic === undefined ? {} : { topic: verified.topic }),
|
|
94
|
+
payload: verified.payload,
|
|
95
|
+
};
|
|
96
|
+
// Outside the try: an org receiver whose `verify` named no org is the app's to fix, and a 503
|
|
97
|
+
// would have the vendor retry it forever.
|
|
98
|
+
const tenantId = handle.tenantFor(row);
|
|
99
|
+
let enqueued: Awaited<ReturnType<typeof handle.enqueue>>;
|
|
100
|
+
try {
|
|
101
|
+
// `outbox: false`: the row must be durable BEFORE the ack, never staged behind a commit.
|
|
102
|
+
enqueued = await handle.enqueue(row, {
|
|
103
|
+
outbox: false,
|
|
104
|
+
...(tenantId === undefined ? {} : { tenantId }),
|
|
105
|
+
});
|
|
106
|
+
} catch (error) {
|
|
107
|
+
throw new WebhookReceiverUnavailableError({
|
|
108
|
+
receiver: definition.name,
|
|
109
|
+
eventId: row.eventId,
|
|
110
|
+
error,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
return jsonResponse({ ok: true, eventId: row.eventId, deduped: enqueued.deduped });
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
const routes: Route[] = [{ method: 'POST', path: definition.path, handler: receive, meta }];
|
|
117
|
+
const handshake = definition.handshake;
|
|
118
|
+
if (handshake !== undefined) {
|
|
119
|
+
routes.push({
|
|
120
|
+
method: 'GET',
|
|
121
|
+
path: definition.path,
|
|
122
|
+
meta: { ...meta, name: `webhook.${definition.name}.handshake` },
|
|
123
|
+
handler: async (request) => handshake(await inboundOf(request, definition.name, maxBytes)),
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
return routes;
|
|
127
|
+
}
|