@sanoma-ai/workflows 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -15
- package/dist/apply.d.ts +2 -1
- package/dist/apply.js +29 -7
- package/dist/apply.js.map +1 -1
- package/dist/approvals.d.ts +27 -2
- package/dist/approvals.js +73 -9
- package/dist/approvals.js.map +1 -1
- package/dist/call.d.ts +3 -2
- package/dist/call.js +38 -30
- package/dist/call.js.map +1 -1
- package/dist/cedar.js +24 -9
- package/dist/cedar.js.map +1 -1
- package/dist/change.d.ts +219 -0
- package/dist/change.js +150 -0
- package/dist/change.js.map +1 -0
- package/dist/chats.d.ts +43 -0
- package/dist/chats.js +48 -0
- package/dist/chats.js.map +1 -0
- package/dist/client.d.ts +52 -6
- package/dist/client.js +88 -47
- package/dist/client.js.map +1 -1
- package/dist/config.d.ts +47 -5
- package/dist/config.js +71 -7
- package/dist/config.js.map +1 -1
- package/dist/database.d.ts +83 -0
- package/dist/database.js +92 -0
- package/dist/database.js.map +1 -0
- package/dist/db.d.ts +31 -1
- package/dist/db.js +89 -7
- package/dist/db.js.map +1 -1
- package/dist/deadlines.d.ts +88 -0
- package/dist/deadlines.js +171 -0
- package/dist/deadlines.js.map +1 -0
- package/dist/define.d.ts +25 -4
- package/dist/define.js +40 -0
- package/dist/define.js.map +1 -1
- package/dist/demo.d.ts +28 -0
- package/dist/demo.js +72 -0
- package/dist/demo.js.map +1 -0
- package/dist/describe.d.ts +6 -3
- package/dist/describe.js +11 -7
- package/dist/describe.js.map +1 -1
- package/dist/errors.d.ts +10 -1
- package/dist/errors.js +13 -0
- package/dist/errors.js.map +1 -1
- package/dist/fake.d.ts +8 -0
- package/dist/fake.js +11 -1
- package/dist/fake.js.map +1 -1
- package/dist/grid.d.ts +34 -0
- package/dist/grid.js +62 -0
- package/dist/grid.js.map +1 -0
- package/dist/host.d.ts +13 -2
- package/dist/host.js +2 -2
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +16 -8
- package/dist/index.js +12 -5
- package/dist/index.js.map +1 -1
- package/dist/ledger.d.ts +11 -2
- package/dist/ledger.js.map +1 -1
- package/dist/lint.js +1 -1
- package/dist/lint.js.map +1 -1
- package/dist/op.d.ts +14 -2
- package/dist/op.js.map +1 -1
- package/dist/policy.d.ts +7 -1
- package/dist/policy.js +2 -1
- package/dist/policy.js.map +1 -1
- package/dist/resource.d.ts +13 -1
- package/dist/resource.js +9 -1
- package/dist/resource.js.map +1 -1
- package/dist/resources.d.ts +2 -0
- package/dist/resources.js +10 -1
- package/dist/resources.js.map +1 -1
- package/dist/run.d.ts +5 -0
- package/dist/run.js.map +1 -1
- package/dist/sandbox.d.ts +38 -10
- package/dist/sandbox.js +66 -35
- package/dist/sandbox.js.map +1 -1
- package/dist/scenario.d.ts +18 -1
- package/dist/scenario.js +77 -27
- package/dist/scenario.js.map +1 -1
- package/dist/scenarios.d.ts +10 -1
- package/dist/scenarios.js +19 -4
- package/dist/scenarios.js.map +1 -1
- package/dist/schedules.d.ts +66 -0
- package/dist/schedules.js +158 -0
- package/dist/schedules.js.map +1 -0
- package/dist/schema.d.ts +6 -0
- package/dist/schema.js +13 -3
- package/dist/schema.js.map +1 -1
- package/dist/shared.d.ts +38 -2
- package/dist/shared.js +50 -4
- package/dist/shared.js.map +1 -1
- package/dist/sites.d.ts +38 -1
- package/dist/sites.js +153 -26
- package/dist/sites.js.map +1 -1
- package/dist/source.d.ts +61 -2
- package/dist/source.js.map +1 -1
- package/dist/supervisor.d.ts +38 -0
- package/dist/supervisor.js +77 -0
- package/dist/supervisor.js.map +1 -0
- package/dist/version.d.ts +3 -1
- package/dist/version.js +3 -1
- package/dist/version.js.map +1 -1
- package/dist/wakeup-service.d.ts +83 -0
- package/dist/wakeup-service.js +421 -0
- package/dist/wakeup-service.js.map +1 -0
- package/dist/wakeup.d.ts +84 -0
- package/dist/wakeup.js +87 -0
- package/dist/wakeup.js.map +1 -0
- package/dist/worker.d.ts +39 -1
- package/dist/worker.js +130 -39
- package/dist/worker.js.map +1 -1
- package/oxlint.json +2 -1
- package/package.json +8 -2
- package/src/apply.ts +25 -6
- package/src/approvals.ts +89 -16
- package/src/call.ts +40 -29
- package/src/cedar.ts +24 -6
- package/src/change.ts +222 -0
- package/src/chats.ts +114 -0
- package/src/client.ts +145 -55
- package/src/config.ts +110 -13
- package/src/database.ts +152 -0
- package/src/db.ts +108 -9
- package/src/deadlines.ts +234 -0
- package/src/define.ts +71 -4
- package/src/demo.ts +87 -0
- package/src/describe.ts +19 -7
- package/src/errors.ts +14 -0
- package/src/fake.ts +12 -0
- package/src/grid.ts +100 -0
- package/src/host.ts +13 -4
- package/src/index.ts +62 -3
- package/src/ledger.ts +15 -3
- package/src/lint.ts +1 -1
- package/src/op.ts +15 -2
- package/src/policy.ts +7 -1
- package/src/resource.ts +22 -3
- package/src/resources.ts +12 -1
- package/src/run.ts +5 -0
- package/src/sandbox.ts +97 -37
- package/src/scenario.ts +97 -28
- package/src/scenarios.ts +21 -5
- package/src/schedules.ts +206 -0
- package/src/schema.ts +16 -3
- package/src/shared.ts +62 -4
- package/src/sites.ts +198 -33
- package/src/source.ts +58 -3
- package/src/supervisor.ts +107 -0
- package/src/version.ts +3 -1
- package/src/wakeup-service.ts +511 -0
- package/src/wakeup.ts +150 -0
- package/src/worker.ts +181 -44
package/src/config.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
|
5
5
|
import { type ApplyDeclared, applyWorkflow } from "./apply.ts";
|
|
6
6
|
import { applySteps } from "./apply-steps.ts";
|
|
7
7
|
import { callWorkflow } from "./call-workflow.ts";
|
|
8
|
+
import { changeDriver, changeOps, reservedFor } from "./change.ts";
|
|
8
9
|
import {
|
|
9
10
|
type Collection,
|
|
10
11
|
collectionsConnector,
|
|
@@ -34,11 +35,15 @@ import {
|
|
|
34
35
|
} from "./op.ts";
|
|
35
36
|
import { type CedarPolicySource, policyAt } from "./cedar.ts";
|
|
36
37
|
import type { Policy } from "./policy.ts";
|
|
37
|
-
import {
|
|
38
|
+
import { type Resource, resourceTypesOf, withoutWriteOnly } from "./resource.ts";
|
|
38
39
|
import { APPLY_WORKFLOW, BUILT_IN_WORKFLOWS, CALL_WORKFLOW, DATA_VENDOR } from "./shared.ts";
|
|
39
40
|
import { sourceDriver, sourceOps } from "./source.ts";
|
|
40
41
|
import type { Steps } from "./steps.ts";
|
|
42
|
+
import { checkSchedule } from "./schedules.ts";
|
|
41
43
|
import { computeVersion } from "./version.ts";
|
|
44
|
+
import type { DeclaredResource } from "./resources.ts";
|
|
45
|
+
import { checkWakeupOps, type DeclaredWakeup } from "./wakeup.ts";
|
|
46
|
+
import { checkDatabaseOps } from "./database.ts";
|
|
42
47
|
|
|
43
48
|
/**
|
|
44
49
|
* What a worker runs, on which connectors: the argument to `startWorker`.
|
|
@@ -65,7 +70,7 @@ export interface SanomaConfig {
|
|
|
65
70
|
* only, for tests.
|
|
66
71
|
*/
|
|
67
72
|
ledger?: LedgerStore;
|
|
68
|
-
/** Postgres for the runtime. Defaults to `SANOMA_DATABASE_URL`, then
|
|
73
|
+
/** Postgres for the runtime. Defaults to `SANOMA_DATABASE_URL`, then `DEFAULT_DATABASE_URL`. */
|
|
69
74
|
databaseUrl?: string;
|
|
70
75
|
/** The file that defined the config, absolute, when known. Set by `defineConfig` from its call site. */
|
|
71
76
|
file?: string;
|
|
@@ -92,6 +97,12 @@ export interface SanomaConfig {
|
|
|
92
97
|
* records in the config's database, and in memory for a sandbox run.
|
|
93
98
|
*/
|
|
94
99
|
collections?: readonly Collection[];
|
|
100
|
+
/**
|
|
101
|
+
* Run the wakeup service inside this instance (`sanoma serve`), for other instances' wakeups
|
|
102
|
+
* to register with. An instance that runs it must not stop, so `startWorker` refuses a config
|
|
103
|
+
* that sets it and declares a wakeup of its own.
|
|
104
|
+
*/
|
|
105
|
+
wakeup?: true;
|
|
95
106
|
}
|
|
96
107
|
|
|
97
108
|
/** A config, checked, with everything a worker, client or app derives from it. */
|
|
@@ -107,8 +118,9 @@ export interface ResolvedConfig {
|
|
|
107
118
|
/** The DBOS queue runs are started on: `sanoma:<appName>`. */
|
|
108
119
|
queueName: string;
|
|
109
120
|
/**
|
|
110
|
-
* The operations the connectors declare, by id, the built-in `sanoma.
|
|
111
|
-
* declares a host type, and the collections'
|
|
121
|
+
* The operations the connectors declare, by id, the built-in `sanoma.change.*` (`changeOps`, in
|
|
122
|
+
* every config), `sanoma.source.resolve` when one declares a host type, and the collections'
|
|
123
|
+
* (`data.<name>.<op>`).
|
|
112
124
|
*/
|
|
113
125
|
ops: Map<string, Op>;
|
|
114
126
|
/** The drivers' functions, by operation id, each guarded by its driver's `env` (`indexDrivers`). */
|
|
@@ -118,10 +130,22 @@ export interface ResolvedConfig {
|
|
|
118
130
|
* `connectors` order: what `drivers` indexes and the credentials are read from (`credentialsOf`).
|
|
119
131
|
*/
|
|
120
132
|
driversByVendor: Map<string, Driver>;
|
|
121
|
-
/**
|
|
133
|
+
/**
|
|
134
|
+
* The makers of the fakes the connectors attach, by vendor id in `connectors` order: each
|
|
135
|
+
* sandbox run makes its own. Resolved with `fakes: "instance"`, each gives its vendor's
|
|
136
|
+
* instance fake (`instanceFakes`).
|
|
137
|
+
*/
|
|
122
138
|
fakes: Map<string, () => Fake<any, any>>;
|
|
139
|
+
/**
|
|
140
|
+
* Resolved with `fakes: "instance"` (a demo): each vendor's one fake, by vendor id, made once
|
|
141
|
+
* for this resolution. Each is its vendor's driver (`driversByVendor`), so live runs, sandbox
|
|
142
|
+
* runs and direct calls all call it, and the world's Givens seed it (`applyGivens`).
|
|
143
|
+
*/
|
|
144
|
+
instanceFakes?: ReadonlyMap<string, Fake<any, any>>;
|
|
123
145
|
/** The makers of the login halves the connectors attach, by vendor id in `connectors` order: the app makes one per public site that names it. */
|
|
124
146
|
logins: ReadonlyMap<string, NonNullable<ConnectorSetup["login"]>>;
|
|
147
|
+
/** The makers of the model halves the connectors attach, by vendor id in `connectors` order: the app makes each once for its model registry. */
|
|
148
|
+
models: ReadonlyMap<string, NonNullable<ConnectorSetup["model"]>>;
|
|
125
149
|
/** `scenarios/` in `root`, as a `file:` URL, when that directory exists. */
|
|
126
150
|
scenarios?: URL;
|
|
127
151
|
/** The resource types the connectors declare, by `<vendor>.<type>`: what the drift workflow checks and scenarios seed. */
|
|
@@ -160,6 +184,11 @@ export interface ResolvedConfig {
|
|
|
160
184
|
root?: string;
|
|
161
185
|
}
|
|
162
186
|
|
|
187
|
+
/**
|
|
188
|
+
* The database when neither the config nor `SANOMA_DATABASE_URL` names one: the dev Postgres
|
|
189
|
+
* `pnpm db:up` starts in this checkout. The `sanoma` launcher never falls back to it: with no
|
|
190
|
+
* URL named it starts the company's own Postgres and sets `SANOMA_DATABASE_URL` to it.
|
|
191
|
+
*/
|
|
163
192
|
export const DEFAULT_DATABASE_URL = "postgresql://postgres:dbos@localhost:5433/sanoma";
|
|
164
193
|
|
|
165
194
|
/** Returns the config, with the file it is called from as its `file`: its directory is the config's `root`. */
|
|
@@ -198,6 +227,17 @@ export function rootOf(config: Pick<SanomaConfig, "root" | "file">): string | un
|
|
|
198
227
|
: undefined;
|
|
199
228
|
}
|
|
200
229
|
|
|
230
|
+
/** What `resolveConfig` is given besides the config. */
|
|
231
|
+
export interface ResolveOptions {
|
|
232
|
+
/**
|
|
233
|
+
* `"instance"`: run the whole instance on the connectors' fakes, as a demo does: each vendor's
|
|
234
|
+
* driver is one fake made for this resolution (`instanceFakes`), which every run and call
|
|
235
|
+
* shares. The collections stay on the config's database. Refused for a vendor whose connector
|
|
236
|
+
* attaches a driver and no fake, which would otherwise run live.
|
|
237
|
+
*/
|
|
238
|
+
fakes?: "instance";
|
|
239
|
+
}
|
|
240
|
+
|
|
201
241
|
/**
|
|
202
242
|
* Checks a config and derives what the runtime needs from it: the drivers and fakes from the
|
|
203
243
|
* connectors' setups (`withSetup`), the collections' operations, driver and fake, and from
|
|
@@ -208,7 +248,7 @@ export function rootOf(config: Pick<SanomaConfig, "root" | "file">): string | un
|
|
|
208
248
|
* `root`. Each entry point resolves once: the worker and the client when they start, the app at
|
|
209
249
|
* `startApp`, the `sanoma` bin when it loads the config.
|
|
210
250
|
*/
|
|
211
|
-
export function resolveConfig(config: SanomaConfig): ResolvedConfig {
|
|
251
|
+
export function resolveConfig(config: SanomaConfig, options: ResolveOptions = {}): ResolvedConfig {
|
|
212
252
|
if (!Array.isArray(config.connectors)) {
|
|
213
253
|
throw new Error("The config needs `connectors`: the connectors its workflows call, as their packages export them");
|
|
214
254
|
}
|
|
@@ -237,21 +277,29 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
|
|
|
237
277
|
const databaseUrl = resolveDatabaseUrl(config);
|
|
238
278
|
const declared = [...collections.values()];
|
|
239
279
|
const dataStore = declared.length ? lazyPool(databaseUrl, "data") : undefined;
|
|
280
|
+
// Built-in, in every config: a change's operations, which the template's policy names (`sanoma.change.merge`).
|
|
240
281
|
const ops = indexConnectors([
|
|
241
282
|
...config.connectors,
|
|
283
|
+
changeOps,
|
|
242
284
|
...(hosts ? [sourceOps] : []),
|
|
243
285
|
...(dataStore ? [collectionsConnector(declared)] : []),
|
|
244
286
|
]);
|
|
245
|
-
|
|
287
|
+
checkWakeupOps(types, ops);
|
|
288
|
+
checkDatabaseOps(types, ops);
|
|
289
|
+
const fakes = makersOf(config.connectors, "fake");
|
|
290
|
+
const instanceFakes = options.fakes === "instance" ? instanceFakesOf(config.connectors, fakes) : undefined;
|
|
291
|
+
const driversByVendor = instanceFakes
|
|
292
|
+
? new Map([...instanceFakes].map(([vendor, fake]) => [vendor, fake.driver]))
|
|
293
|
+
: new Map([...makersOf(config.connectors, "driver")].map(([vendor, make]) => [vendor, make()]));
|
|
246
294
|
const drivers = indexDrivers(
|
|
247
295
|
[
|
|
248
296
|
...driversByVendor.values(),
|
|
297
|
+
changeDriver(),
|
|
249
298
|
...(hosts ? [sourceDriver] : []),
|
|
250
299
|
...(dataStore ? [collectionsDriver(declared, dataStore)] : []),
|
|
251
300
|
],
|
|
252
301
|
ops,
|
|
253
302
|
);
|
|
254
|
-
const fakes = makersOf(config.connectors, "fake");
|
|
255
303
|
if (dataStore) fakes.set(DATA_VENDOR, () => collectionsFake(declared));
|
|
256
304
|
const names = new Map<string, WorkflowDefinition<any, any>>();
|
|
257
305
|
for (const wf of config.workflows) {
|
|
@@ -259,6 +307,7 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
|
|
|
259
307
|
throw new Error(`A workflow is named "${wf.name}", the name of a built-in workflow; name it otherwise`);
|
|
260
308
|
}
|
|
261
309
|
checkUses(wf, ops, drivers, types);
|
|
310
|
+
checkSchedule(wf);
|
|
262
311
|
const other = names.get(wf.name);
|
|
263
312
|
if (other && other !== wf) {
|
|
264
313
|
throw new Error(
|
|
@@ -293,7 +342,9 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
|
|
|
293
342
|
drivers,
|
|
294
343
|
driversByVendor,
|
|
295
344
|
fakes,
|
|
345
|
+
...(instanceFakes && { instanceFakes }),
|
|
296
346
|
logins: makersOf(config.connectors, "login"),
|
|
347
|
+
models: makersOf(config.connectors, "model"),
|
|
297
348
|
...(scenarios && { scenarios }),
|
|
298
349
|
types,
|
|
299
350
|
steps: [...new Set([...config.connectors.flatMap((c) => setupOf(c).steps ?? []), ...(config.steps ?? [])])],
|
|
@@ -360,6 +411,25 @@ function makersOf<K extends keyof ConnectorSetup>(
|
|
|
360
411
|
return makers;
|
|
361
412
|
}
|
|
362
413
|
|
|
414
|
+
/**
|
|
415
|
+
* Each vendor's one fake, made now, for `resolveConfig`'s `fakes: "instance"`; `fakes` (the
|
|
416
|
+
* makers) then give these. Refuses a connector that attaches a driver and no fake.
|
|
417
|
+
*/
|
|
418
|
+
function instanceFakesOf(
|
|
419
|
+
connectors: readonly Connector<any, any>[],
|
|
420
|
+
fakes: Map<string, () => Fake<any, any>>,
|
|
421
|
+
): Map<string, Fake<any, any>> {
|
|
422
|
+
const live = [...makersOf(connectors, "driver").keys()].filter((vendor) => !fakes.has(vendor));
|
|
423
|
+
if (live.length) {
|
|
424
|
+
throw new Error(
|
|
425
|
+
`The instance runs on fakes, but the connector of ${live.join(", ")} attaches no fake, so it would run live; attach one (\`withSetup\`) or leave it out`,
|
|
426
|
+
);
|
|
427
|
+
}
|
|
428
|
+
const made = new Map([...fakes].map(([vendor, make]) => [vendor, make()]));
|
|
429
|
+
for (const [vendor, fake] of made) fakes.set(vendor, () => fake);
|
|
430
|
+
return made;
|
|
431
|
+
}
|
|
432
|
+
|
|
363
433
|
/**
|
|
364
434
|
* The `name` of `<root>/package.json`, kept whole (`@acme/ops`), when that file exists and names
|
|
365
435
|
* one; else "sanoma", as for a config with no `root`. Never a `package.json` above the root: a
|
|
@@ -385,6 +455,33 @@ function scenariosAt(root: string | undefined): URL | undefined {
|
|
|
385
455
|
return pathToFileURL(`${dir}/`);
|
|
386
456
|
}
|
|
387
457
|
|
|
458
|
+
/**
|
|
459
|
+
* The one wakeup the config's data files declare, or undefined: with none, and with one the reader
|
|
460
|
+
* leaves out for a problem (its declaration's or its host's; several wakeups are each one).
|
|
461
|
+
* Undefined means the instance has no wakeup, so it must not stop. Server-side: it reads the data
|
|
462
|
+
* files and the lockfile, only when a connector declares a wakeup type.
|
|
463
|
+
*/
|
|
464
|
+
export async function declaredWakeup(config: SanomaConfig): Promise<DeclaredWakeup | undefined> {
|
|
465
|
+
const connectors = connectorsOf(config);
|
|
466
|
+
const types = resourceTypesOf(connectors);
|
|
467
|
+
if (![...types.values()].some((type) => type.wakeup)) return undefined;
|
|
468
|
+
// Loaded when it is needed, not with the worker: the reader parses with oxc-parser.
|
|
469
|
+
const { readDataFiles } = await import("./resources.ts");
|
|
470
|
+
const { resources } = readDataFiles(rootOf(config), connectors, types);
|
|
471
|
+
const typeOf = (r: DeclaredResource) => types.get(`${r.vendor}.${r.type}`)!;
|
|
472
|
+
const declared = resources.find((r) => typeOf(r).wakeup);
|
|
473
|
+
if (!declared) return undefined;
|
|
474
|
+
const type = typeOf(declared);
|
|
475
|
+
const host = resources.find((r) => r.id === declared.refs[type.wakeup!.inputs.host]);
|
|
476
|
+
return {
|
|
477
|
+
id: declared.id,
|
|
478
|
+
type,
|
|
479
|
+
declared,
|
|
480
|
+
...(declared.binding !== undefined && { binding: declared.binding }),
|
|
481
|
+
...(host && { host: { id: host.id, type: typeOf(host) } }),
|
|
482
|
+
};
|
|
483
|
+
}
|
|
484
|
+
|
|
388
485
|
/**
|
|
389
486
|
* The data files under `root` and the lockfile beside them, in one read, as a drift or apply run
|
|
390
487
|
* records them: each resource with its binding, without its write-only values (a secret marker
|
|
@@ -478,7 +575,8 @@ function guarded(vendor: string, env: DriverEnv, fn: DriverFn): DriverFn {
|
|
|
478
575
|
* Every operation a workflow uses must be one the config's connectors declare, with a driver.
|
|
479
576
|
* The workflow normally imports the same connector object; a copy is accepted only if it
|
|
480
577
|
* declares the same effect and retry setting, and the worker's declaration is used either way.
|
|
481
|
-
* None may
|
|
578
|
+
* None may use an operation a built-in workflow alone calls (`reservedFor`): a declared resource's
|
|
579
|
+
* change, which only apply makes, or a change's step, which only the change workflow takes.
|
|
482
580
|
*/
|
|
483
581
|
function checkUses(
|
|
484
582
|
wf: WorkflowDefinition<any, any>,
|
|
@@ -489,11 +587,10 @@ function checkUses(
|
|
|
489
587
|
const problems: string[] = [];
|
|
490
588
|
for (const op of (wf.uses as readonly Use[]).filter(isOp)) {
|
|
491
589
|
const known = ops.get(op.id);
|
|
590
|
+
const reserved = known && reservedFor(known, types);
|
|
492
591
|
if (!known) problems.push(`${op.id} is not declared by any connector in \`connectors\``);
|
|
493
|
-
else if (
|
|
494
|
-
problems.push(
|
|
495
|
-
`${op.id} changes a declared resource, which only ${APPLY_WORKFLOW} does: change the data file instead`,
|
|
496
|
-
);
|
|
592
|
+
else if (reserved) {
|
|
593
|
+
problems.push(`${op.id} ${reserved.does}, which only ${reserved.workflow} does: ${reserved.instead} instead`);
|
|
497
594
|
} else if (known !== op && (known.effect !== op.effect || known.idempotent !== op.idempotent)) {
|
|
498
595
|
problems.push(
|
|
499
596
|
`${op.id} is declared with effect "${op.effect}"${op.idempotent ? " (idempotent)" : ""}, ` +
|
package/src/database.ts
ADDED
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import type { z } from "zod";
|
|
2
|
+
import { hasPath } from "./host.ts";
|
|
3
|
+
import { DriverError, type Op } from "./op.ts";
|
|
4
|
+
import type { AfterCreate, Resource } from "./resource.ts";
|
|
5
|
+
import type { Secrets } from "./secret.ts";
|
|
6
|
+
import { RESOURCE_OPS } from "./wakeup.ts";
|
|
7
|
+
|
|
8
|
+
// The `Database` contract: what a resource type that holds the company's database (a Neon
|
|
9
|
+
// project, a Supabase project) declares about itself, beside `Host` and `Wakeup`, so databases
|
|
10
|
+
// are swappable and one conformance kit decides whether one works. Going live chooses one by it.
|
|
11
|
+
|
|
12
|
+
/** What a database can do, for choosing one: declared, never checked at runtime. */
|
|
13
|
+
export interface DatabaseCapabilities {
|
|
14
|
+
/** Its compute scales to zero and wakes on the next connection (Neon's). */
|
|
15
|
+
pauses: boolean;
|
|
16
|
+
/** It makes copy-on-write branches that carry the data, each with its own URL. */
|
|
17
|
+
branch: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** What a database's state says of it. */
|
|
21
|
+
export interface DatabaseOutputs {
|
|
22
|
+
healthy: boolean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* A resource type's `database`: the fields it takes going live's choices in, by dotted path into
|
|
27
|
+
* its declared fields, the operation that stores its URL, and how its state reads.
|
|
28
|
+
*/
|
|
29
|
+
export interface DatabaseSpec {
|
|
30
|
+
capabilities: DatabaseCapabilities;
|
|
31
|
+
inputs: {
|
|
32
|
+
/** Where it runs, one of `regions` when the type lists them. */
|
|
33
|
+
region: string;
|
|
34
|
+
/** Its compute's size, when the vendor takes one: absent for its default. */
|
|
35
|
+
size?: string;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* The id of the operation that stores the database's URL, `<vendor>.<group>.<op>`, an
|
|
39
|
+
* operation of the type's own vendor: in the type's own group beside `read` and `import`
|
|
40
|
+
* (`neon.project.storeUrl`), or in a group of its own (`supabase.database.storeUrl`, which
|
|
41
|
+
* predates the contract), never a resource type's own operation. It takes the new object's
|
|
42
|
+
* binding id as `ref` or `id` and the data file's export as `export`, and stores the URL that
|
|
43
|
+
* carries `LISTEN` (the direct one, or a session pooler's; never a transaction-mode pooler's)
|
|
44
|
+
* as the secret `SANOMA_DATABASE_URL_<EXPORT>`, never returning it. The type's `afterCreate`
|
|
45
|
+
* calls it, so apply stores the URL once it makes the database.
|
|
46
|
+
*/
|
|
47
|
+
storeUrl: string;
|
|
48
|
+
/** The regions `inputs.region` takes, for a choice; absent when the vendor's list is not known here. */
|
|
49
|
+
regions?: readonly string[];
|
|
50
|
+
outputs: (state: Record<string, unknown>) => DatabaseOutputs;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* The secret a database's `storeUrl` stores its URL in, for the data file's export that declares
|
|
55
|
+
* it: `store` → `SANOMA_DATABASE_URL_STORE`, `mainStore` → `SANOMA_DATABASE_URL_MAIN_STORE`.
|
|
56
|
+
*/
|
|
57
|
+
export const databaseUrlSecret = (exportName: string): string =>
|
|
58
|
+
`SANOMA_DATABASE_URL_${exportName.replaceAll(/([a-z0-9])([A-Z])/g, "$1_$2").toUpperCase()}`;
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* A `storeUrl`'s store: `url` as `databaseUrlSecret(exportName)`, whose name it returns. The store
|
|
62
|
+
* only adds: a secret that holds another URL (another database, or a password since changed) is
|
|
63
|
+
* refused, not kept, with a `DriverError` (`stale`) that begins `where` and names what else it
|
|
64
|
+
* could be (`others`).
|
|
65
|
+
*/
|
|
66
|
+
export async function storeDatabaseUrl(
|
|
67
|
+
secrets: Secrets,
|
|
68
|
+
exportName: string,
|
|
69
|
+
url: string,
|
|
70
|
+
{ where, others }: { where: string; others: string },
|
|
71
|
+
): Promise<{ secret: string }> {
|
|
72
|
+
const secret = databaseUrlSecret(exportName);
|
|
73
|
+
if ((await secrets.ensure(secret, () => url)) !== url) {
|
|
74
|
+
throw new DriverError(`${where}: ${secret} holds another URL (${others}); delete it to store this one`, {
|
|
75
|
+
retryable: false,
|
|
76
|
+
vendorCode: "stale",
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
return { secret };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Throws what is wrong with a resource type's `database`: a type apply cannot change (not
|
|
84
|
+
* `writable`), one apply may recreate without asking (`stateless`: a database holds data), an
|
|
85
|
+
* input that names no field, a capability that is not a boolean, a `storeUrl` that is not an
|
|
86
|
+
* operation of its vendor or is one of the type's resource operations, or that `afterCreate`
|
|
87
|
+
* does not call, `regions` that is not a list of strings, `outputs` that is not a function.
|
|
88
|
+
* Whether the connector has that operation, and it takes `{ ref | id, export }`, is
|
|
89
|
+
* `checkDatabaseOps`'s, which `resolveConfig` runs.
|
|
90
|
+
*/
|
|
91
|
+
export function checkDatabase(
|
|
92
|
+
what: string,
|
|
93
|
+
database: DatabaseSpec,
|
|
94
|
+
type: {
|
|
95
|
+
vendor: string;
|
|
96
|
+
type: string;
|
|
97
|
+
schema: z.ZodObject;
|
|
98
|
+
writable?: boolean;
|
|
99
|
+
stateless?: boolean;
|
|
100
|
+
afterCreate?: readonly AfterCreate[];
|
|
101
|
+
},
|
|
102
|
+
): void {
|
|
103
|
+
const problems: string[] = [];
|
|
104
|
+
if (!type.writable) problems.push("it must be writable, so apply can create and update it");
|
|
105
|
+
if (type.stateless) problems.push("it must not be stateless: it holds data, so apply asks before it deletes one");
|
|
106
|
+
const capabilities: Partial<DatabaseCapabilities> = database.capabilities ?? {};
|
|
107
|
+
for (const name of ["pauses", "branch"] as const) {
|
|
108
|
+
if (typeof capabilities[name] !== "boolean") problems.push(`capabilities.${name} must be true or false`);
|
|
109
|
+
}
|
|
110
|
+
const { region, size } = database.inputs ?? {};
|
|
111
|
+
if (typeof region !== "string" || !hasPath(type.schema, region)) problems.push(`region names no field (${region})`);
|
|
112
|
+
if (size !== undefined && !hasPath(type.schema, size)) problems.push(`size names no field (${size})`);
|
|
113
|
+
const storeUrl: unknown = database.storeUrl;
|
|
114
|
+
const [vendor, group, name, ...more] = typeof storeUrl === "string" ? storeUrl.split(".") : [];
|
|
115
|
+
if (vendor !== type.vendor || !/^\w+$/.test(group ?? "") || !/^\w+$/.test(name ?? "") || more.length) {
|
|
116
|
+
problems.push(`storeUrl must name an operation of its vendor, ${type.vendor}.<group>.<op> (${String(storeUrl)})`);
|
|
117
|
+
} else if (group === type.type && RESOURCE_OPS.has(name!)) {
|
|
118
|
+
problems.push(`storeUrl must be an operation of its own, not the type's ${name}`);
|
|
119
|
+
} else if (!type.afterCreate?.some((step) => step.op === storeUrl)) {
|
|
120
|
+
problems.push(`afterCreate must call ${storeUrl}, so apply stores the URL once it makes one`);
|
|
121
|
+
}
|
|
122
|
+
if (
|
|
123
|
+
database.regions !== undefined &&
|
|
124
|
+
(!Array.isArray(database.regions) || database.regions.some((r) => typeof r !== "string" || !r))
|
|
125
|
+
) {
|
|
126
|
+
problems.push("regions must be a list of region names");
|
|
127
|
+
}
|
|
128
|
+
if (typeof database.outputs !== "function") problems.push("outputs must be a function of the state");
|
|
129
|
+
if (problems.length) throw new Error(`${what}: its database: ${problems.join("; ")}`);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Throws when a database type's `storeUrl` is not among the config's operations, is one of a
|
|
134
|
+
* resource type's own, or does not take `{ ref | id, export }`. Here rather than in
|
|
135
|
+
* `defineResource`, which sees one type before its package merges in the operations it adds
|
|
136
|
+
* (Supabase's pattern), as `checkWakeupOps`.
|
|
137
|
+
*/
|
|
138
|
+
export function checkDatabaseOps(types: ReadonlyMap<string, Resource>, ops: ReadonlyMap<string, Op>): void {
|
|
139
|
+
for (const [id, type] of types) {
|
|
140
|
+
if (!type.database) continue;
|
|
141
|
+
const op = ops.get(type.database.storeUrl);
|
|
142
|
+
if (!op)
|
|
143
|
+
throw new Error(`${id}: its database's storeUrl, ${type.database.storeUrl}, is no operation of its connector`);
|
|
144
|
+
if (types.has(`${op.vendor}.${op.resource}`) && RESOURCE_OPS.has(op.name)) {
|
|
145
|
+
throw new Error(`${id}: its database's storeUrl, ${op.id}, is a resource type's own operation`);
|
|
146
|
+
}
|
|
147
|
+
// By its fields, not a sample's parse: an id or export with a format of its own is still one.
|
|
148
|
+
const has = (field: string) => hasPath(op.input, field);
|
|
149
|
+
if (!(has("export") && (has("ref") || has("id"))))
|
|
150
|
+
throw new Error(`${id}: its database's storeUrl, ${op.id}, must take { ref or id, export }`);
|
|
151
|
+
}
|
|
152
|
+
}
|
package/src/db.ts
CHANGED
|
@@ -5,9 +5,9 @@ import { warn } from "./log.ts";
|
|
|
5
5
|
import { errorMessage } from "./shared.ts";
|
|
6
6
|
|
|
7
7
|
// The runtime's own tables live in the config's database, beside DBOS's: the credentials, the
|
|
8
|
-
// ledger
|
|
9
|
-
// database, a pool that opens once it is prepared and
|
|
10
|
-
// creation two processes may race.
|
|
8
|
+
// ledger, the collections' records, the wakeup's row and the operator agent's chats. What they
|
|
9
|
+
// share is here: the one preparation of a database, a pool that opens once it is prepared and
|
|
10
|
+
// survives a dropped connection, and a table creation two processes may race.
|
|
11
11
|
|
|
12
12
|
/** A connection or a pool: what the queries take. */
|
|
13
13
|
export type Db = Pick<Pool | Client, "query">;
|
|
@@ -18,6 +18,17 @@ export const CREDENTIALS_TABLE = "sanoma_credentials";
|
|
|
18
18
|
export const LEDGER_TABLE = "sanoma_ledger";
|
|
19
19
|
/** Where the collections keep their records (`collection.ts`): one row per collection and key, the record as JSON. */
|
|
20
20
|
export const DATA_TABLE = "sanoma_data";
|
|
21
|
+
/**
|
|
22
|
+
* The instance's side of its wakeup: one row, made with the table. `owner` is the worker that
|
|
23
|
+
* registers, `generation` counts registrations up, `at` is the deadline registered, `stopped_at`
|
|
24
|
+
* when the worker last stopped, `registered_at` when the wakeup last took one, `last_error` why
|
|
25
|
+
* the last registration failed.
|
|
26
|
+
*/
|
|
27
|
+
export const WAKEUP_TABLE = "sanoma_wakeup";
|
|
28
|
+
/** The operator agent's chats (`chats.ts`): who each is for, its engine and its draft. */
|
|
29
|
+
export const CHATS_TABLE = "sanoma_chats";
|
|
30
|
+
/** Each chat's messages, the AI SDK's UI messages as JSON, in `seq` order. */
|
|
31
|
+
export const CHAT_MESSAGES_TABLE = "sanoma_chat_messages";
|
|
21
32
|
|
|
22
33
|
/** The runtime's tables, each with the word the messages name it by. */
|
|
23
34
|
const TABLES: readonly [what: string, ddl: string][] = [
|
|
@@ -39,6 +50,23 @@ const TABLES: readonly [what: string, ddl: string][] = [
|
|
|
39
50
|
`CREATE TABLE IF NOT EXISTS ${DATA_TABLE} (collection text NOT NULL, key text NOT NULL, record jsonb NOT NULL, ` +
|
|
40
51
|
"updated_at timestamptz NOT NULL DEFAULT now(), updated_by text NOT NULL, PRIMARY KEY (collection, key))",
|
|
41
52
|
],
|
|
53
|
+
[
|
|
54
|
+
"wakeup",
|
|
55
|
+
`CREATE TABLE IF NOT EXISTS ${WAKEUP_TABLE} (id integer PRIMARY KEY CHECK (id = 1), owner uuid, ` +
|
|
56
|
+
"generation bigint NOT NULL DEFAULT 0, at timestamptz, stopped_at timestamptz, registered_at timestamptz, " +
|
|
57
|
+
`last_error text); INSERT INTO ${WAKEUP_TABLE} (id) VALUES (1) ON CONFLICT (id) DO NOTHING`,
|
|
58
|
+
],
|
|
59
|
+
[
|
|
60
|
+
"chats",
|
|
61
|
+
`CREATE TABLE IF NOT EXISTS ${CHATS_TABLE} (id text PRIMARY KEY, person text NOT NULL, engine text NOT NULL, ` +
|
|
62
|
+
"draft_id text, title text NOT NULL, created_at timestamptz NOT NULL DEFAULT now()); " +
|
|
63
|
+
`CREATE INDEX IF NOT EXISTS ${CHATS_TABLE}_person ON ${CHATS_TABLE} (person, created_at)`,
|
|
64
|
+
],
|
|
65
|
+
[
|
|
66
|
+
"chat messages",
|
|
67
|
+
`CREATE TABLE IF NOT EXISTS ${CHAT_MESSAGES_TABLE} (chat_id text NOT NULL REFERENCES ${CHATS_TABLE} (id) ` +
|
|
68
|
+
"ON DELETE CASCADE, id text NOT NULL, seq integer NOT NULL, message jsonb NOT NULL, PRIMARY KEY (chat_id, id))",
|
|
69
|
+
],
|
|
42
70
|
];
|
|
43
71
|
|
|
44
72
|
/** The Postgres error code of `err`, or an empty string when it has none. */
|
|
@@ -142,7 +170,7 @@ const prepared = new Map<string, Promise<void>>();
|
|
|
142
170
|
* Makes the database at `databaseUrl` ready for the runtime, once per process and URL: the
|
|
143
171
|
* database itself and DBOS's tables when they are not there yet (as a worker's launch makes
|
|
144
172
|
* them; credentials may be set, and a ledger read, before any worker has run), then the
|
|
145
|
-
* credentials, ledger and
|
|
173
|
+
* credentials, ledger, data, wakeup and chat tables, and its idle-transaction timeout (`guardIdleTransactions`).
|
|
146
174
|
* Two processes preparing one database both succeed: the second waits for the first
|
|
147
175
|
* (`lockPreparation`). A preparation that fails is forgotten, so the next call tries again.
|
|
148
176
|
*/
|
|
@@ -188,11 +216,7 @@ async function prepare(databaseUrl: string): Promise<void> {
|
|
|
188
216
|
* cannot see (the lock's holder is this connection, not the migration's).
|
|
189
217
|
*/
|
|
190
218
|
async function lockPreparation(databaseUrl: string): Promise<Client> {
|
|
191
|
-
const conn = await
|
|
192
|
-
if (pgCode(err) !== NO_DATABASE) throw err;
|
|
193
|
-
await createDatabase(databaseUrl);
|
|
194
|
-
return connect(databaseUrl);
|
|
195
|
-
});
|
|
219
|
+
const conn = await connectMaking(databaseUrl);
|
|
196
220
|
try {
|
|
197
221
|
const ask = "SELECT pg_try_advisory_lock(hashtext('sanoma.prepareDatabase')) AS locked";
|
|
198
222
|
while (!(await conn.query<{ locked: boolean }>(ask)).rows[0]!.locked) await delay(50);
|
|
@@ -203,6 +227,15 @@ async function lockPreparation(databaseUrl: string): Promise<Client> {
|
|
|
203
227
|
}
|
|
204
228
|
}
|
|
205
229
|
|
|
230
|
+
/** A connection to the database at `databaseUrl`, made first when it is not there (`createDatabase`). */
|
|
231
|
+
export async function connectMaking(databaseUrl: string): Promise<Client> {
|
|
232
|
+
return connect(databaseUrl).catch(async (err: unknown) => {
|
|
233
|
+
if (pgCode(err) !== NO_DATABASE) throw err;
|
|
234
|
+
await createDatabase(databaseUrl);
|
|
235
|
+
return connect(databaseUrl);
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
|
|
206
239
|
async function connect(databaseUrl: string): Promise<Client> {
|
|
207
240
|
const conn = new Client({ connectionString: databaseUrl, connectionTimeoutMillis: 10_000 });
|
|
208
241
|
// An error nobody listens for ends the process; the query in flight fails with it.
|
|
@@ -263,3 +296,69 @@ export async function createTable(db: Db, ddl: string, what: string): Promise<vo
|
|
|
263
296
|
throw new Error(`could not prepare the ${what} table: ${errorMessage(err)}`, { cause: err });
|
|
264
297
|
}
|
|
265
298
|
}
|
|
299
|
+
|
|
300
|
+
// The wakeup row's reads and writes. Every write after the claim names the owner: a worker
|
|
301
|
+
// another process has replaced since (a second instance on the database) changes nothing.
|
|
302
|
+
|
|
303
|
+
// `at` is the deadline registered for the row's generation, null until one is: a new generation
|
|
304
|
+
// (each start and each stop attempt) clears it, so a wake registered earlier is stale.
|
|
305
|
+
|
|
306
|
+
/** Makes `owner` the worker that registers the instance's wakeups; a running worker needs no wake registered before. */
|
|
307
|
+
export async function claimWakeup(db: Db, owner: string): Promise<void> {
|
|
308
|
+
await db.query(`UPDATE ${WAKEUP_TABLE} SET owner = $1, generation = generation + 1, at = NULL WHERE id = 1`, [owner]);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** Counts the row's generation up while `owner` still owns it: the new generation, or undefined when another worker owns the row now. */
|
|
312
|
+
export async function nextWakeupGeneration(db: Db, owner: string): Promise<number | undefined> {
|
|
313
|
+
const { rows } = await db.query<{ generation: string }>(
|
|
314
|
+
`UPDATE ${WAKEUP_TABLE} SET generation = generation + 1, at = NULL WHERE id = 1 AND owner = $1 RETURNING generation`,
|
|
315
|
+
[owner],
|
|
316
|
+
);
|
|
317
|
+
return rows[0] && Number(rows[0].generation);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/** Records that the wakeup took `at` for `generation`. */
|
|
321
|
+
export async function recordWakeupRegistration(db: Db, owner: string, generation: number, at: string): Promise<void> {
|
|
322
|
+
await db.query(
|
|
323
|
+
`UPDATE ${WAKEUP_TABLE} SET at = $3, registered_at = now(), last_error = NULL WHERE id = 1 AND owner = $1 AND generation = $2`,
|
|
324
|
+
[owner, generation, at],
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Records why registering `generation` failed. */
|
|
329
|
+
export async function recordWakeupError(db: Db, owner: string, generation: number, error: string): Promise<void> {
|
|
330
|
+
await db.query(`UPDATE ${WAKEUP_TABLE} SET last_error = $3 WHERE id = 1 AND owner = $1 AND generation = $2`, [
|
|
331
|
+
owner,
|
|
332
|
+
generation,
|
|
333
|
+
error,
|
|
334
|
+
]);
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/** Records that the worker stops now, with `generation` registered. */
|
|
338
|
+
export async function recordWakeupStop(db: Db, owner: string, generation: number): Promise<void> {
|
|
339
|
+
await db.query(`UPDATE ${WAKEUP_TABLE} SET stopped_at = now() WHERE id = 1 AND owner = $1 AND generation = $2`, [
|
|
340
|
+
owner,
|
|
341
|
+
generation,
|
|
342
|
+
]);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* True when `generation` is the wakeup row's and the worker stopped with it: the wake
|
|
347
|
+
* `/wake?generation=N` asks for is the one the worker last registered before it stopped, not
|
|
348
|
+
* one a later stop replaced, nor one registered by a worker that then kept running. For the
|
|
349
|
+
* route's supervisor, which decides on its own what a wake means when no wakeup is declared.
|
|
350
|
+
*/
|
|
351
|
+
export async function shouldWake(databaseUrl: string, generation: number): Promise<boolean> {
|
|
352
|
+
await prepareDatabase(databaseUrl);
|
|
353
|
+
const conn = await connect(databaseUrl);
|
|
354
|
+
try {
|
|
355
|
+
const { rows } = await conn.query<{ current: boolean }>(
|
|
356
|
+
`SELECT generation = $1 AND at IS NOT NULL AND stopped_at >= registered_at ` +
|
|
357
|
+
`AS current FROM ${WAKEUP_TABLE} WHERE id = 1`,
|
|
358
|
+
[generation],
|
|
359
|
+
);
|
|
360
|
+
return rows[0]?.current === true;
|
|
361
|
+
} finally {
|
|
362
|
+
await conn.end().catch(() => undefined);
|
|
363
|
+
}
|
|
364
|
+
}
|