@sanoma-ai/workflows 0.1.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/README.md +186 -76
  2. package/dist/apply-steps.d.ts +6 -0
  3. package/dist/apply-steps.js +93 -0
  4. package/dist/apply-steps.js.map +1 -0
  5. package/dist/apply.d.ts +197 -0
  6. package/dist/apply.js +470 -0
  7. package/dist/apply.js.map +1 -0
  8. package/dist/approvals.d.ts +11 -6
  9. package/dist/approvals.js +38 -10
  10. package/dist/approvals.js.map +1 -1
  11. package/dist/call-workflow.d.ts +16 -0
  12. package/dist/call-workflow.js +35 -0
  13. package/dist/call-workflow.js.map +1 -0
  14. package/dist/call.d.ts +35 -2
  15. package/dist/call.js +123 -43
  16. package/dist/call.js.map +1 -1
  17. package/dist/cedar.d.ts +87 -0
  18. package/dist/cedar.js +365 -0
  19. package/dist/cedar.js.map +1 -0
  20. package/dist/client.d.ts +123 -9
  21. package/dist/client.js +355 -34
  22. package/dist/client.js.map +1 -1
  23. package/dist/collection.d.ts +107 -0
  24. package/dist/collection.js +306 -0
  25. package/dist/collection.js.map +1 -0
  26. package/dist/config.d.ts +42 -14
  27. package/dist/config.js +109 -28
  28. package/dist/config.js.map +1 -1
  29. package/dist/constructors.d.ts +2 -0
  30. package/dist/constructors.js +5 -0
  31. package/dist/constructors.js.map +1 -0
  32. package/dist/credentials.d.ts +25 -0
  33. package/dist/credentials.js +73 -5
  34. package/dist/credentials.js.map +1 -1
  35. package/dist/data.d.ts +2 -0
  36. package/dist/data.js +3 -0
  37. package/dist/data.js.map +1 -0
  38. package/dist/datafile.d.ts +38 -3
  39. package/dist/datafile.js +166 -13
  40. package/dist/datafile.js.map +1 -1
  41. package/dist/db.d.ts +5 -3
  42. package/dist/db.js +88 -21
  43. package/dist/db.js.map +1 -1
  44. package/dist/define.d.ts +10 -4
  45. package/dist/define.js +1 -5
  46. package/dist/define.js.map +1 -1
  47. package/dist/describe.d.ts +54 -8
  48. package/dist/describe.js +84 -11
  49. package/dist/describe.js.map +1 -1
  50. package/dist/direct.d.ts +9 -0
  51. package/dist/direct.js +11 -0
  52. package/dist/direct.js.map +1 -0
  53. package/dist/drift.d.ts +64 -12
  54. package/dist/drift.js +118 -41
  55. package/dist/drift.js.map +1 -1
  56. package/dist/errors.d.ts +16 -1
  57. package/dist/errors.js +30 -1
  58. package/dist/errors.js.map +1 -1
  59. package/dist/expectations.d.ts +4 -4
  60. package/dist/expectations.js +43 -9
  61. package/dist/expectations.js.map +1 -1
  62. package/dist/host.d.ts +56 -0
  63. package/dist/host.js +45 -0
  64. package/dist/host.js.map +1 -0
  65. package/dist/index.d.ts +13 -5
  66. package/dist/index.js +15 -5
  67. package/dist/index.js.map +1 -1
  68. package/dist/ledger.d.ts +12 -1
  69. package/dist/ledger.js +18 -11
  70. package/dist/ledger.js.map +1 -1
  71. package/dist/lint.d.ts +5 -4
  72. package/dist/lint.js +24 -7
  73. package/dist/lint.js.map +1 -1
  74. package/dist/lockfile.d.ts +36 -0
  75. package/dist/lockfile.js +69 -0
  76. package/dist/lockfile.js.map +1 -0
  77. package/dist/op.d.ts +9 -0
  78. package/dist/op.js.map +1 -1
  79. package/dist/outline.d.ts +2 -0
  80. package/dist/outline.js +1 -1
  81. package/dist/outline.js.map +1 -1
  82. package/dist/paths.d.ts +4 -0
  83. package/dist/paths.js +17 -3
  84. package/dist/paths.js.map +1 -1
  85. package/dist/person.d.ts +61 -0
  86. package/dist/person.js +69 -0
  87. package/dist/person.js.map +1 -0
  88. package/dist/plugin.js +1 -1
  89. package/dist/plugin.js.map +1 -1
  90. package/dist/policy.d.ts +51 -55
  91. package/dist/policy.js +14 -36
  92. package/dist/policy.js.map +1 -1
  93. package/dist/resource.d.ts +207 -25
  94. package/dist/resource.js +184 -30
  95. package/dist/resource.js.map +1 -1
  96. package/dist/resources.d.ts +76 -9
  97. package/dist/resources.js +170 -30
  98. package/dist/resources.js.map +1 -1
  99. package/dist/run.d.ts +31 -1
  100. package/dist/run.js +6 -1
  101. package/dist/run.js.map +1 -1
  102. package/dist/sandbox.js +5 -1
  103. package/dist/sandbox.js.map +1 -1
  104. package/dist/scenario.d.ts +25 -9
  105. package/dist/scenario.js +125 -27
  106. package/dist/scenario.js.map +1 -1
  107. package/dist/scenarios.d.ts +10 -6
  108. package/dist/scenarios.js +8 -4
  109. package/dist/scenarios.js.map +1 -1
  110. package/dist/schema.d.ts +32 -0
  111. package/dist/schema.js +225 -0
  112. package/dist/schema.js.map +1 -0
  113. package/dist/sdk.d.ts +34 -0
  114. package/dist/sdk.js +168 -0
  115. package/dist/sdk.js.map +1 -0
  116. package/dist/secret.d.ts +42 -0
  117. package/dist/secret.js +58 -0
  118. package/dist/secret.js.map +1 -0
  119. package/dist/shared.d.ts +188 -4
  120. package/dist/shared.js +165 -5
  121. package/dist/shared.js.map +1 -1
  122. package/dist/sites.d.ts +52 -0
  123. package/dist/sites.js +174 -0
  124. package/dist/sites.js.map +1 -0
  125. package/dist/source.d.ts +58 -0
  126. package/dist/source.js +54 -0
  127. package/dist/source.js.map +1 -0
  128. package/dist/version.d.ts +3 -2
  129. package/dist/version.js +3 -2
  130. package/dist/version.js.map +1 -1
  131. package/dist/worker.d.ts +6 -2
  132. package/dist/worker.js +37 -8
  133. package/dist/worker.js.map +1 -1
  134. package/dist/writer.d.ts +35 -0
  135. package/dist/writer.js +201 -0
  136. package/dist/writer.js.map +1 -0
  137. package/oxlint.json +5 -16
  138. package/package.json +11 -1
  139. package/src/apply-steps.ts +110 -0
  140. package/src/apply.ts +626 -0
  141. package/src/approvals.ts +49 -13
  142. package/src/call-workflow.ts +44 -0
  143. package/src/call.ts +173 -46
  144. package/src/cedar.ts +493 -0
  145. package/src/client.ts +450 -49
  146. package/src/collection.ts +431 -0
  147. package/src/config.ts +168 -48
  148. package/src/constructors.ts +4 -0
  149. package/src/credentials.ts +90 -5
  150. package/src/data.ts +10 -0
  151. package/src/datafile.ts +207 -15
  152. package/src/db.ts +90 -22
  153. package/src/define.ts +11 -9
  154. package/src/describe.ts +146 -23
  155. package/src/direct.ts +15 -0
  156. package/src/drift.ts +155 -45
  157. package/src/errors.ts +36 -1
  158. package/src/expectations.ts +46 -11
  159. package/src/host.ts +96 -0
  160. package/src/index.ts +56 -2
  161. package/src/ledger.ts +35 -17
  162. package/src/lint.ts +29 -12
  163. package/src/lockfile.ts +95 -0
  164. package/src/op.ts +9 -0
  165. package/src/outline.ts +1 -1
  166. package/src/paths.ts +18 -3
  167. package/src/person.ts +75 -0
  168. package/src/plugin.ts +1 -1
  169. package/src/policy.ts +59 -63
  170. package/src/resource.ts +302 -52
  171. package/src/resources.ts +226 -30
  172. package/src/run.ts +33 -1
  173. package/src/sandbox.ts +5 -1
  174. package/src/scenario.ts +175 -43
  175. package/src/scenarios.ts +15 -8
  176. package/src/schema.ts +261 -0
  177. package/src/sdk.ts +211 -0
  178. package/src/secret.ts +92 -0
  179. package/src/shared.ts +279 -6
  180. package/src/sites.ts +220 -0
  181. package/src/source.ts +82 -0
  182. package/src/version.ts +3 -2
  183. package/src/worker.ts +45 -11
  184. package/src/writer.ts +225 -0
package/src/config.ts CHANGED
@@ -2,10 +2,22 @@ import { existsSync, readFileSync, statSync } from "node:fs";
2
2
  import { dirname, join, resolve } from "node:path";
3
3
  import { pathToFileURL } from "node:url";
4
4
  import { DBOS } from "@dbos-inc/dbos-sdk";
5
+ import { type ApplyDeclared, applyWorkflow } from "./apply.ts";
6
+ import { applySteps } from "./apply-steps.ts";
7
+ import { callWorkflow } from "./call-workflow.ts";
8
+ import {
9
+ type Collection,
10
+ collectionsConnector,
11
+ collectionsDriver,
12
+ collectionsFake,
13
+ isCollection,
14
+ } from "./collection.ts";
15
+ import { lazyPool, type LazyPool } from "./db.ts";
5
16
  import { callerFile, type Use, type WorkflowDefinition } from "./define.ts";
6
- import { DRIFT_WORKFLOW, type DriftDeclared, driftWorkflow } from "./drift.ts";
17
+ import { DRIFT_WORKFLOW, driftWorkflow } from "./drift.ts";
7
18
  import { statusesOf, unconfigured } from "./env.ts";
8
19
  import type { Fake } from "./fake.ts";
20
+ import { peopleConnector } from "./person.ts";
9
21
  import type { LedgerStore } from "./ledger.ts";
10
22
  import {
11
23
  type Connector,
@@ -20,17 +32,20 @@ import {
20
32
  setupOf,
21
33
  VENDOR,
22
34
  } from "./op.ts";
35
+ import { policyAt } from "./cedar.ts";
23
36
  import type { Policy } from "./policy.ts";
24
- import { type Resource, resourceTypesOf, withoutWriteOnly } from "./resource.ts";
37
+ import { isChangeOp, type Resource, resourceTypesOf, withoutWriteOnly } from "./resource.ts";
38
+ import { APPLY_WORKFLOW, BUILT_IN_WORKFLOWS, CALL_WORKFLOW, DATA_VENDOR } from "./shared.ts";
39
+ import { sourceDriver, sourceOps } from "./source.ts";
25
40
  import type { Steps } from "./steps.ts";
26
41
  import { computeVersion } from "./version.ts";
27
42
 
28
43
  /**
29
- * What a worker runs, on which connectors, under which policy: the argument to `startWorker`.
44
+ * What a worker runs, on which connectors: the argument to `startWorker`.
30
45
  * A project usually keeps it in `sanoma.config.ts` and passes it in; nothing loads that file
31
46
  * automatically. The rest is convention: each connector carries its driver and its fake
32
- * (`withSetup`), the app's name is its `package.json`'s, and the scenarios and data files are
33
- * `scenarios/` and `resources/` in `root`.
47
+ * (`withSetup`), the app's name is its `package.json`'s, and the scenarios, data files and
48
+ * policies are `scenarios/`, `resources/` and `policies/` in `root`.
34
49
  */
35
50
  export interface SanomaConfig {
36
51
  workflows: WorkflowDefinition<any, any>[];
@@ -42,8 +57,6 @@ export interface SanomaConfig {
42
57
  * package attaches the maker of.
43
58
  */
44
59
  connectors: Connector<any, any>[];
45
- /** Checked before every operation call. Required: `allowAll` says that every call is allowed. */
46
- policy: Policy;
47
60
  /**
48
61
  * Where the audit record goes. Default: `postgresLedger` on the config's database, which the
49
62
  * worker, the app and other processes share; the worker and the client each make their own
@@ -70,8 +83,15 @@ export interface SanomaConfig {
70
83
  appName?: never;
71
84
  /** @deprecated Gone: the scenarios are `<root>/scenarios/`. Leave it out. */
72
85
  scenarios?: never;
86
+ /** @deprecated Gone: the policy is `<root>/policies/**\/*.cedar`. Leave it out. */
87
+ policy?: never;
73
88
  /** The company's own steps for its scenarios (`defineSteps`), besides those its connectors ship. */
74
89
  steps?: Steps[];
90
+ /**
91
+ * The company's collections (`collection()`): their operations are `data.<name>.<op>`, their
92
+ * records in the config's database, and in memory for a sandbox run.
93
+ */
94
+ collections?: readonly Collection[];
75
95
  }
76
96
 
77
97
  /** A config, checked, with everything a worker, client or app derives from it. */
@@ -86,7 +106,10 @@ export interface ResolvedConfig {
86
106
  version: string;
87
107
  /** The DBOS queue runs are started on: `sanoma:<appName>`. */
88
108
  queueName: string;
89
- /** The operations the connectors declare, by id. */
109
+ /**
110
+ * The operations the connectors declare, by id, the built-in `sanoma.source.resolve` when one
111
+ * declares a host type, and the collections' (`data.<name>.<op>`).
112
+ */
90
113
  ops: Map<string, Op>;
91
114
  /** The drivers' functions, by operation id, each guarded by its driver's `env` (`indexDrivers`). */
92
115
  drivers: Map<string, DriverFn>;
@@ -103,12 +126,27 @@ export interface ResolvedConfig {
103
126
  types: ReadonlyMap<string, Resource>;
104
127
  /** The steps scenarios can use besides the generic ones: each connector's (`withSetup`), once each, then the config's own. */
105
128
  steps: readonly Steps[];
129
+ /** The config's collections, by name: their operations are in `ops` and `drivers`, their fake in `fakes`, under `DATA_VENDOR`. */
130
+ collections: ReadonlyMap<string, Collection>;
131
+ /**
132
+ * The pool the collections' driver keeps their records through, when there are any: opened on
133
+ * its first call, and closed by whoever calls the driver (the worker, and the client, which
134
+ * calls it directly, when each stops).
135
+ */
136
+ dataStore?: LazyPool;
106
137
  /**
107
- * By name; each checked against `ops` and `drivers`. The config's, and the built-in `drift`
108
- * (`DRIFT_WORKFLOW`) when its connectors declare resource types.
138
+ * By name; each checked against `ops` and `drivers`. The config's, the built-ins `drift`
139
+ * (`DRIFT_WORKFLOW`) and `apply` (`APPLY_WORKFLOW`) when its connectors declare resource types,
140
+ * and `call` (`CALL_WORKFLOW`) when an operation may be called directly (`isDirect`).
109
141
  */
110
142
  workflows: Map<string, WorkflowDefinition<any, any>>;
143
+ /**
144
+ * Checked before every operation call and run start: the root's `policies/**\/*.cedar`
145
+ * (`policyAt`), `allowAll` without any, and one that refuses everything while they have problems.
146
+ */
111
147
  policy: Policy;
148
+ /** What is wrong in the policies, one printable line each (`file:line:col: <policy id>: <message>`). */
149
+ policyProblems: string[];
112
150
  /** The config's `ledger`, when it gives one; the worker and the client otherwise make `postgresLedger` on `databaseUrl` (`ownedLedger`). */
113
151
  ledger?: LedgerStore;
114
152
  /**
@@ -158,12 +196,13 @@ export function rootOf(config: Pick<SanomaConfig, "root" | "file">): string | un
158
196
 
159
197
  /**
160
198
  * Checks a config and derives what the runtime needs from it: the drivers and fakes from the
161
- * connectors' setups (`withSetup`), and from `root`, the app's name (`package.json`) and the
162
- * scenarios (`scenarios/`). Throws what `startWorker` would refuse, with the same message, so
163
- * `describeConfig`, `SanomaClient.connect` and the app fail the same way the worker does. It
164
- * opens nothing and reads no file outside `root`. Each entry
165
- * point resolves once: the worker and the client when they start, the app at `startApp`, the
166
- * `sanoma` bin when it loads the config.
199
+ * connectors' setups (`withSetup`), the collections' operations, driver and fake, and from
200
+ * `root`, the app's name (`package.json`), the scenarios (`scenarios/`) and the policy
201
+ * (`policies/`, `policyAt`). Throws what
202
+ * `startWorker` would refuse, with the same message, so `describeConfig`, `SanomaClient.connect`
203
+ * and the app fail the same way the worker does. It opens nothing and reads no file outside
204
+ * `root`. Each entry point resolves once: the worker and the client when they start, the app at
205
+ * `startApp`, the `sanoma` bin when it loads the config.
167
206
  */
168
207
  export function resolveConfig(config: SanomaConfig): ResolvedConfig {
169
208
  if (!Array.isArray(config.connectors)) {
@@ -173,30 +212,49 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
173
212
  if (!Array.isArray(config.workflows)) {
174
213
  throw new Error("The config needs `workflows`: an array of defineWorkflow definitions");
175
214
  }
176
- if (typeof config.policy !== "function") {
177
- throw new Error("The config needs a `policy`; use `allowAll` to allow every operation call");
178
- }
179
215
  // An untyped config naming fake drivers would otherwise run live. Delete once sanoma-ops has migrated.
180
216
  if (config.drivers !== undefined || config.fakes !== undefined)
181
217
  throw new Error(
182
218
  "The config's `drivers` and `fakes` are gone: each connector carries its driver and its fake; leave them out",
183
219
  );
220
+ // An untyped config naming its policy would otherwise run allowing everything. Delete once sanoma-ops has migrated.
221
+ if (config.policy !== undefined) {
222
+ throw new Error("The config's `policy` is gone: the policy is policies/*.cedar in the root; leave it out");
223
+ }
224
+ const collections = collectionsOf(config);
184
225
  const ledger = ledgerOf(config);
185
226
  const root = rootOf(config);
186
227
  const appName = appNameAt(root);
187
228
  const scenarios = scenariosAt(root);
188
- const ops = indexConnectors(config.connectors);
229
+ const types = resourceTypesOf(config.connectors);
230
+ // Built-in: `sanoma.source.resolve`, which apply and drift call for a host's revision.
231
+ const hosts = [...types.values()].some((type) => type.host);
232
+ // Built-in: the collections' operations, `data.<name>.<op>`, on the config's database.
233
+ const databaseUrl = resolveDatabaseUrl(config);
234
+ const declared = [...collections.values()];
235
+ const dataStore = declared.length ? lazyPool(databaseUrl, "data") : undefined;
236
+ const ops = indexConnectors([
237
+ ...config.connectors,
238
+ ...(hosts ? [sourceOps] : []),
239
+ ...(dataStore ? [collectionsConnector(declared)] : []),
240
+ ]);
189
241
  const driversByVendor = new Map([...makersOf(config.connectors, "driver")].map(([vendor, make]) => [vendor, make()]));
190
- const drivers = indexDrivers([...driversByVendor.values()], ops);
242
+ const drivers = indexDrivers(
243
+ [
244
+ ...driversByVendor.values(),
245
+ ...(hosts ? [sourceDriver] : []),
246
+ ...(dataStore ? [collectionsDriver(declared, dataStore)] : []),
247
+ ],
248
+ ops,
249
+ );
191
250
  const fakes = makersOf(config.connectors, "fake");
251
+ if (dataStore) fakes.set(DATA_VENDOR, () => collectionsFake(declared));
192
252
  const names = new Map<string, WorkflowDefinition<any, any>>();
193
253
  for (const wf of config.workflows) {
194
- if (wf.name === DRIFT_WORKFLOW) {
195
- throw new Error(
196
- `A workflow is named "${DRIFT_WORKFLOW}", the name of the built-in drift workflow; name it otherwise`,
197
- );
254
+ if (BUILT_IN_WORKFLOWS.has(wf.name)) {
255
+ throw new Error(`A workflow is named "${wf.name}", the name of a built-in workflow; name it otherwise`);
198
256
  }
199
- checkUses(wf, ops, drivers);
257
+ checkUses(wf, ops, drivers, types);
200
258
  const other = names.get(wf.name);
201
259
  if (other && other !== wf) {
202
260
  throw new Error(
@@ -205,16 +263,25 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
205
263
  }
206
264
  names.set(wf.name, wf);
207
265
  }
208
- // Built-in: checking the resources the data files declare against their vendors.
209
- const types = resourceTypesOf(config.connectors);
266
+ // Built-in: checking the resources the data files declare against their vendors, and applying them.
210
267
  if (types.size) {
211
- const declared = () =>
212
- DBOS.runStep(() => readDeclared(root, config.connectors, types), { name: "drift:data-files" });
213
- names.set(DRIFT_WORKFLOW, driftWorkflow(types, ops, drivers, declared));
268
+ const connectors = connectorsOf(config);
269
+ // The types the data files may declare: the vendors' and the people's, whose resources are left out.
270
+ const declarable = resourceTypesOf(connectors);
271
+ const read = () => readDeclared(root, connectors, declarable, types);
272
+ names.set(
273
+ DRIFT_WORKFLOW,
274
+ driftWorkflow(types, ops, drivers, () => DBOS.runStep(read, { name: "drift:data-files" })),
275
+ );
276
+ names.set(APPLY_WORKFLOW, applyWorkflow(types, ops, drivers, applySteps(appName, root, read)));
214
277
  }
278
+ // Built-in: a direct call the policy holds for an approval, made a run, when any operation may be called directly.
279
+ const call = callWorkflow(ops, drivers, types);
280
+ if (call.uses.length) names.set(CALL_WORKFLOW, call);
281
+ const { policy, problems: policyProblems } = policyAt(root, { ops, workflows: names, collections });
215
282
  return {
216
283
  appName,
217
- databaseUrl: resolveDatabaseUrl(config),
284
+ databaseUrl,
218
285
  // The built-in's code and operations are part of the app's version, like the config's own.
219
286
  version: computeVersion({ appName, workflows: [...names.values()] }),
220
287
  queueName: `sanoma:${appName}`,
@@ -225,13 +292,46 @@ export function resolveConfig(config: SanomaConfig): ResolvedConfig {
225
292
  ...(scenarios && { scenarios }),
226
293
  types,
227
294
  steps: [...new Set([...config.connectors.flatMap((c) => setupOf(c).steps ?? []), ...(config.steps ?? [])])],
295
+ collections,
296
+ ...(dataStore && { dataStore }),
228
297
  workflows: names,
229
- policy: config.policy,
298
+ policy,
299
+ policyProblems,
230
300
  ...(ledger !== undefined && { ledger }),
231
301
  ...(root !== undefined && { root }),
232
302
  };
233
303
  }
234
304
 
305
+ /**
306
+ * The config's collections by name, checked: each one `collection()` made, no two with one name,
307
+ * and no connector of their vendor, `DATA_VENDOR`.
308
+ */
309
+ function collectionsOf(config: SanomaConfig): Map<string, Collection> {
310
+ const list: unknown = config.collections ?? [];
311
+ if (!Array.isArray(list)) {
312
+ throw new Error("The config's `collections` must be an array of the collections `collection()` declares");
313
+ }
314
+ if (config.connectors.some((c) => c[VENDOR].id === DATA_VENDOR)) {
315
+ throw new Error(
316
+ `A connector in \`connectors\` is of the vendor "${DATA_VENDOR}", which the collections' operations are of; name it otherwise`,
317
+ );
318
+ }
319
+ const collections = new Map<string, Collection>();
320
+ for (const c of list) {
321
+ if (!isCollection(c)) throw new Error("`collections` holds something `collection()` did not make");
322
+ const other = collections.get(c.name);
323
+ if (other && other !== c) throw new Error(`Two collections are named "${c.name}"; name each once`);
324
+ collections.set(c.name, c);
325
+ }
326
+ return collections;
327
+ }
328
+
329
+ /** The config's connectors and the runtime's people types' (`peopleConnector`): what its data files are read against. */
330
+ export const connectorsOf = (config: Pick<SanomaConfig, "connectors">): Connector<any, any>[] => [
331
+ ...config.connectors,
332
+ peopleConnector,
333
+ ];
334
+
235
335
  /**
236
336
  * What the connectors' setups attach under `key` (`withSetup`), by vendor id in `connectors`
237
337
  * order. One vendor's connectors may name one maker, as copies of one connector do, never two.
@@ -280,26 +380,36 @@ function scenariosAt(root: string | undefined): URL | undefined {
280
380
  }
281
381
 
282
382
  /**
283
- * The data files under `root`, as a drift run records them: each resource without its write-only
284
- * values, which a run never compares and never records, and the problems.
383
+ * The data files under `root` and the lockfile beside them, in one read, as a drift or apply run
384
+ * records them: each resource with its binding, without its write-only values (a secret marker
385
+ * names one, and stays), which a run never compares and never records, the bindings, the orphans
386
+ * and the problems.
285
387
  */
286
388
  async function readDeclared(
287
389
  root: string | undefined,
288
390
  connectors: readonly Connector<any, any>[],
391
+ declarable: ReadonlyMap<string, Resource>,
289
392
  types: ReadonlyMap<string, Resource>,
290
- ): Promise<DriftDeclared> {
291
- // Loaded when a drift run reads the files, not with the worker: the reader parses with oxc-parser.
393
+ ): Promise<Omit<ApplyDeclared, "rebinds">> {
394
+ // Loaded when a run reads the files, not with the worker: the reader parses with oxc-parser.
292
395
  const { readDataFiles } = await import("./resources.ts");
293
- const { resources, problems } = readDataFiles(root, connectors, types);
396
+ const { resources, bindings, orphans, problems } = readDataFiles(root, connectors, declarable);
294
397
  return {
295
- resources: resources.map(({ id, vendor, type, name, desired, refs }) => ({
296
- id,
297
- vendor,
298
- type,
299
- name,
300
- desired: withoutWriteOnly(types.get(`${vendor}.${type}`)!.fields, desired),
301
- refs,
302
- })),
398
+ // Only `types`' resources have a vendor object to check or apply: not the people's (vendorless).
399
+ resources: resources
400
+ .filter((r) => types.has(`${r.vendor}.${r.type}`))
401
+ .map(({ id, vendor, type, name, identity, binding, desired, refs }) => ({
402
+ id,
403
+ vendor,
404
+ type,
405
+ name,
406
+ ...(identity !== undefined && { identity }),
407
+ ...(binding !== undefined && { binding }),
408
+ desired: withoutWriteOnly(types.get(`${vendor}.${type}`)!.fields, desired),
409
+ refs,
410
+ })),
411
+ bindings,
412
+ orphans,
303
413
  problems,
304
414
  };
305
415
  }
@@ -362,13 +472,23 @@ function guarded(vendor: string, env: DriverEnv, fn: DriverFn): DriverFn {
362
472
  * Every operation a workflow uses must be one the config's connectors declare, with a driver.
363
473
  * The workflow normally imports the same connector object; a copy is accepted only if it
364
474
  * declares the same effect and retry setting, and the worker's declaration is used either way.
475
+ * None may change a declared resource (`isChangeOp`): only the built-in apply workflow does.
365
476
  */
366
- function checkUses(wf: WorkflowDefinition<any, any>, ops: Map<string, Op>, impls: Map<string, unknown>) {
477
+ function checkUses(
478
+ wf: WorkflowDefinition<any, any>,
479
+ ops: Map<string, Op>,
480
+ impls: Map<string, unknown>,
481
+ types: ReadonlyMap<string, Resource>,
482
+ ) {
367
483
  const problems: string[] = [];
368
484
  for (const op of (wf.uses as readonly Use[]).filter(isOp)) {
369
485
  const known = ops.get(op.id);
370
486
  if (!known) problems.push(`${op.id} is not declared by any connector in \`connectors\``);
371
- else if (known !== op && (known.effect !== op.effect || known.idempotent !== op.idempotent)) {
487
+ else if (isChangeOp(known, types)) {
488
+ problems.push(
489
+ `${op.id} changes a declared resource, which only ${APPLY_WORKFLOW} does: change the data file instead`,
490
+ );
491
+ } else if (known !== op && (known.effect !== op.effect || known.idempotent !== op.idempotent)) {
372
492
  problems.push(
373
493
  `${op.id} is declared with effect "${op.effect}"${op.idempotent ? " (idempotent)" : ""}, ` +
374
494
  `but its connector says "${known.effect}"${known.idempotent ? " (idempotent)" : ""}`,
@@ -0,0 +1,4 @@
1
+ // `@sanoma-ai/workflows/resources`: what a data file imports from the runtime: `secret`, which
2
+ // names a stored secret, and `sanoma`, the constructors of its people types (`sanoma.person`, …).
3
+ export { secret, type Secret } from "./secret.ts";
4
+ export { sanoma } from "./person.ts";
@@ -1,20 +1,73 @@
1
+ import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto";
1
2
  import { setTimeout as sleep } from "node:timers/promises";
2
3
  import { Client } from "pg";
3
4
  import { credentialsOf } from "./env.ts";
4
5
  import { CREDENTIALS_TABLE as TABLE, type Db, prepareDatabase } from "./db.ts";
5
6
  import { info, warn } from "./log.ts";
6
7
  import type { Driver } from "./op.ts";
8
+ import type { Secrets } from "./secret.ts";
7
9
  import { errorMessage } from "./shared.ts";
8
10
 
9
11
  // Where the credentials set from the app live: one table in the config's database, beside DBOS's
10
12
  // own (made by `prepareDatabase` in db.ts). A value is read here only, by the worker to load it
11
- // into its environment and by the client to check it, and leaves neither. The write
12
- // (`storeCredential`) and the read (`storedCredentials`) are the only places that touch `value`,
13
- // so a hosted deployment's encryption goes there.
13
+ // into its environment or hand it to a driver (`storedSecrets`) and by the client to check it,
14
+ // and leaves none of them. The write (`storeCredential`, `ensureCredential`) and the read
15
+ // (`storedCredentials`) are the only places that touch `value`, which holds it encrypted
16
+ // (`encrypt`) under `SANOMA_SECRETS_KEY`.
14
17
 
15
18
  /** Where a change is announced, with the variable's name as the payload. */
16
19
  const CHANNEL = "sanoma_credentials";
17
20
 
21
+ /** The variable holding the key the stored values are encrypted with: 32 bytes, base64. */
22
+ export const SECRETS_KEY = "SANOMA_SECRETS_KEY";
23
+
24
+ /** What starts an encrypted value; a value without it is a plaintext one stored before encryption, read as it is. */
25
+ const V1 = "v1:";
26
+ const CIPHER = "aes-256-gcm";
27
+ const TAG_BYTES = 16;
28
+
29
+ /** The key from the environment, or an error naming the variable. */
30
+ function secretsKey(): Buffer {
31
+ const text = process.env[SECRETS_KEY];
32
+ if (!text) {
33
+ throw new Error(
34
+ `${SECRETS_KEY} is not set: it encrypts the stored credentials; set it to 32 random bytes, base64 (\`openssl rand -base64 32\`)`,
35
+ );
36
+ }
37
+ const key = Buffer.from(text, "base64");
38
+ if (key.length !== 32) throw new Error(`${SECRETS_KEY} must be 32 bytes, base64; it is ${key.length} bytes`);
39
+ return key;
40
+ }
41
+
42
+ /**
43
+ * `name`'s value encrypted for the table: `v1:<iv, base64>:<ciphertext and tag, base64>`,
44
+ * AES-256-GCM, with the name as associated data, so a value moved to another row does not decrypt.
45
+ */
46
+ export function encrypt(name: string, value: string): string {
47
+ const iv = randomBytes(12);
48
+ const cipher = createCipheriv(CIPHER, secretsKey(), iv).setAAD(Buffer.from(name));
49
+ const sealed = Buffer.concat([cipher.update(value, "utf8"), cipher.final(), cipher.getAuthTag()]);
50
+ return `${V1}${iv.toString("base64")}:${sealed.toString("base64")}`;
51
+ }
52
+
53
+ /** The value `name`'s row holds: decrypted, or a plaintext one as it is (it is encrypted on its next write). */
54
+ export function decrypt(name: string, stored: string): string {
55
+ if (!stored.startsWith(V1)) return stored;
56
+ const key = secretsKey();
57
+ const [iv = "", body = ""] = stored.slice(V1.length).split(":");
58
+ const sealed = Buffer.from(body, "base64");
59
+ try {
60
+ const decipher = createDecipheriv(CIPHER, key, Buffer.from(iv, "base64")).setAAD(Buffer.from(name));
61
+ decipher.setAuthTag(sealed.subarray(-TAG_BYTES));
62
+ return Buffer.concat([decipher.update(sealed.subarray(0, -TAG_BYTES)), decipher.final()]).toString("utf8");
63
+ } catch (err) {
64
+ throw new Error(
65
+ `the stored ${name} cannot be decrypted: ${SECRETS_KEY} is not the key it was stored with, or the row was not stored as ${name}`,
66
+ { cause: err },
67
+ );
68
+ }
69
+ }
70
+
18
71
  /** The stored rows of these names, values included: for loading into an environment or checking, never for showing. */
19
72
  export async function storedCredentials(
20
73
  db: Db,
@@ -24,7 +77,9 @@ export async function storedCredentials(
24
77
  `SELECT name, value, set_by, set_at FROM ${TABLE} WHERE name = ANY($1)`,
25
78
  [names],
26
79
  );
27
- return new Map(rows.map((r) => [r.name, { value: r.value, setBy: r.set_by, setAt: r.set_at.toISOString() }]));
80
+ return new Map(
81
+ rows.map((r) => [r.name, { value: decrypt(r.name, r.value), setBy: r.set_by, setAt: r.set_at.toISOString() }]),
82
+ );
28
83
  }
29
84
 
30
85
  /** Upserts the value and announces it, in one statement, so a worker never hears of a change it cannot read. */
@@ -36,10 +91,40 @@ export async function storeCredential(
36
91
  `WITH stored AS (INSERT INTO ${TABLE} (name, value, set_by) VALUES ($1, $2, $3) ` +
37
92
  "ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value, set_by = EXCLUDED.set_by, set_at = now() " +
38
93
  `RETURNING name) SELECT pg_notify('${CHANNEL}', name) FROM stored`,
39
- [name, value, setBy],
94
+ [name, encrypt(name, value), setBy],
40
95
  );
41
96
  }
42
97
 
98
+ /**
99
+ * The stored value of `name`; when there is none, `make()`'s, stored first and announced. Of two
100
+ * at once, the first stored wins, and both return it.
101
+ */
102
+ export async function ensureCredential(
103
+ db: Db,
104
+ { name, make, setBy }: { name: string; make: () => string; setBy: string },
105
+ ): Promise<string> {
106
+ const stored = (await storedCredentials(db, [name])).get(name);
107
+ if (stored) return stored.value;
108
+ await db.query(
109
+ `WITH stored AS (INSERT INTO ${TABLE} (name, value, set_by) VALUES ($1, $2, $3) ON CONFLICT (name) DO NOTHING ` +
110
+ `RETURNING name) SELECT pg_notify('${CHANNEL}', name) FROM stored`,
111
+ [name, encrypt(name, make()), setBy],
112
+ );
113
+ return (await storedCredentials(db, [name])).get(name)!.value;
114
+ }
115
+
116
+ /**
117
+ * The secrets store a live driver is given (`CallContext.secrets`): a variable the environment
118
+ * sets wins, as it does for a driver's `env`; else the stored value. `ensure` stores what it
119
+ * makes as `setBy`.
120
+ */
121
+ export function storedSecrets(db: () => Promise<Db>, setBy: string): Secrets {
122
+ return {
123
+ get: async (name) => process.env[name] || (await storedCredentials(await db(), [name])).get(name)?.value,
124
+ ensure: async (name, make) => process.env[name] || ensureCredential(await db(), { name, make, setBy }),
125
+ };
126
+ }
127
+
43
128
  /** Deletes the row, announcing it when there was one. */
44
129
  export async function deleteCredential(db: Db, name: string): Promise<void> {
45
130
  await db.query(
package/src/data.ts ADDED
@@ -0,0 +1,10 @@
1
+ // `@sanoma-ai/workflows/data`: what a company's `resources/data.ts` declares its collections with.
2
+ export {
3
+ collection,
4
+ type Collection,
5
+ type Field,
6
+ type FieldType,
7
+ type RecordOf,
8
+ type RequiredTextKeys,
9
+ } from "./collection.ts";
10
+ export type { ListQuery, Saved } from "./shared.ts";