@sanoma-ai/workflows 0.10.3 → 0.10.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/CLOUD-IDENTITY.md +128 -17
  2. package/README.md +15 -4
  3. package/dist/apply.d.ts +11 -4
  4. package/dist/apply.js +126 -35
  5. package/dist/apply.js.map +1 -1
  6. package/dist/call.js +1 -1
  7. package/dist/call.js.map +1 -1
  8. package/dist/cloud-identity.d.ts +5 -1
  9. package/dist/cloud-identity.js +13 -77
  10. package/dist/cloud-identity.js.map +1 -1
  11. package/dist/config.d.ts +3 -0
  12. package/dist/config.js +3 -2
  13. package/dist/config.js.map +1 -1
  14. package/dist/define.d.ts +9 -2
  15. package/dist/define.js +1 -0
  16. package/dist/define.js.map +1 -1
  17. package/dist/dns.d.ts +20 -0
  18. package/dist/dns.js +26 -0
  19. package/dist/dns.js.map +1 -0
  20. package/dist/drift.d.ts +4 -2
  21. package/dist/drift.js +13 -4
  22. package/dist/drift.js.map +1 -1
  23. package/dist/identity-broker.d.ts +14 -0
  24. package/dist/identity-broker.js +84 -0
  25. package/dist/identity-broker.js.map +1 -0
  26. package/dist/identity-provisioning.d.ts +29 -0
  27. package/dist/identity-provisioning.js +178 -0
  28. package/dist/identity-provisioning.js.map +1 -0
  29. package/dist/identity-service.d.ts +25 -0
  30. package/dist/identity-service.js +148 -0
  31. package/dist/identity-service.js.map +1 -0
  32. package/dist/index.d.ts +2 -1
  33. package/dist/index.js +1 -0
  34. package/dist/index.js.map +1 -1
  35. package/dist/op.d.ts +15 -8
  36. package/dist/op.js.map +1 -1
  37. package/dist/provisioning-identity.d.ts +27 -0
  38. package/dist/provisioning-identity.js +81 -0
  39. package/dist/provisioning-identity.js.map +1 -0
  40. package/dist/resource.d.ts +67 -6
  41. package/dist/resource.js +95 -9
  42. package/dist/resource.js.map +1 -1
  43. package/dist/resources.d.ts +3 -3
  44. package/dist/resources.js +6 -5
  45. package/dist/resources.js.map +1 -1
  46. package/dist/runtime-ops.d.ts +60 -1
  47. package/dist/runtime-ops.js +73 -1
  48. package/dist/runtime-ops.js.map +1 -1
  49. package/dist/scenario.d.ts +6 -3
  50. package/dist/scenario.js +13 -1
  51. package/dist/scenario.js.map +1 -1
  52. package/oxlint.json +1 -0
  53. package/package.json +5 -1
  54. package/src/apply.ts +165 -47
  55. package/src/call.ts +2 -2
  56. package/src/cloud-identity.ts +16 -76
  57. package/src/config.ts +6 -2
  58. package/src/define.ts +10 -2
  59. package/src/dns.ts +29 -0
  60. package/src/drift.ts +13 -4
  61. package/src/identity-broker.ts +94 -0
  62. package/src/identity-provisioning.ts +208 -0
  63. package/src/identity-service.ts +179 -0
  64. package/src/index.ts +2 -0
  65. package/src/op.ts +13 -3
  66. package/src/provisioning-identity.ts +106 -0
  67. package/src/resource.ts +147 -15
  68. package/src/resources.ts +6 -5
  69. package/src/runtime-ops.ts +79 -0
  70. package/src/scenario.ts +24 -6
package/CLOUD-IDENTITY.md CHANGED
@@ -1,8 +1,8 @@
1
- # Managed Cloud runtime identity
1
+ # Managed Cloud identity
2
2
 
3
3
  `@sanoma-ai/workflows/cloud-identity` supplies the identity-broker handler and refreshable Google SDK authentication for [D1, #492](https://github.com/Sanoma-AI/sanoma/issues/492). It is a reusable implementation boundary. Isolated live checks on a temporary Cloud Run service in `sanoma-513217` have verified attached-identity issuance, scoped KMS use and separate database permissions; production deployment remains open.
4
4
 
5
- This runtime route is separate from the provisioning OIDC identity in [#489](https://github.com/Sanoma-AI/sanoma/issues/489). Provisioning manages enrollment and infrastructure. The company runtime receives only its deployment's bootstrap credential and short-lived access tokens. The broker issues identity; it does not read company credential rows, decrypt credentials or unwrap data keys.
5
+ Runtime and provisioning identity are separate trust boundaries. `identityBroker` issues ten-minute OAuth access tokens for company runtimes. `provisioningIdentityBroker` supplies Google-issued OIDC tokens for the control plane’s Workload Identity Federation (WIF) flow. Deploy them with separate attached accounts, enrollment databases/readers and bootstrap credentials. Sharing handler code does not establish that separation. Neither handler reads company credential rows or unwraps data keys.
6
6
 
7
7
  ## Request and authority
8
8
 
@@ -39,7 +39,7 @@ GRANT USAGE ON SCHEMA sanoma_identity
39
39
  TO identity_broker_reader, identity_enrollment_writer;
40
40
  GRANT SELECT ON sanoma_identity.deployments, sanoma_identity.credentials
41
41
  TO identity_broker_reader;
42
- GRANT SELECT, INSERT, UPDATE, DELETE
42
+ GRANT SELECT, INSERT, UPDATE
43
43
  ON sanoma_identity.deployments, sanoma_identity.credentials
44
44
  TO identity_enrollment_writer;
45
45
  -- Separate pre-created logins, neither superuser nor schema/table owner:
@@ -49,24 +49,39 @@ GRANT identity_enrollment_writer TO example_provisioner_login;
49
49
 
50
50
  The broker login must have no writer-role membership, inherited write grants or authority to grant itself privileges. The provisioning login stays outside company runtimes and outside the broker's serving process. Database connection/authentication is separate bootstrap material; it cannot be fetched from the same service it unlocks.
51
51
 
52
- Trusted provisioning calls `createDeploymentCredential()` to generate 32 random bytes encoded as base64url plus its SHA-256 hash. Deliver the credential once through that deployment's protected secret channel; persist only its hash. Enrollment writes should be parameterized. For example, with synthetic deployment identifiers and the generated hash supplied as `$1`:
52
+ ### Provisioning operations
53
53
 
54
- ```sql
55
- BEGIN;
56
- INSERT INTO sanoma_identity.deployments
57
- (id, company, environment, service_account)
58
- VALUES ('example-deployment', 'example-company', 'production',
59
- 'runtime@example-project.iam.gserviceaccount.com');
60
- INSERT INTO sanoma_identity.credentials
61
- (hash, deployment, generation, valid_from, valid_until)
62
- VALUES ($1, 'example-deployment', 1, clock_timestamp(),
63
- clock_timestamp() + interval '1 day');
64
- COMMIT;
54
+ The trusted control-plane config supplies `identityProvisioning`; company runtimes and the serving broker do not receive this writer. Install `CLOUD_IDENTITY_SQL` separately with administrative authority, then construct the writer over its dedicated database pool:
55
+
56
+ ```ts
57
+ import { postgresIdentityProvisioning } from "@sanoma-ai/workflows/cloud-identity";
58
+
59
+ const identityProvisioning = postgresIdentityProvisioning(async () => enrollmentWriterPool, {
60
+ maxValidityMs: 24 * 60 * 60 * 1000,
61
+ });
62
+ // Supply identityProvisioning to defineConfig in the trusted control plane.
63
+ // Deployment composition owns the writer pool and closes it on shutdown.
65
64
  ```
66
65
 
67
- The interval is illustrative. The SQL schema requires finite timestamps and checks that `valid_until` follows `valid_from`; provisioning must enforce its maximum validity and operational renewal policy. There is no HTTP enrollment endpoint.
66
+ The one-day limit is illustrative; choose a validity and renewal policy for the deployment. `maxValidityMs` is required and enforced against the enrollment database clock. The adapter does not install schemas, create roles, provision IAM, deliver credentials or schedule renewal.
67
+
68
+ Workflows declare the built-in `sanoma.identity.enroll`, `.retire` and `.revoke` operations in `uses`. They pass through the existing policy and ledger path; direct calls are refused. All three identify the same fixed mapping: `deployment`, `company`, `environment` and `serviceAccount`. The workflow and policy must authorize the actor against that company and environment before granting access.
68
69
 
69
- For rotation, provision a new random credential under a higher unused generation, deliver it to the deployment, and set the old generation's `valid_until` to a bounded overlap deadline. Revoke a generation with `revoked = true`, or disable the deployment to prevent renewal for all its generations. Do not repoint an existing deployment to another company's service account as a credential-rotation shortcut. Restore enrollment backups carefully so revoked credentials do not become active again.
70
+ | Operation | Additional input | Result and behavior |
71
+ | --------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
72
+ | `enroll` | `generation`, `validUntil`, `store` | Ensures a random bootstrap credential in the selected secret store, enrolls its SHA-256 hash and returns `{ deployment, generation, store }`. No bearer or hash enters the workflow result. |
73
+ | `retire` | `generation`, `validUntil` | Shortens a generation's validity without extending it; a deadline at or before the database's current time revokes it. Returns `{}`. |
74
+ | `revoke` | None | Disables renewal for every generation of the deployment. This is terminal for that deployment ID. Returns `{}`. |
75
+
76
+ Use a distinct secret name for each deployment and generation, such as `ACME_PRODUCTION_BOOTSTRAP_G1`. A higher generation must use a new credential; reusing an enrolled hash is refused. Neither rotation nor retry may change the deployment's company, environment or service account.
77
+
78
+ Enrollment writes take a deployment-row lock, so concurrent enrollment, retirement and revocation serialize. A retry with the same mapping, generation, hash and still-valid deadline succeeds. A changed or retired deadline, revoked credential, disabled deployment, reused hash or lower unused generation is refused. If the database response is lost after commit, retry with the original inputs and secret name; the stored credential is retained. Do not delete or replace that secret while the outcome is uncertain. A failed enrollment can leave an inactive secret; reconcile the enrollment before removing it.
79
+
80
+ Rotation consists of enrolling a higher generation under a new secret name, securely delivering it, verifying renewal with it, then retiring the old generation with a bounded overlap deadline. These operations do not perform delivery or readiness verification. Revocation stops further issuance, but already issued Google tokens can remain usable until their expiry.
81
+
82
+ Database failures are sanitized and classified as unavailable; invalid inputs and conflicting enrollment state are not retryable. There is no HTTP enrollment endpoint. Sandbox runs use separate in-memory enrollment authority and never touch the live enrollment database; they do not validate the deployment-specific maximum validity.
83
+
84
+ Restore enrollment backups carefully so revoked credentials do not become active again. Bootstrap values remain in the selected secret store and the deployment's protected delivery channel; the enrollment database stores only their hashes.
70
85
 
71
86
  Broker composition uses its read-only database pool and a separate durable audit sink:
72
87
 
@@ -138,3 +153,99 @@ Before rollout:
138
153
  - Deliver each bootstrap credential only to its intended deployment. For Cloudflare, explicitly pass its protected Worker binding to that company's container. Prove cold start, resume, refresh through token expiry, bounded credential rotation, revoked/expired generations and broker/audit outages.
139
154
  - Prove company A's bearer cannot select B via body/query, A's token cannot use B's key, A's database login cannot read B's rows, and preview cannot access production. Synthetic handler tests are insufficient evidence for Google IAM or database isolation.
140
155
  - Verify issuance and KMS audit attribution, credential/header redaction, service quotas, backup recovery and retained key versions. Record the actual deployment evidence before declaring D1 or managed Cloud ready.
156
+
157
+ ## Provisioning identity from a non-Google host
158
+
159
+ The trusted control plane can obtain renewable provisioning credentials without a service-account private key:
160
+
161
+ ```mermaid
162
+ flowchart LR
163
+ control[Trusted control plane] -->|GET /identity + provisioning bearer| issuer[Separate Cloud Run provisioning issuer]
164
+ issuer -->|Read fixed enrollment| db[(Provisioning enrollment database)]
165
+ issuer -->|Attached identity: generateIdToken| google[Google IAM Credentials]
166
+ google -->|OIDC token for fixed source account| issuer
167
+ issuer -->|idToken + expiresAt| control
168
+ control -->|OIDC token| sts[Google STS / WIF]
169
+ sts -->|Federated credential| target[Impersonate provisioning service account]
170
+ target -->|Scoped access token| provider[Google OpenTofu provider]
171
+ ```
172
+
173
+ Three identities have different responsibilities:
174
+
175
+ - The **attached issuer account** has `iam.serviceAccounts.getOpenIdToken` on the fixed source account only.
176
+ - The **source account** identifies the control plane to WIF through its immutable numeric Google `uniqueId`; it needs no infrastructure-administration roles.
177
+ - The **provisioning account** receives scoped infrastructure permissions and grants impersonation only to the intended WIF principal.
178
+
179
+ Trusted `ProvisioningIdentity` configuration fixes the source `serviceAccount`, numeric `subject` and exact WIF provider `audience`. Configure WIF for issuer `https://accounts.google.com`, map `google.subject` from `assertion.sub`, and restrict trust to the intended subject and audience. The source and target provisioning accounts are different identities.
180
+
181
+ `provisioningIdentityBroker` accepts **GET `/identity`** with a provisioning bearer, no query and no body. Its enrollment must map to the configured source account. The caller cannot select an account, audience or lifetime. It checks enrollment before issuance and before delivery; audit failure prevents delivery. Responses contain only `{ idToken, expiresAt }` and prohibit caching.
182
+
183
+ `attachedGoogleIdTokenIssuer(identity)` uses the attached `Compute` identity to call [Google’s generateIdToken API](https://docs.cloud.google.com/iam/docs/reference/credentials/rest/v1/projects.serviceAccounts/generateIdToken). It validates returned claims and expiry; Google STS verifies the token signature during federation. This trusted callback does not accept caller-supplied JWTs.
184
+
185
+ ```ts
186
+ import {
187
+ provisioningIdentityBroker,
188
+ attachedGoogleIdTokenIssuer,
189
+ postgresIdentityEnrollments,
190
+ } from "@sanoma-ai/workflows/cloud-identity";
191
+
192
+ const identity = {
193
+ serviceAccount: "provision-source@example-project.iam.gserviceaccount.com",
194
+ subject: "123456789012345678901",
195
+ audience:
196
+ "//iam.googleapis.com/projects/123456789012/locations/global/workloadIdentityPools/provision/providers/google",
197
+ };
198
+ const handle = provisioningIdentityBroker({
199
+ identity,
200
+ enrollments: postgresIdentityEnrollments(async () => provisioningReadOnlyPool),
201
+ issue: attachedGoogleIdTokenIssuer(identity),
202
+ audit: writeDurableProvisioningAudit,
203
+ });
204
+ ```
205
+
206
+ The pool and audit sink belong to this separate service. Use the GCP connector’s `gcpProvisioningIdentity` helper to supply this endpoint as an external-account URL credential source to both the Node verifier and the Google provider. Its protected header contains a provisioning bearer; never put the resulting configuration in source control, workflow inputs/results or logs. The target account still comes from trusted deployment configuration.
207
+
208
+ Google-issued OIDC tokens last up to **one hour**. A ten-minute final impersonated access token does not shorten that lifetime. Revoking enrollment stops new issuance, but an issued OIDC or federated credential may still obtain access tokens. Incident revocation must also remove federation and/or target impersonation trust, accounting for IAM propagation and access tokens already issued. These helpers do not implement that IAM revocation workflow. The runtime route remains POST `/token` with ten-minute OAuth access tokens.
209
+
210
+ ### Evidence and remaining integration
211
+
212
+ An isolated live check in `sanoma-513217` verified Google-issued OIDC tokens, STS exchange, scoped impersonation and rejection of another subject and target. The observed OIDC lifetime was 3,600 seconds. Temporary federation and service accounts were removed. Local tests cover both broker routes and Google’s URL-source exchange using synthetic transports.
213
+
214
+ This establishes the Google trust path, not a deployed provisioning broker or provider renewal through that broker. Deploying the separate issuer/enrollment/audit boundary, delivering and rotating protected bootstrap configuration, and proving renewal and revocation through the actual host remain [#532](https://github.com/Sanoma-AI/sanoma/issues/532).
215
+
216
+ ## Standalone identity service
217
+
218
+ `sanoma identity` serves one broker mode from protected deployment environment variables. It loads no company config, `.env`, workflows or sites, starts no worker, and creates no database or schema. Deploy runtime and provisioning modes as separate services with separate attached identities, enrollment readers, bootstraps and audit boundaries; a mode flag alone does not establish that separation.
219
+
220
+ | Setting | Purpose |
221
+ | ------------------------------------ | ------------------------------------------------------------------------------------------------------------------- |
222
+ | `SANOMA_IDENTITY_MODE` | Required: `runtime` or `provisioning`. |
223
+ | `SANOMA_IDENTITY_READ_DATABASE_URL` | Required protected enrollment-reader connection string. |
224
+ | `SANOMA_IDENTITY_AUDIT_DATABASE_URL` | Required protected audit-writer connection string. |
225
+ | `SANOMA_IDENTITY_HOST` | Listening address, default `0.0.0.0` for the command. |
226
+ | `SANOMA_IDENTITY_CONCURRENCY` | Maximum in-flight broker requests, default `32`, allowed `1`–`256`; excess requests receive 429 with a retry delay. |
227
+ | `PORT` | Listening port, default `8080`. |
228
+ | `SANOMA_IDENTITY_SOURCE_ACCOUNT` | Provisioning only: fixed source service-account email. |
229
+ | `SANOMA_IDENTITY_SOURCE_SUBJECT` | Provisioning only: its immutable numeric Google `uniqueId`. |
230
+ | `SANOMA_IDENTITY_AUDIENCE` | Provisioning only: exact WIF provider audience. |
231
+
232
+ Supply connection strings through the host's protected deployment mechanism. Never put them in source, resource plans, workflow inputs/results or logs. Runtime mode serves POST `/token`; provisioning mode serves GET `/identity`. Both require the corresponding enrollment bearer, use an attached Google identity and expose no enrollment endpoint. Put the HTTP listener behind HTTPS ingress, such as Cloud Run's TLS endpoint.
233
+
234
+ An administrator installs `CLOUD_IDENTITY_SQL` from `@sanoma-ai/workflows/cloud-identity` in the enrollment database and `CLOUD_IDENTITY_AUDIT_SQL` from `@sanoma-ai/workflows/identity-service` in the audit database. The service performs no DDL. The enrollment login needs schema USAGE and SELECT on the two enrollment tables, with no table mutation or schema CREATE privileges. The audit login needs schema USAGE and INSERT on `sanoma_identity.audit_events`, with no SELECT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER or schema CREATE privileges. For example, in the audit database:
235
+
236
+ ```sql
237
+ CREATE ROLE identity_audit_writer NOLOGIN;
238
+ GRANT USAGE ON SCHEMA sanoma_identity TO identity_audit_writer;
239
+ GRANT INSERT ON sanoma_identity.audit_events TO identity_audit_writer;
240
+ GRANT identity_audit_writer TO example_audit_login;
241
+ ```
242
+
243
+ Create the login and deliver its authentication separately. Neither serving role may be superuser, create roles/databases, bypass RLS or inherit the disallowed privileges. Startup checks these role and table/schema permissions and refuses excessive authority. Keep schema ownership and enrollment writes with separate administrative/provisioning identities.
244
+
245
+ Audit rows contain fixed issuance metadata: result, deployment, company, environment, service account, generation and expiry. They contain no tokens, bearer hashes, authorization headers or arbitrary request fields. Audit failure prevents token delivery and returns a sanitized error; recovery never switches identities.
246
+
247
+ GET or HEAD `/healthz` and the launcher's `ready` event mean the process is listening. They do **not** prove current Google issuance, continuing database/audit availability, provider renewal or managed-store readiness. Before declaring a company ready, verify its deployed runtime can write and read a credential under its own key/database identity. The two-company, preview/production, renewal, cold-start, outage and recovery checks remain #532 gates.
248
+
249
+ Trusted programmatic composition may call `startIdentityService(options)` from `@sanoma-ai/workflows/identity-service`, returning `{ url, stop }`. Its default host is loopback; the caller owns shutdown. The serving package does not load company code.
250
+
251
+ The product's [identity Dockerfile](../sanoma/identity/Dockerfile) installs a specified published `sanoma` version and runs `sanoma identity` as a non-root user. Build it only after that version is published, tag the result immutably and deploy by digest. It is separate from the company runtime image, which clones company code. Database credentials and mode configuration enter at deployment, not image build. Neither the service tests nor building the image substitutes for live deployment verification.
package/README.md CHANGED
@@ -53,7 +53,7 @@ export const shop = withSetup(defineConnector("shop", specs, info), {
53
53
 
54
54
  `withSetup` lays what it is given over what the connector already carries, and keeps it on the connector's vendor record, so a spread copy of the connector keeps it too. A config that needs a driver made otherwise (another bridge, other options) swaps the driver and keeps the fake: `withSetup(github, { driver: () => githubDriver({ bridge }) })`. It is for connector packages and configs, not for workflows, which the lint does not let import it. Both are optional: a connector without a driver declares its operations and implements none, so a workflow using one is refused; a connector without a fake has none for sandbox runs, which the app's connector page says.
55
55
 
56
- A connector that signs visitors in to a public site attaches a login half too, as `login`: `withSetup(connector, { login: (context) => half })`. The app calls it once per public site whose `site.ts` names the connector (`defineSite({ public: true, login: "<vendor>" })`), when it starts, with a `LoginContext` (the database, a key of its own derived from `SANOMA_SIGNING_KEY` to sign its cookies with, a vendor's secrets, the site's origins, and `sendCode(email, code)` when the config names a `login.sender`); a login that keeps tables in the database migrates them there, serialized and idempotent. The `LoginHalf` it makes (see its type) answers the provider's stable subject for a request, with the email when it verified one, never a principal: the app makes the visitor's principal from it, so a connector cannot set a visitor's groups. A visitor's address, for rate limits, is the `CLIENT_IP_HEADER` header the app sets, never `X-Forwarded-For`. One vendor's connectors attach one `login`, as for a driver. [`@sanoma-ai/connector-better-auth`](https://www.npmjs.com/package/@sanoma-ai/connector-better-auth) is one, email and password (and an email code, with a sender) on the company's database; it also exports `betterAuthStaffLogin`, the staff sign-in the app uses on a host (`LoginContext.staff`: who is declared, the GitHub client, and where a sign-in is recorded). The app makes it as it starts and sends it every request to the site's own host ([public sites](https://www.npmjs.com/package/@sanoma-ai/app#public-sites)).
56
+ A connector that signs visitors in to a public site attaches a login half too, as `login`: `withSetup(connector, { login: (context) => half })`. The app calls it once per public site whose `site.ts` names the connector (`defineSite({ public: true, login: "<vendor>" })`), when it starts, with a `LoginContext` (the database, a key of its own derived from `SANOMA_SIGNING_KEY` to sign its cookies with, a vendor's secrets, the site's origins, `sendCode(email, code)` when the config names a `login.sender`, and `social`, the company's GitHub App's OAuth client when the app is given it); a login that keeps tables in the database migrates them there, serialized and idempotent. The `LoginHalf` it makes (see its type) answers the provider's stable subject for a request, with the email and the GitHub login when it verified them, never a principal: the app makes the visitor's principal from it, so a connector cannot set a visitor's groups. A visitor's address, for rate limits, is the `CLIENT_IP_HEADER` header the app sets, never `X-Forwarded-For`. One vendor's connectors attach one `login`, as for a driver. [`@sanoma-ai/connector-better-auth`](https://www.npmjs.com/package/@sanoma-ai/connector-better-auth) is one, email and password (and an email code, with a sender) on the company's database; it also exports `betterAuthStaffLogin`, the staff sign-in the app uses on a host (`LoginContext.staff`: who is declared, the GitHub client, and where a sign-in is recorded). The app makes it as it starts and sends it every request to the site's own host ([public sites](https://www.npmjs.com/package/@sanoma-ai/app#public-sites)).
57
57
 
58
58
  A model connector attaches a model half, as `model`: `withSetup(connector, { model: (context) => provider })`, an AI SDK provider (`ProviderV4` from `@ai-sdk/provider`) with the vendor's adapter as the transport. The app makes it for its model registry, once per resolved config and client (again after one that throws), where its models are `<vendor>/<model-id>`; it answers `languageModel("default")` with the connector's own default. Its `ModelContext` gives it the secrets as a login's does (the selected store and explicit bindings), which it reads when a model is called; its driver declares the key in `env`, with no operations if it needs none. One vendor's connectors attach one `model`. [`@sanoma-ai/connector-anthropic`](https://www.npmjs.com/package/@sanoma-ai/connector-anthropic) is one.
59
59
 
@@ -233,7 +233,7 @@ A run's ctx calls run one at a time, in program order. Even inside `Promise.all`
233
233
 
234
234
  A workflow gets these on `ctx` by listing them in `uses`, next to its operations: `"approval"` (see [Approvals](#approvals)), `"sleep"` and `"all"`. `describeConfig` lists them as each workflow's `builtins`.
235
235
 
236
- Every workflow has `ctx.runId`, `ctx.startedBy` and `ctx.now()`, without `uses`. `ctx.startedBy` is who started the run, `{ id, email?, subject? }` (`StartedBy`): the principal's id (a person's, `visitor:<login>:<subject>`, `token:<id>`), for a visitor the email their site's login verified, and for an api token the subject its row acts for (`SanomaClient.tokens`, read at the start, never taken from the caller), so a workflow can tell whose a record is. `await ctx.now()` is the time in epoch milliseconds, recorded so a replay reads the same. A date is made from it: with `const now = await ctx.now()`, `new Date(now).toISOString()` is ISO 8601 text, which a collection's `date` field takes (`.slice(0, 10)` for the day alone), and `new Date(now + 86_400_000)` is a day later, from the same reading. The lint refuses only the uses of `Date` that read the clock, `Date.now()`, `new Date()` and `Date()` (and `Date` passed on, which it cannot follow); `new Date(value)`, `Date.parse`, `Date.UTC` and `instanceof Date` read none (see [Keeping workflows replay-safe](#keeping-workflows-replay-safe)). Read a date in UTC (`toISOString`, `getUTCHours`) and parse ISO 8601 text with an offset: the local-time methods (`getHours`, `toLocaleString`) and other text depend on the worker's time zone, which a replay on another worker may not share.
236
+ Every workflow has `ctx.runId`, `ctx.startedBy` and `ctx.now()`, without `uses`. `ctx.startedBy` is who started the run, `{ id, email?, github?, subject? }` (`StartedBy`): the principal's id (a person's, `visitor:<login>:<subject>`, `token:<id>`), for a visitor the email and the GitHub login their site's login verified (the login when they signed in with GitHub or linked it, with the company's GitHub App's client), and for an api token the subject its row acts for (`SanomaClient.tokens`, read at the start, never taken from the caller), so a workflow can tell whose a record is. `await ctx.now()` is the time in epoch milliseconds, recorded so a replay reads the same. A date is made from it: with `const now = await ctx.now()`, `new Date(now).toISOString()` is ISO 8601 text, which a collection's `date` field takes (`.slice(0, 10)` for the day alone), and `new Date(now + 86_400_000)` is a day later, from the same reading. The lint refuses only the uses of `Date` that read the clock, `Date.now()`, `new Date()` and `Date()` (and `Date` passed on, which it cannot follow); `new Date(value)`, `Date.parse`, `Date.UTC` and `instanceof Date` read none (see [Keeping workflows replay-safe](#keeping-workflows-replay-safe)). Read a date in UTC (`toISOString`, `getUTCHours`) and parse ISO 8601 text with an offset: the local-time methods (`getHours`, `toLocaleString`) and other text depend on the worker's time zone, which a replay on another worker may not share.
237
237
 
238
238
  `ctx.sleep({ until })` takes an ISO 8601 date-time with an offset (`2026-10-07T09:00:00Z`) or epoch milliseconds, and a time already past does not wait; `ctx.sleep({ minutes: 5 })` takes a duration from `ms`, `seconds`, `minutes`, `hours` and `days`, none negative. A request it can't read fails the run with `invalid_input`, as does an operation input its schema refuses. Each sleep writes a `sleep.started` record before it waits, with `until`: the time asked for, or for a duration, the record's `at` plus the duration, as the runtime saw it.
239
239
 
@@ -458,6 +458,15 @@ Calling the type declares one resource, which is what a data file does: `reposit
458
458
 
459
459
  A field may name another declared resource where the type's `fields.references` says so, by dotted path, with the type it names and, in `by`, the fields of that resource the vendor may hold there instead: GitHub's `branch_protection` has `{ repository_id: { type: "github.repository", by: ["name", "node_id"] } }`, since GitHub holds a rule's repository by its node id. So `branch_protection({ repository_id: site, pattern: "main" })` takes the repository `site`. It stands for that resource's `name`: `desired` holds `repository_id: "acme/website"` (a GitHub repository's identity is `<owner>/<name>`), which the schema and `find` (the rule is `acme/website:main`) read, and a drift check takes any of the repository's `by` values, read in the same run, as that name; and `refs` says where a resource was named, `{ repository_id: "github.repository:acme/website" }` (a list item's path has its index, `teams.0`). Anywhere else, or of another type, a resource is refused, and the constructor's type (`Declaring`) says the same, so the editor marks it first.
460
460
 
461
+ A reference with `from`, a dotted path in the named resource's state, takes that state instead: apply and drift give the field the resource's state at that path, as a whole (a list or an object), read in the same run or made by an earlier step of it. So a sending domain's DNS records go to the zone that holds them, the record set's type declaring `{ records: { type: "resend.domain", from: "dns_records" } }`:
462
+
463
+ ```ts
464
+ export const mail = resend.domain({ name: "sanoma.work" });
465
+ export const mailDns = cloudflare.records({ zone: "sanoma.work", name: "resend", records: mail });
466
+ ```
467
+
468
+ The resource stands for the whole field, never an item of a list (`records: [mail]` is refused), and a literal value is still allowed there; a reference has `from` or `by`, not both, and is refused at a write-only field or one that holds a write-only field. Resolving a config refuses a `from` path that is no field of a type the reference may name (`checkFromRefs`), naming that type's fields. `desired` holds the resource's `name` in its place, which the schema does not judge, and the declaration is applied after it. Where the same apply changes the resource at that path (creates or replaces it, or updates it with the path in its plan's `unknown` or another value planned there; an update that touches other fields, such as a domain's re-verify, changes nothing here), or, for a create, the resource has no state yet or nothing at that path, the value is known only after apply: the plan is given `null` there, so a type that takes a `from` field must accept `null` in its `plan` and report the path in `unknown`, and its `update` must keep what the vendor holds for a field left out of `desired`, and a change the new value makes (a replaced domain's new DKIM key) is planned, and approved, with the rest; an update or replace step carries those paths in its own `unknown`. A plan of a bound object whose resource is absent, gone or unreadable and not about to change is given its fields without it, as drift compares them, and its step runs without it. A step planned with the field `null` is planned again at its step, with the value now known: `none` (the value turned out unchanged) leaves it done without a write, and otherwise its update expects the fresh plan's action and replace, since the approval covered the change in kind. An empty list or object at the path counts as known: a vendor that answers `[]` while still filling the list makes the dependent write an empty one. At its step the field takes the resource's state as the earlier step left it; one planned `null` whose value is still not known, as when the resource's own step failed or waited, fails before its driver is called: `its records come from resources/mail.ts#mail, which has no state yet` (or `whose state has no dns_records yet`), and so does a step down the chain from it. A policy and an approval therefore see `null` for such a field: its value comes from another declared resource's state at the vendor, known after apply. Drift compares the field against the bound resource's state, read in the same run, and leaves it out while that resource is absent, gone, unreadable or has nothing at the path, which its own row reports. A bridge-backed type (`tfConnector`) cannot take a `from` field yet: an OpenTofu provider's plan reads `null` as unset, not as known after apply. `dnsRecord` (`DnsRecord`), from the main entry, is the shape a DNS record passes in between two connectors: `{ type: "TXT" | "MX" | "CNAME" | "CAA", name, content, priority?, ttl? }`, `name` the full name in lower case with no trailing dot, a TXT record's `content` unquoted (each refused otherwise, never rewritten), and `ttl` 1 for automatic. Later, not yet: the cases apply hard-codes, a wakeup's `host` (its host's `outputs(state).url`) and a host's `source` (a repository's clone URL, by its `by` fields), could be `from` references, once `from` can name a host's outputs as well as its state.
469
+
461
470
  Only declared fields are compared for drift, as CloudFormation does: `compareDeclared(fields, state, desired)` keeps the fields `desired` declares, minus `vendorOwned` and `writeOnly` ones, picking objects field by field, list items against the declared item at the same index, and the items of an `unordered` list (a set) against what any declared item declares, sorted, so a set is compared as a multiset whatever order the vendor returns it in. A drift check compares `normalize(actual, desired)` with `normalize(desired, desired)`, leaf by leaf: `diffDeclared(resource, state, desired, accept?)` returns each declared field that differs, `{ path, desired, actual }`, where `accept` gives, by a reference's path, the values that stand for the resource it names. `withoutWriteOnly(fields, desired)` drops write-only values from a declaration, keeping a secret's marker, which names a stored value and holds none. So a field left out is never drift, which covers attributes an OpenTofu provider marks `computed` and `optional` (GitHub's `etag`, `topics`): they are the user's to set, so not `vendorOwned`, but the vendor fills them in when nobody does.
462
471
 
463
472
  `describeConfig` lists the resource types as `resourceTypes`, each `{ id: "<vendor>.<type>", vendor, type, title, identity, fields, schema, ops }`: `schema` is its state as JSON Schema, with `$id` `sanoma:resource-type/<id>`, which its operations' contracts `$ref` rather than repeat. `@sanoma-ai/workflows/shared` exports the `Resource`, `ResourceSpec`, `ResourceFields`, `References`, `FieldReference`, `Declared`, `Declaring`, `DeclaredResource` and `ResourceProblem` types.
@@ -605,9 +614,9 @@ const runId = await client.apply(
605
614
  const report = await client.applyReport(runId); // { plan: { problems, rebinds, steps }, results, lockfile: { written, revision?, bindings } }
606
615
  ```
607
616
 
608
- The run reads the data files and the lockfile in its first step (`apply:data-files`; a secret's marker stays in `desired`, a plain write-only value does not), with the lockfile's `rebinds`: what changed in it since apply last wrote it (`diffLockfiles` against the bindings the latest live apply run's last `apply:lockfile` step wrote), changes someone else made. It imports each declared resource as drift does (by the binding `bind` gives, else the lockfile's, else its identity), then plans each one `only` wants, each after the resources it names, as a `PlanStep` (`{ id, vendor, type, action, … }`): `bind` (to the object `bind` gives, or the one its identity `found`, by its binding id), `create` (unbound and nothing found: `planned` and `unknown` from its type's `plan`), `update` or `replace` (its bound object as `plan` says, with drift's `fields` and the paths that force a replace), `none`, `gone` (its bound object is no longer there: skipped unless answered, since a recreated database looks fine and has lost its data; a type with `stateless: true`, holding no data, is planned as a `create` instead, unless answered), and `orphan` for each binding no declaration has (skipped unless answered: deleting a declaration never deletes its object). What cannot be planned (a read-only type that is absent or drifted, an answer that cannot be done, an id the input names that nothing declares) is one of the plan's `problems`, with no step.
617
+ The run reads the data files and the lockfile in its first step (`apply:data-files`; a secret's marker stays in `desired`, a plain write-only value does not), with the lockfile's `rebinds`: what changed in it since apply last wrote it (`diffLockfiles` against the bindings the latest live apply run's last `apply:lockfile` step wrote), changes someone else made. It imports each declared resource as drift does (by the binding `bind` gives, else the lockfile's, else its identity), then plans each one `only` wants, each after the resources it names, as a `PlanStep` (`{ id, vendor, type, action, … }`): `bind` (to the object `bind` gives, or the one its identity `found`, by its binding id), `create` (unbound and nothing found: `planned` and `unknown` from its type's `plan`), `update` or `replace` (its bound object as `plan` says, with drift's `fields`, the paths that force a replace, and any `unknown` until it is done), `none`, `gone` (its bound object is no longer there: skipped unless answered, since a recreated database looks fine and has lost its data; a type with `stateless: true`, holding no data, is planned as a `create` instead, unless answered), and `orphan` for each binding no declaration has (skipped unless answered: deleting a declaration never deletes its object). What cannot be planned (a read-only type that is absent or drifted, an answer that cannot be done, an id the input names that nothing declares) is one of the plan's `problems`, with no step.
609
618
 
610
- When the policy would hold any change call the plan makes (asked ahead, in the step `apply:policy`, as each call would ask it), the run asks one approval of the first approver it names, titled "Apply resource changes", covering every change operation in the plan, with the plan as JSON in its `details`; a policy that lets a covered call through once approved (`approvedFor`) then holds none of them again. Then each step runs, in order: `create`, `update` (with `expect`, the plan's action and replace, so a vendor that now plans otherwise refuses with `plan_changed`), `delete`, or only the lockfile changed. A reference to a resource the vendor holds by another field (`by`) is given that resource's binding id, one made earlier in the run included. Right after any step that changes a binding, that binding is changed on the lockfile as it is then (so another run's change made meanwhile stays), written whole and committed through the worker's `source` (`apply:lockfile`: `Apply: create resources/infra.ts#store (supabase.project abcd…)`, by the person who started the run), so an object made is never left unbound. Right after a create's binding is written, the run calls its type's `afterCreate` operations in order, by id, each given the new object's binding id and its export (a Supabase project waits for its health, then stores its database URL); one with `until` is called again every 5 seconds until its output passes, for up to 10 minutes, and one that never does fails the step, its object still bound. The approval covers the `write` ones among them too. Each step's result is `done`, `skipped` (unanswered) or `failed` (with `error`; the run goes on); a host's that it made or changed carries `host`, its `outputs`. A sandbox run writes nothing: the report's `lockfile.bindings` is its copy. `ApplyPlan`, `PlanStep`, `ApplyReport` and `ApplyResult` are the types, also in `@sanoma-ai/workflows/describe` and `/shared`.
619
+ When the policy would hold any change call the plan makes (asked ahead, in the step `apply:policy`, as each call would ask it), the run asks one approval of the first approver it names, titled "Apply resource changes", covering every change operation in the plan, with the plan as JSON in its `details`; a policy that lets a covered call through once approved (`approvedFor`) then holds none of them again. Then each step runs, in order: `create`, `update` (with `expect`, the plan's action and replace, so a vendor that now plans otherwise refuses with `plan_changed`), `delete`, or only the lockfile changed. A reference to a resource the vendor holds by another field (`by`) is given that resource's binding id, one made earlier in the run included, and a `from` reference that resource's state at its path, as found or as an earlier step left it (see [Resources](#resources)). Right after any step that changes a binding, that binding is changed on the lockfile as it is then (so another run's change made meanwhile stays), written whole and committed through the worker's `source` (`apply:lockfile`: `Apply: create resources/infra.ts#store (supabase.project abcd…)`, by the person who started the run), so an object made is never left unbound. Right after a create's binding is written, the run calls its type's `afterCreate` operations in order, by id, each given the new object's binding id and its export (a Supabase project waits for its health, then stores its database URL); one with `until` is called again every 5 seconds until its output passes, for up to 10 minutes, and one that never does fails the step, its object still bound. The approval covers the `write` ones among them too. Each step's result is `done`, `skipped` (unanswered) or `failed` (with `error`; the run goes on); a host's that it made or changed carries `host`, its `outputs`. A sandbox run writes nothing: the report's `lockfile.bindings` is its copy. `ApplyPlan`, `PlanStep`, `ApplyReport` and `ApplyResult` are the types, also in `@sanoma-ai/workflows/describe` and `/shared`.
611
620
 
612
621
  This package knows nothing of OpenTofu. Resource types from an OpenTofu provider are generated by [`@sanoma-ai/bridge/tfschema`](../bridge/src/tfschema/AGENTS.md), which turns the provider's schema document into zod schemas and flagged fields (`resources.gen.ts`); `@sanoma-ai/bridge` depends on this package, never the other way round.
613
622
 
@@ -719,6 +728,8 @@ The steps, as [Cucumber expressions](https://github.com/cucumber/cucumber-expres
719
728
  | Given | `{resource} {string} is gone` | The vendor's fake no longer holds the object: `Given github.repository "docs" is gone` |
720
729
  | Given | `{who} is acting` | Starts the run as that declared person or agent, or `anonymous`: `Given refund-bot is acting` (else `runScenario`'s `startedBy`, `scenarios` by default) |
721
730
  | Given | `a visitor is acting` | Starts the run as a visitor of a public site signed in (`visitor:scenario:visitor`, in `visitors`), which only a `permit` lets start one |
731
+ | Given | `a visitor {string} is acting` | The same visitor with the email its login verified: `Given a visitor "hopper@example.com" is acting` (`ctx.startedBy.email`) |
732
+ | Given | `a visitor {string} with GitHub {string} is acting` | The same, with the GitHub login it linked too: `Given a visitor "hopper@example.com" with GitHub "hopper" is acting` (`ctx.startedBy.github`) |
722
733
  | When | `{workflow} runs with` and a JSON doc string or a two-column table (`name \| value`) | The workflow the scenario runs, and its input |
723
734
  | When | `{workflow} runs` | The same, with the input made up |
724
735
  | Any | `{string} is approved by {who}`, `… with note {string}`, and the `rejected` forms | How the approval with that title (or id, such as `approval-2`) is decided |
package/dist/apply.d.ts CHANGED
@@ -30,7 +30,8 @@ export type ApplyInput = z.infer<typeof ApplyInput>;
30
30
  /**
31
31
  * One declared resource, or orphaned binding, by what apply does with it: `create` it (as the
32
32
  * vendor plans: `planned`, the paths `unknown` until it exists), `update` or `replace` its bound
33
- * object (the declared `fields` that differ, the paths that force the replace), `bind` it to an
33
+ * object (the declared `fields` that differ, the paths that force the replace, any `unknown` until
34
+ * it is done: a `from` field whose resource changes first), `bind` it to an
34
35
  * object its identity `found` or the input `given`, ask about a `gone` object or an `orphan`ed
35
36
  * binding (unless `answer`ed), or `none`.
36
37
  */
@@ -48,6 +49,7 @@ export type PlanStep = {
48
49
  binding: string;
49
50
  fields: DriftField[];
50
51
  replace: string[];
52
+ unknown?: string[];
51
53
  } | {
52
54
  action: "bind";
53
55
  binding: string;
@@ -194,8 +196,13 @@ type ApplyUse = ImportOp | PlanOp | CreateOp | UpdateOp | DeleteOp | ResolveOp |
194
196
  * and right after one that changes a binding, the lockfile written and committed
195
197
  * (`steps.write`), so an object made is never left unbound. An update is given the plan's action
196
198
  * and replace (`expect`), and refuses (`plan_changed`) when the vendor now plans otherwise; a
197
- * reference to a resource made in this run is given its new binding, and a wakeup's `host` the
198
- * URL its host's state gives (`outputs(state).url`), found or made in this run. A failed step is reported
199
- * and the run goes on; a failure that ends the run (`isRunControlError`) ends it.
199
+ * reference to a resource made in this run is given its new binding, a `from` reference that
200
+ * resource's state at its path as an earlier step left it (planned as `null` while this apply
201
+ * changes it there, or a create's while it has none there, so the change is planned and approved
202
+ * up front, and the step planned again once the value is known: done without a write when it
203
+ * plans `none`, else expecting the fresh plan; a step whose resource still has none there, its
204
+ * step failed or not done, fails before its driver is called), and a wakeup's `host` the URL its host's state gives
205
+ * (`outputs(state).url`), found or made in this run. A failed step is reported and the run
206
+ * goes on; a failure that ends the run (`isRunControlError`) ends it.
200
207
  */
201
208
  export declare function applyWorkflow(types: ReadonlyMap<string, Resource>, ops: ReadonlyMap<string, Op>, drivers: ReadonlyMap<string, unknown>, steps: ApplySteps): WorkflowDefinition<readonly ApplyUse[], typeof ApplyInput>;
package/dist/apply.js CHANGED
@@ -2,7 +2,7 @@ import { z } from "zod";
2
2
  import { defineWorkflow } from "./define.js";
3
3
  import { compare, importsOf, observeBy, observed, sourceUrl, } from "./drift.js";
4
4
  import { isRunControlError } from "./errors.js";
5
- import { AFTER_CREATE_CALLS, AFTER_CREATE_SECONDS, afterCreateStuck, fieldPathOf, holderOf, valueAt, } from "./resource.js";
5
+ import { AFTER_CREATE_CALLS, AFTER_CREATE_SECONDS, afterCreateStuck, fieldPathOf, holderOf, same, valueAt, withFrom, } from "./resource.js";
6
6
  import { errorMessage } from "./shared.js";
7
7
  import { SOURCE_RESOLVE } from "./source.js";
8
8
  export { APPLY_WORKFLOW } from "./shared.js";
@@ -51,13 +51,17 @@ function inDependencyOrder(list) {
51
51
  /**
52
52
  * What a change operation is given of a declaration: its fields, where it names a resource the
53
53
  * vendor may hold by another field (`by`, as a repository's node id), that resource's binding id
54
- * when it has one, made in this run or before; a wakeup's `host`, the URL its host answers at
55
- * (`urls`, by the host's declared id) once its state says.
54
+ * when it has one, made in this run or before; where a `from` reference names one, that
55
+ * resource's state at its path, or while it has none there, `null` (`unknown`) or nothing
56
+ * (`drop`), as `missing` says, with those references as `unfilled`; a wakeup's `host`,
57
+ * the URL its host answers at (`urls`, by the host's declared id) once its state says.
56
58
  */
57
- function desiredOf(r, types, bindings, hosts, urls) {
59
+ function desiredOf(r, types, known, missing) {
60
+ const { bindings, hosts, urls, stateOf } = known;
58
61
  const { fields, host, wakeup } = types.get(`${r.vendor}.${r.type}`);
59
62
  const { references = {} } = fields;
60
- const out = structuredClone(r.desired);
63
+ const filled = withFrom(references, r.desired, r.refs, stateOf, missing);
64
+ const out = filled.desired === r.desired ? structuredClone(r.desired) : filled.desired;
61
65
  for (const [at, target] of Object.entries(r.refs)) {
62
66
  const bound = bindings[target]?.id;
63
67
  const place = holderOf(out, at);
@@ -82,8 +86,12 @@ function desiredOf(r, types, bindings, hosts, urls) {
82
86
  place.holder[place.key] = value;
83
87
  }
84
88
  }
85
- return out;
89
+ return { desired: out, unfilled: filled.unfilled };
86
90
  }
91
+ /** Whether two dotted paths name the same field, or one a field inside the other. */
92
+ const overlaps = (a, b) => a === b || a.startsWith(`${b}.`) || b.startsWith(`${a}.`);
93
+ /** Why a step cannot write a field a `from` reference takes: its resource has no state, or nothing at the path. */
94
+ const waitingFor = ({ at, named, from, stated }) => `its ${at} come from ${named}, ${stated ? `whose state has no ${from} yet` : "which has no state yet"}`;
87
95
  /** Why apply cannot `what` (create, update, delete) an object of a type. */
88
96
  const cannotChange = (type, what) => type?.writable
89
97
  ? `apply cannot ${what} a ${type.vendor}.${type.type}: no driver does`
@@ -140,15 +148,26 @@ function decide(r, result, now, input, desired, types, usable) {
140
148
  const step = { ...head, action: "none", binding };
141
149
  return result.status === "drifted" ? { step, problem: cannot("update") } : { step };
142
150
  }
151
+ /** Whether apply makes a declared resource: one unbound (`absent`) or `gone`. */
152
+ const creates = (result) => result.status === "absent" || result.status === "gone";
143
153
  /** The step a `plan` answer makes of a declared resource: a create for one unbound or gone, else an update, a replace or none. */
144
154
  function fromPlan(r, result, p) {
145
155
  const head = { id: r.id, vendor: r.vendor, type: r.type };
146
- const creating = result.status === "absent" || result.status === "gone";
156
+ const creating = creates(result);
147
157
  if (creating && p.action === "create") {
148
158
  return { step: { ...head, action: "create", planned: p.planned ?? {}, unknown: p.unknown } };
149
159
  }
150
160
  if (!creating && (p.action === "update" || p.action === "replace")) {
151
- return { step: { ...head, action: p.action, binding: result.binding, fields: result.fields, replace: p.replace } };
161
+ return {
162
+ step: {
163
+ ...head,
164
+ action: p.action,
165
+ binding: result.binding,
166
+ fields: result.fields,
167
+ replace: p.replace,
168
+ ...(p.unknown.length && { unknown: p.unknown }),
169
+ },
170
+ };
152
171
  }
153
172
  if (!creating && p.action === "none")
154
173
  return { step: { ...head, action: "none", binding: result.binding } };
@@ -261,9 +280,14 @@ function messageOf(step, before, after) {
261
280
  * and right after one that changes a binding, the lockfile written and committed
262
281
  * (`steps.write`), so an object made is never left unbound. An update is given the plan's action
263
282
  * and replace (`expect`), and refuses (`plan_changed`) when the vendor now plans otherwise; a
264
- * reference to a resource made in this run is given its new binding, and a wakeup's `host` the
265
- * URL its host's state gives (`outputs(state).url`), found or made in this run. A failed step is reported
266
- * and the run goes on; a failure that ends the run (`isRunControlError`) ends it.
283
+ * reference to a resource made in this run is given its new binding, a `from` reference that
284
+ * resource's state at its path as an earlier step left it (planned as `null` while this apply
285
+ * changes it there, or a create's while it has none there, so the change is planned and approved
286
+ * up front, and the step planned again once the value is known: done without a write when it
287
+ * plans `none`, else expecting the fresh plan; a step whose resource still has none there, its
288
+ * step failed or not done, fails before its driver is called), and a wakeup's `host` the URL its host's state gives
289
+ * (`outputs(state).url`), found or made in this run. A failed step is reported and the run
290
+ * goes on; a failure that ends the run (`isRunControlError`) ends it.
267
291
  */
268
292
  export function applyWorkflow(types, ops, drivers, steps) {
269
293
  const imports = importsOf(types, ops, drivers);
@@ -316,9 +340,15 @@ export function applyWorkflow(types, ops, drivers, steps) {
316
340
  }
317
341
  }
318
342
  }
319
- // Where each declared host answers, as its state says, found now or made below: a wakeup is given it.
343
+ // Each declared resource's state, found now or made below, for a `from` reference to it, and
344
+ // where each declared host answers, as its state says: a wakeup is given it.
345
+ const states = new Map();
320
346
  const urls = new Map();
321
- const answersAt = (r, state) => {
347
+ // The resources whose step to change them has started and not left a state yet.
348
+ const unsettled = new Set();
349
+ const keep = (r, state) => {
350
+ states.set(r.id, state);
351
+ unsettled.delete(r.id);
322
352
  const url = types.get(`${r.vendor}.${r.type}`)?.host?.outputs(state).url;
323
353
  if (url)
324
354
  urls.set(r.id, url);
@@ -326,7 +356,7 @@ export function applyWorkflow(types, ops, drivers, steps) {
326
356
  for (const resource of resources) {
327
357
  const now = found.get(resource.id);
328
358
  if (now && "state" in now)
329
- answersAt(resource, now.state);
359
+ keep(resource, now.state);
330
360
  }
331
361
  // A host wanted runs the commit its ref names now, resolved before planning: a merge is a change
332
362
  // to it. One bound by the input needs none.
@@ -354,20 +384,55 @@ export function applyWorkflow(types, ops, drivers, steps) {
354
384
  }
355
385
  }
356
386
  const drift = new Map(compare(types, resources, found, revisions).map((result) => [result.id, result]));
357
- const desired = new Map(resources.map((r) => [r.id, desiredOf(r, types, locked, hosts, urls)]));
387
+ // Each declaration's fields as planned, each after the ones it names. A `from` field whose
388
+ // value this apply changes is planned unknown (`null`), so the change it makes is in the plan,
389
+ // and the approval: its resource created or replaced, or updated where the update's plan
390
+ // leaves the path unknown or plans another value there. One whose resource has no state
391
+ // there is `null` in a create's plan, and left out of a bound object's, as drift does, and
392
+ // run so.
393
+ const desired = new Map();
394
+ // For each resource a step changes: whether the change touches a path of its state.
395
+ const touches = new Map();
396
+ // For each step, the `from` fields planned left out: run so, not waited for.
397
+ const dropped = new Map();
398
+ // The steps planned with a `from` field `null`: planned again at their step, once known.
399
+ const awaited = new Set();
358
400
  const planned = [];
401
+ const add = (step, after) => {
402
+ planned.push(step);
403
+ const before = states.get(step.id);
404
+ if (runOf(step) === "create" || step.action === "replace")
405
+ touches.set(step.id, () => true);
406
+ else if (step.action === "update")
407
+ touches.set(step.id, (from) => (step.unknown ?? []).some((path) => overlaps(path, from)) ||
408
+ after === undefined ||
409
+ !same(valueAt(after, from), valueAt(before, from)));
410
+ };
359
411
  for (const resource of inDependencyOrder(resources).filter((r) => wanted(r.id))) {
360
412
  const result = drift.get(resource.id);
413
+ const changesAt = (named, from) => touches.get(named)?.(from) ?? false;
414
+ const missing = (named, from) => creates(result) || changesAt(named, from) ? "unknown" : "drop";
415
+ const stateOf = (named, from) => (changesAt(named, from) ? undefined : states.get(named));
416
+ const { desired: fields, unfilled } = desiredOf(resource, types, { bindings: locked, hosts, urls, stateOf }, missing);
417
+ desired.set(resource.id, fields);
418
+ const left = unfilled.filter((u) => missing(u.named, u.from) === "drop").map((u) => u.at);
419
+ if (left.length)
420
+ dropped.set(resource.id, new Set(left));
421
+ if (left.length < unfilled.length)
422
+ awaited.add(resource.id);
361
423
  const unresolved = revisions.get(resource.id);
362
424
  // A host whose ref could not be resolved has no commit to run (compare says so only for one it found).
363
425
  const next = unresolved && "error" in unresolved && result.status !== "error"
364
426
  ? { problem: problemOf(resource.id, unresolved.error) }
365
- : decide(resource, result, found.get(resource.id), input, desired.get(resource.id), types, usable);
427
+ : decide(resource, result, found.get(resource.id), input, fields, types, usable);
366
428
  if (next.ask !== undefined) {
367
429
  try {
368
- const answer = fromPlan(resource, result, await ctx[resource.vendor][resource.type].plan(next.ask, { connection: resource.connection }));
430
+ const answered = await ctx[resource.vendor][resource.type].plan(next.ask, {
431
+ connection: resource.connection,
432
+ });
433
+ const answer = fromPlan(resource, result, answered);
369
434
  if (answer.step)
370
- planned.push(answer.step);
435
+ add(answer.step, answered.planned);
371
436
  if (answer.problem)
372
437
  problems.push(answer.problem);
373
438
  }
@@ -378,7 +443,7 @@ export function applyWorkflow(types, ops, drivers, steps) {
378
443
  }
379
444
  }
380
445
  if (next.step)
381
- planned.push(next.step);
446
+ add(next.step);
382
447
  if (next.problem)
383
448
  problems.push(next.problem);
384
449
  }
@@ -397,35 +462,60 @@ export function applyWorkflow(types, ops, drivers, steps) {
397
462
  await ctx.approval("Apply resource changes", { approver, covers, details: JSON.stringify(plan, null, 2) });
398
463
  }
399
464
  const bindings = { ...locked };
465
+ // A resource whose step failed or waited has no state known where its change touches it: a
466
+ // `from` reference there fails, never given the state before; elsewhere the state stands.
467
+ const settled = (named, from) => unsettled.has(named) && touches.get(named)?.(from) ? undefined : states.get(named);
468
+ const known = { bindings, hosts, urls, stateOf: settled };
400
469
  const results = [];
401
470
  let written = false;
402
471
  let revision;
403
472
  for (const step of plan.steps) {
404
- const todo = runOf(step);
473
+ const declared = declarations.get(step.id);
474
+ const binding = "binding" in step ? step.binding : "";
475
+ const run = runOf(step);
476
+ const writing = run === "create" || run === "update";
477
+ // For a step that writes them, its fields as the run knows them now, a `from` field from
478
+ // its resource's state as an earlier step left it. One not known yet is left out, so a
479
+ // driver is never given `null`: planned unknown, the step fails, its driver not called;
480
+ // planned left out (its resource had no state there and no change), it runs without.
481
+ const filled = declared && writing ? desiredOf(declared, types, known, "drop") : undefined;
482
+ const unfilled = filled?.unfilled.find((u) => !dropped.get(step.id)?.has(u.at));
483
+ const waiting = unfilled && waitingFor(unfilled);
484
+ // Unsettled until the step leaves its new state (`keep`). Its URL stays: a host answers
485
+ // where it did until a change moves it.
486
+ if (writing)
487
+ unsettled.add(step.id);
488
+ const todo = waiting === undefined ? run : "none";
405
489
  const key = keyOf(step);
406
490
  const type = `${step.vendor}.${step.type}`;
407
491
  const before = bindings[key];
408
- const binding = "binding" in step ? step.binding : "";
409
- const declared = declarations.get(step.id);
410
492
  const spec = types.get(type);
411
493
  let state;
412
494
  try {
413
495
  if (todo === "create") {
414
- const made = await ctx[step.vendor][step.type].create({
415
- desired: desiredOf(declared, types, bindings, hosts, urls),
416
- }, { connection: step.connection });
496
+ const made = await ctx[step.vendor][step.type].create({ desired: filled.desired }, { connection: step.connection });
417
497
  bindings[key] = { type, id: made.id, ...(step.connection && { connection: step.connection }) };
418
498
  state = made.state;
419
499
  }
420
500
  else if (todo === "update" && (step.action === "update" || step.action === "replace")) {
421
- const expect = { action: step.action, replace: step.replace };
422
- const changed = await ctx[step.vendor][step.type].update({
423
- id: binding,
424
- desired: desiredOf(declared, types, bindings, hosts, urls),
425
- expect,
426
- }, { connection: step.connection });
427
- bindings[key] = { type, id: changed.id, ...(step.connection && { connection: step.connection }) };
428
- state = changed.state;
501
+ // A step planned with a `from` field unknown (its resource changed first) is planned again with its fields as now known: the approval covered the change in kind,
502
+ // and the vendor now says which. Nothing to write, it is done; else the update expects
503
+ // the fresh plan, an action other than an update or replace refused as changed.
504
+ let expect = { action: step.action, replace: step.replace };
505
+ let unchanged = false;
506
+ if (awaited.has(step.id)) {
507
+ const fresh = await ctx[step.vendor][step.type].plan({ id: binding, desired: filled.desired }, { connection: step.connection });
508
+ unchanged = fresh.action === "none";
509
+ if (fresh.action === "update" || fresh.action === "replace")
510
+ expect = { action: fresh.action, replace: fresh.replace };
511
+ }
512
+ if (unchanged)
513
+ unsettled.delete(step.id);
514
+ else {
515
+ const changed = await ctx[step.vendor][step.type].update({ id: binding, desired: filled.desired, expect }, { connection: step.connection });
516
+ bindings[key] = { type, id: changed.id, ...(step.connection && { connection: step.connection }) };
517
+ state = changed.state;
518
+ }
429
519
  }
430
520
  else if (todo === "delete") {
431
521
  await ctx[step.vendor][step.type].delete({ id: binding }, { connection: step.connection });
@@ -436,7 +526,7 @@ export function applyWorkflow(types, ops, drivers, steps) {
436
526
  else if (todo === "forget")
437
527
  delete bindings[key];
438
528
  if (state && declared)
439
- answersAt(declared, state);
529
+ keep(declared, state);
440
530
  const after = bindings[key];
441
531
  if (after?.id !== before?.id) {
442
532
  const wrote = await steps.write(key, after, messageOf(step, before, after));
@@ -447,7 +537,8 @@ export function applyWorkflow(types, ops, drivers, steps) {
447
537
  // id, again every few seconds until its output passes `until`; one that never does fails
448
538
  // the step, and the rest are not called. The loop `callAfterCreate` has, inline: a
449
539
  // function defined here may not close over `ctx`.
450
- let stuck;
540
+ // A step waiting on a `from` reference ran as `none`, and fails here with why.
541
+ let stuck = waiting;
451
542
  for (const hook of todo === "create" ? (spec?.afterCreate ?? []) : []) {
452
543
  if (stuck === undefined) {
453
544
  const [vendor = "", group = "", name = ""] = hook.op.split(".");