@pikku/core 0.12.74 → 0.12.77

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +378 -0
  2. package/dist/column-form.d.ts +32 -0
  3. package/dist/column-form.js +42 -0
  4. package/dist/crypto-utils.d.ts +15 -4
  5. package/dist/crypto-utils.js +18 -2
  6. package/dist/data-classification.d.ts +44 -0
  7. package/dist/function/functions.types.d.ts +23 -10
  8. package/dist/function/index.d.ts +1 -1
  9. package/dist/index.d.ts +6 -3
  10. package/dist/index.js +3 -1
  11. package/dist/middleware/auth-bearer.js +2 -1
  12. package/dist/middleware/remote-auth.js +1 -1
  13. package/dist/remote.js +1 -1
  14. package/dist/secret-value.d.ts +56 -0
  15. package/dist/secret-value.js +46 -0
  16. package/dist/services/audit-service.d.ts +74 -4
  17. package/dist/services/audit-service.js +7 -5
  18. package/dist/services/credential-wire-service.d.ts +5 -0
  19. package/dist/services/credential-wire-service.js +9 -1
  20. package/dist/services/email-service.d.ts +2 -1
  21. package/dist/services/index.d.ts +3 -3
  22. package/dist/services/index.js +1 -1
  23. package/dist/services/local-content-request-handler.d.ts +29 -0
  24. package/dist/services/local-content-request-handler.js +176 -0
  25. package/dist/services/local-secrets.d.ts +4 -3
  26. package/dist/services/local-secrets.js +7 -3
  27. package/dist/services/logger.d.ts +22 -5
  28. package/dist/services/queue-webhook-service.js +1 -1
  29. package/dist/services/scoped-secret-service.d.ts +4 -3
  30. package/dist/services/secret-service.d.ts +8 -3
  31. package/dist/services/typed-secret-service.d.ts +5 -4
  32. package/dist/services/webhook-service.d.ts +2 -1
  33. package/dist/testing/service-tests.js +6 -6
  34. package/dist/types/core.types.d.ts +25 -4
  35. package/dist/wirings/ai-agent/ai-agent-agui.js +13 -1
  36. package/dist/wirings/ai-agent/ai-agent-prepare.js +7 -1
  37. package/dist/wirings/ai-agent/ai-agent-runner.js +14 -2
  38. package/dist/wirings/ai-agent/ai-agent-stream.js +27 -1
  39. package/dist/wirings/ai-agent/ai-agent.types.d.ts +40 -0
  40. package/dist/wirings/ai-agent/index.d.ts +1 -1
  41. package/dist/wirings/ai-agent/index.js +1 -1
  42. package/dist/wirings/ai-agent/voice-input.d.ts +20 -0
  43. package/dist/wirings/ai-agent/voice-input.js +44 -9
  44. package/dist/wirings/ai-agent/voice-output.d.ts +15 -0
  45. package/dist/wirings/ai-agent/voice-output.js +10 -1
  46. package/dist/wirings/cli/channel/cli-raw-client-runner.d.ts +21 -3
  47. package/dist/wirings/cli/channel/cli-raw-client-runner.js +13 -5
  48. package/dist/wirings/cli/channel/index.d.ts +1 -0
  49. package/dist/wirings/persona/define-personas.d.ts +4 -0
  50. package/dist/wirings/persona/define-personas.js +4 -0
  51. package/dist/wirings/persona/persona.types.d.ts +11 -0
  52. package/dist/wirings/queue/queue-identity.js +2 -1
  53. package/dist/wirings/queue/queue.types.d.ts +2 -1
  54. package/dist/wirings/queue/signed-queue-service.d.ts +2 -1
  55. package/dist/wirings/rpc/remote-addon-auth.d.ts +2 -1
  56. package/dist/wirings/rpc/remote-addon-auth.js +6 -2
  57. package/dist/wirings/virtual-user/index.d.ts +3 -0
  58. package/dist/wirings/virtual-user/index.js +2 -0
  59. package/dist/wirings/virtual-user/prepare-virtual-user-run.d.ts +54 -0
  60. package/dist/wirings/virtual-user/prepare-virtual-user-run.js +49 -0
  61. package/dist/wirings/virtual-user/virtual-user-run-store.d.ts +90 -0
  62. package/dist/wirings/virtual-user/virtual-user-run-store.js +1 -0
  63. package/dist/wirings/workflow/dsl/workflow-dsl.types.d.ts +14 -10
  64. package/dist/wirings/workflow/pikku-scenario-service.js +1 -2
  65. package/dist/wirings/workflow/scenario-prose.js +1 -1
  66. package/dist/wirings/workflow/scenario-step.types.d.ts +13 -7
  67. package/dist/wirings/workflow/workflow.types.d.ts +7 -0
  68. package/knowledge/decisions/internals/a-virtual-user-run-is-not-a-workflow-and-not-a-queued-job.md +48 -0
  69. package/knowledge/decisions/internals/core-column-form-is-an-axis-of-its-own.md +84 -0
  70. package/knowledge/decisions/internals/core-data-classification-brand-is-an-optional-property.md +9 -2
  71. package/knowledge/decisions/internals/index.md +4 -0
  72. package/knowledge/decisions/internals/one-project-shape-check-two-validators.md +53 -0
  73. package/knowledge/decisions/internals/scenarios-live-in-files-named-for-them.md +48 -0
  74. package/knowledge/decisions/internals/validate-checks-personas-through-a-shared-module.md +43 -0
  75. package/package.json +3 -2
  76. package/src/column-form.test.ts +97 -0
  77. package/src/column-form.ts +58 -0
  78. package/src/crypto-utils.ts +25 -6
  79. package/src/data-classification.ts +44 -0
  80. package/src/function/functions.types.ts +47 -10
  81. package/src/function/index.ts +1 -0
  82. package/src/index.ts +24 -2
  83. package/src/middleware/auth-bearer.test.ts +3 -2
  84. package/src/middleware/auth-bearer.ts +2 -1
  85. package/src/middleware/remote-auth.test.ts +2 -1
  86. package/src/middleware/remote-auth.ts +1 -1
  87. package/src/remote.test.ts +2 -1
  88. package/src/remote.ts +1 -1
  89. package/src/secret-value.test.ts +204 -0
  90. package/src/secret-value.ts +111 -0
  91. package/src/services/audit-service.ts +87 -9
  92. package/src/services/credential-wire-service.ts +9 -1
  93. package/src/services/email-service.ts +3 -1
  94. package/src/services/index.ts +3 -3
  95. package/src/services/local-content-request-handler.test.ts +202 -0
  96. package/src/services/local-content-request-handler.ts +267 -0
  97. package/src/services/local-secrets.test.ts +20 -5
  98. package/src/services/local-secrets.ts +15 -7
  99. package/src/services/logger.ts +27 -7
  100. package/src/services/queue-webhook-service.test.ts +2 -1
  101. package/src/services/queue-webhook-service.ts +1 -1
  102. package/src/services/scoped-secret-service.ts +4 -3
  103. package/src/services/secret-service.ts +8 -3
  104. package/src/services/typed-secret-service.ts +11 -7
  105. package/src/services/webhook-service.ts +4 -1
  106. package/src/testing/service-tests.ts +6 -6
  107. package/src/types/core.types.ts +25 -4
  108. package/src/wirings/ai-agent/ai-agent-agui.test.ts +16 -0
  109. package/src/wirings/ai-agent/ai-agent-agui.ts +14 -1
  110. package/src/wirings/ai-agent/ai-agent-prepare.ts +7 -1
  111. package/src/wirings/ai-agent/ai-agent-runner.ts +18 -2
  112. package/src/wirings/ai-agent/ai-agent-stream.ts +32 -1
  113. package/src/wirings/ai-agent/ai-agent.types.ts +45 -1
  114. package/src/wirings/ai-agent/index.ts +2 -0
  115. package/src/wirings/ai-agent/voice-input.test.ts +65 -0
  116. package/src/wirings/ai-agent/voice-input.ts +48 -9
  117. package/src/wirings/ai-agent/voice-output.test.ts +91 -1
  118. package/src/wirings/ai-agent/voice-output.ts +28 -1
  119. package/src/wirings/cli/channel/cli-raw-client-runner.ts +39 -9
  120. package/src/wirings/cli/channel/index.ts +4 -0
  121. package/src/wirings/persona/define-personas.ts +4 -0
  122. package/src/wirings/persona/persona.types.ts +11 -0
  123. package/src/wirings/queue/queue-identity.test.ts +2 -1
  124. package/src/wirings/queue/queue-identity.ts +4 -1
  125. package/src/wirings/queue/queue.types.ts +6 -1
  126. package/src/wirings/queue/signed-queue-service.ts +2 -1
  127. package/src/wirings/rpc/remote-addon-auth.ts +8 -3
  128. package/src/wirings/rpc/rpc-runner.test.ts +6 -4
  129. package/src/wirings/virtual-user/index.ts +12 -0
  130. package/src/wirings/virtual-user/prepare-virtual-user-run.test.ts +115 -0
  131. package/src/wirings/virtual-user/prepare-virtual-user-run.ts +95 -0
  132. package/src/wirings/virtual-user/virtual-user-run-store.ts +98 -0
  133. package/src/wirings/workflow/dsl/workflow-dsl.types.ts +14 -16
  134. package/src/wirings/workflow/pikku-scenario-service.ts +1 -10
  135. package/src/wirings/workflow/scenario-prose.test.ts +5 -7
  136. package/src/wirings/workflow/scenario-prose.ts +1 -1
  137. package/src/wirings/workflow/scenario-service.test.ts +0 -1
  138. package/src/wirings/workflow/scenario-step.test.ts +4 -5
  139. package/src/wirings/workflow/scenario-step.types.ts +13 -7
  140. package/src/wirings/workflow/scenario-surface.test.ts +6 -5
  141. package/src/wirings/workflow/workflow.types.ts +7 -0
  142. package/tsconfig.tsbuildinfo +1 -1
  143. package/tsconfig.type-tests.json +12 -0
@@ -3,14 +3,20 @@ import type { ScenarioPersona } from '../../services/personas-service.js';
3
3
  * Scenario steps: named, typed units of scenario behaviour.
4
4
  *
5
5
  * A step's body is an ordinary pikku function, so it may drive a browser, call
6
- * an RPC as its actor, or run a workflow. `given`/`when`/`then` are sugar over
7
- * `step` they only change the prose the reporter renders.
6
+ * an RPC as its actor, or run a workflow. `given` and `when` differ only in the
7
+ * prose the reporter renders; `then` also changes what runs, because its
8
+ * bindings are witnesses — every declared surface is observed and the
9
+ * observations must agree. See {@link ScenarioSurfaceResolution}.
8
10
  */
9
11
  /**
10
- * Which Gherkin-style keyword the reporter prefixes this step with. `step`
11
- * renders no prefix at all.
12
+ * Which Gherkin-style keyword the reporter prefixes this step with.
13
+ *
14
+ * Every step takes one. A scenario is read by people deciding whether it
15
+ * describes the behaviour they wanted, and a step that says only what it does
16
+ * without saying whether it is setup, action or claim is the one nobody can
17
+ * check — which is also why PKU680 can tell a scenario that never asserts.
12
18
  */
13
- export type ScenarioStepPhase = 'step' | 'given' | 'when' | 'then';
19
+ export type ScenarioStepPhase = 'given' | 'when' | 'then';
14
20
  /**
15
21
  * Who acts in a step.
16
22
  *
@@ -41,7 +47,7 @@ export declare const SCENARIO_SURFACES: readonly ScenarioSurface[];
41
47
  /**
42
48
  * How a step's declared surfaces resolve for one run.
43
49
  *
44
- * `given`/`when`/`step` bindings are **alternatives** — clicking Buy and calling
50
+ * `given`/`when` bindings are **alternatives** — clicking Buy and calling
45
51
  * `createOrder` are two ways to cause one effect, so exactly one runs.
46
52
  *
47
53
  * `then` bindings are **witnesses** — "the order row says paid" and "the
@@ -73,7 +79,7 @@ export type ScenarioSurfaceResolution = {
73
79
  unwitnessed: boolean;
74
80
  };
75
81
  /**
76
- * Options accepted by `scenario.step/given/when/then`.
82
+ * Options accepted by `scenario.given/when/then`.
77
83
  *
78
84
  * Note the retry default differs from an ordinary workflow step: retrying a
79
85
  * failed assertion is the wrong behaviour for a test primitive, so steps
@@ -188,6 +188,13 @@ export type WorkflowsMeta = Record<string, CommonWireMeta & {
188
188
  dsl?: boolean;
189
189
  expose?: boolean;
190
190
  scenario?: boolean;
191
+ /**
192
+ * The flow asserts through an expectation helper — `expectService`,
193
+ * `expectError`, `expectEventually` — rather than a `then` step. Those are
194
+ * inline steps and carry no phase, so without this PKU680 reads a scenario
195
+ * whose only witness is a recorded service call as asserting nothing.
196
+ */
197
+ asserts?: boolean;
191
198
  skip?: string;
192
199
  actors?: string[];
193
200
  }>;
@@ -0,0 +1,48 @@
1
+ ---
2
+ type: decision
3
+ title: A virtual user run is not a workflow and not a queued job
4
+ description: runVirtualUser writes its record, dispatches the run without awaiting it, and returns the id — because an exploratory run has nothing to replay and the record already carries what a queue would be holding
5
+ tags: virtual-user, storage
6
+ ---
7
+
8
+ # A virtual user run is not a workflow and not a queued job
9
+
10
+ `runVirtualUser` — the RPC `scaffold.virtualUser` generates — does three things
11
+ in order: writes a `VirtualUserRunStore` record, dispatches
12
+ `executeVirtualUserRun` **without awaiting it**, and returns the `runId`. There
13
+ is no workflow, no queue, and no worker.
14
+
15
+ Both of the alternatives are the obvious ones, and both are wrong for this.
16
+
17
+ **A workflow** is a replayable step graph: its value is that a run can be
18
+ resumed at the step it died on, and that the same input reaches the same step.
19
+ A virtual user is the opposite by construction — it is an LLM deciding what to
20
+ try next, so no two attempts take the same steps, and there is no step to resume
21
+ *to*. Recording a run as a workflow puts entries in the workflow store that can
22
+ never be replayed, and gives every operator reading that store a row that lies
23
+ about what it is. The seed makes a run *reproducible* — run it again and it
24
+ explores the same way — which is a different property from resumable, and one
25
+ the record already carries.
26
+
27
+ **A queue** buys durability across a restart and a retry on failure. It costs a
28
+ broker dependency in every application that turns the scaffold on, plus a worker
29
+ whose progress cannot be read anyway: the run's state lives in the store, not in
30
+ the queue entry. The queue would be holding a copy of what the record already
31
+ has, on the way to the same place.
32
+
33
+ So the record is the run's only trace, and that is what `VirtualUserRunStore`
34
+ exists for. It is also why `fail()` is a method rather than an absence: a run
35
+ that crashed and a run that found nothing are different answers, and a record
36
+ left at `running` is neither.
37
+
38
+ The cost is real and is stated on the type: **a restart mid-run strands a record
39
+ at `running` with nothing left to finish it.** A run older than its budget
40
+ window and still `running` is dead, not working — that is a read-side rule, and
41
+ it is cheaper than the two dependencies avoided. Nothing retries; a stranded run
42
+ is started again, with its seed if the caller wants the same exploration.
43
+
44
+ **What this rules out:** dispatching the run through `startWorkflow`; a
45
+ scaffolded queue worker; awaiting the engine inside the request (a run takes
46
+ minutes and survives neither a rollout nor a proxy timeout); and inferring
47
+ `status` from `finishedAt` being unset, which cannot separate a crash from a run
48
+ still going.
@@ -0,0 +1,84 @@
1
+ ---
2
+ type: decision
3
+ title: A column's at-rest form is an axis of its own
4
+ description: How a value is stored is independent of how sensitive it is, so form carries a required nominal brand on writes while classification stays optional on reads
5
+ tags: core
6
+ ---
7
+
8
+ # A column's at-rest form is an axis of its own
9
+
10
+ `ColumnForm` in `packages/core/src/data-classification.ts` is a second,
11
+ independent annotation on a column: `plain | hashed | wrapped | sealed`. It
12
+ answers "how are these bytes held?", where `Classification` answers "may this
13
+ value leave the process?".
14
+
15
+ The two axes were one field before this, with `security: 'encrypted'` sitting
16
+ alongside `secret` as though they were alternatives. They are not, and the
17
+ conflation made the field unanswerable: a token hash and a live bearer token are
18
+ both `secret`, one must never be encrypted — the digest *is* the lookup key —
19
+ and the other must always be. Nothing in a single enum could tell them apart, so
20
+ nothing could check either.
21
+
22
+ ## Why `wrapped` and `sealed` rather than `encrypted`
23
+
24
+ Sealed values *are* encrypted, so an `encrypted` member sitting beside `sealed`
25
+ would be a supertype posing as a sibling, and every new column would be an
26
+ even-odds guess. What actually separates them is who can read the value back:
27
+ `wrapped` is symmetric and the application holds the key; `sealed` is asymmetric
28
+ and the application holds only the public half. Writing one where the other
29
+ belongs produces a row nobody can ever open, which is why the type system is
30
+ made to know the difference.
31
+
32
+ ## Why these brands are required when `Secret<T>` is optional
33
+
34
+ `WrappedValue`, `SealedValue` and `HashedValue` are `string & { readonly [sym]:
35
+ true }` with a `unique symbol` — nominal, and **required**, which
36
+ [the classification-brand decision](core-data-classification-brand-is-an-optional-property.md)
37
+ explicitly rules out for `Private`/`Pii`/`Secret`. That decision still stands and
38
+ this does not weaken it. It applies to a different side of a different set of
39
+ columns:
40
+
41
+ - `Secret<T>` brands **every** classified column's SELECT type. A required brand
42
+ there would break `where('email', '=', someString)` in every downstream
43
+ project.
44
+ - A form brands **only** the INSERT/UPDATE type, and only on the columns that
45
+ opt in by declaring a form. There is nothing to break, because a column
46
+ without a form generates exactly what it generated before.
47
+
48
+ The brands compose rather than compete: a wrapped secret column selects as
49
+ `Secret<WrappedValue>`, so the inspector's PKU910 check still finds
50
+ `__classification__`, while a row read back is already a `WrappedValue` and
51
+ flows into a rewrap or re-seal without a cast.
52
+
53
+ Each brand widens to `string`, so query operands, serialization and template
54
+ literals are unaffected. The constraint is on **construction**: the only way to
55
+ produce one is `envelopeEncrypt`/`envelopeRewrap`/`wrapDEK` (wrapped) or
56
+ `hashToken` (hashed), or the deliberately-named `unsafeAs*` assertions in
57
+ `column-form.ts` for the three cases a bare string legitimately arrives —
58
+ backfill migrations, test fixtures, and values sealed by another service.
59
+
60
+ ## What is deliberately NOT enforced
61
+
62
+ `envelopeDecrypt` and `unwrapDEK` take plain `string`, not the brand. Requiring
63
+ it would buy nothing — feeding in the wrong string already fails at the AEAD tag
64
+ — while forcing a cast into every path that reads ciphertext out of a row or off
65
+ the wire, which is where casts are least reviewable.
66
+
67
+ The brand proves *provenance*, not correctness. It cannot know a value was
68
+ wrapped under the right key, and making it know would mean phantom-typing key
69
+ ids per scope, which the multi-recipient path would fight constantly.
70
+
71
+ ## The plain-secret diagnostic
72
+
73
+ A `secret` column with no declared form raises **PKU483** as a warning, not an
74
+ error. Every project predating the axis has such columns, and failing their next
75
+ `db migrate` would be a breaking change for a diagnosis they have not had a
76
+ chance to act on. `pikku db --fail-on-warn` is how a project opts into the
77
+ ratchet. An explicit `form: 'plain'` silences it — that is the acknowledgement
78
+ that reading the row is *meant* to yield a usable credential.
79
+
80
+ **What this rules out:** collapsing `wrapped` and `sealed` back into one
81
+ `encrypted`; making the form brands optional (they would enforce nothing);
82
+ re-declaring them in generated schema files, since a local `unique symbol` is a
83
+ distinct nominal type and core's own ciphertext would not be assignable to the
84
+ column it belongs in — `db-codegen` imports them from `@pikku/core` instead.
@@ -22,10 +22,17 @@ goes into one.
22
22
 
23
23
  The consumer is `@pikku/inspector`, whose `findPiiPaths` reads the level union
24
24
  directly and whose PKU910 output check detects the brand on a function's return
25
- type. The brands are populated from `-- @private` / `-- @secret` / `-- @public`
26
- SQL comment annotations via `pikku db migrate`, which regenerates
25
+ type. The brands are populated from the hand-authored `db/annotations.ts`
26
+ (`DbClassificationMap`) via `pikku db migrate`, which regenerates
27
27
  `outDir/db/schema.d.ts` and `outDir/db/classification.gen.ts`.
28
28
 
29
+ A column's **at-rest form** is a separate axis making the opposite trade — see
30
+ [form is an axis of its own](core-column-form-is-an-axis-of-its-own.md). Its
31
+ brands are nominal and required, which is safe precisely because they land on
32
+ the INSERT/UPDATE side of the columns that opt in, rather than on the SELECT
33
+ side of every classified column. That is not an exception to the rule below; it
34
+ is a different rule about a different side.
35
+
29
36
  **What this rules out:** making `__classification__` required to get stronger
30
37
  guarantees, or replacing the optional property with a unique symbol / nominal
31
38
  brand that behaves like a required one. Either change compiles here and then
@@ -38,6 +38,7 @@ caller is entitled to assume.
38
38
  - [CLI parse errors are routed by message prefix](cli-parse-errors-are-routed-by-message-prefix.md) — The CLI runners decide between printing help and printing errors by string-matching the prefixes the parser writes, so those message strings are an interface
39
39
  - [CLI stdout is reserved for machine-readable output](cli-stdout-is-reserved-for-machine-readable-output.md) — The default renderer emits single-line NDJSON, diagnostics go to stderr, and --json only hijacks rendering for commands that declared a renderer
40
40
  - [CLI unknown long options warn instead of failing](cli-unknown-long-options-warn-instead-of-failing.md) — Unrecognised --long options are accepted, warned about and dropped so older binaries tolerate newer invocations, while unknown short flags stay hard errors
41
+ - [A column's at-rest form is an axis of its own](core-column-form-is-an-axis-of-its-own.md) — How a value is stored is independent of how sensitive it is, so form carries a required nominal brand on writes while classification stays optional on reads
41
42
  - [The data-classification brand is an optional property](core-data-classification-brand-is-an-optional-property.md) — Making __classification__ required would break ordinary Kysely operands, so the brand only constrains values flowing out
42
43
  - [The function runner restores the wire fields it overwrites](core-function-runner-restores-the-wire-fields-it-overwrites.md) — One wire object is reused across nested calls, so functionId, audit, addonNamespace and rpc are saved and put back in a finally
43
44
  - [Hot reload merges generated meta and never replaces it](core-hot-reload-merges-generated-meta-never-replaces-it.md) — Reloading codegen output must preserve runtime-registered meta, which no generated JSON contains
@@ -77,8 +78,11 @@ caller is entitled to assume.
77
78
  - [The in-memory workflow service is inline-only and single-process](the-in-memory-workflow-service-is-inline-only-and-single-process.md) — InMemoryWorkflowService wires no queues and implements withRunLock/withStepLock as pass-throughs, because inline execution has no second holder to exclude
78
79
  - [The KEK salt is scoped to the key version, not the secret](the-kek-salt-is-scoped-to-the-key-version.md) — One stored salt per key version means N secrets cost one derivation, which is the point of envelope encryption
79
80
  - [The schema service is never stubbed, or tests validate nothing](the-schema-service-is-never-stubbed.md) — createStubProxy returns undefined for the schema property so the real schema service is built — a stubbed one turns validation into a silent no-op
81
+ - [One project-shape check, called by both validators](one-project-shape-check-two-validators.md) — workspace validate and fabric validate were separate walks over the same project that duplicated sixteen findings verbatim; the shared half now lives in shared-checks.ts
82
+ - [Scenarios, steps and personas live in files named for them](scenarios-live-in-files-named-for-them.md) — pikkuScenario/pikkuScenarioStep must be in *.scenario.ts/*.steps.ts and definePersonas in *.virtual-user.ts, an error because the mixing is only cheap to undo while it is one file
80
83
  - [Trigger declaration is split from trigger source](trigger-declaration-is-split-from-trigger-source.md) — Triggers are declared everywhere but subscribed only in the trigger worker, so app processes never open the underlying subscription
81
84
  - [TypedSecretService caches for the process lifetime](typed-secret-service-caches-for-the-process-lifetime.md) — Resolved secrets are cached with no TTL, so a secret rotated out of band is not picked up until restart — tracked as pikkujs/pikku#964
85
+ - [Both validators check personas through one shared module, and a persona counts from either the meta or the source](validate-checks-personas-through-a-shared-module.md) — workspace validate and fabric validate are separate implementations, so the persona/actor/knowledge checks live in persona-checks.ts and are called by both
82
86
  - [Webhook delivery history records every attempt, best effort](webhook-delivery-history-records-every-attempt-best-effort.md) — The webhook worker persists each attempt before it throws, and a failure to persist is logged rather than allowed to mask the delivery result
83
87
  - [Webhook service collaborators are constructor args, not locator lookups](webhook-service-collaborators-are-constructor-args-not-locator-lookups.md) — QueueWebhookService takes its queue as a constructor parameter so a project wiring webhooks without a queue fails to compile instead of at first send
84
88
  - [Whether a run is inline is read from the run record](whether-a-run-is-inline-is-read-from-the-run-record.md) — The runContexts map is a read-through cache over WorkflowRun.inline and a lifetime for replay ordinals, never the answer to what a run is
@@ -0,0 +1,53 @@
1
+ ---
2
+ type: decision
3
+ title: One project-shape check, called by both validators
4
+ description: workspace validate and fabric validate were separate walks over the same project that duplicated sixteen findings verbatim; the shared half now lives in shared-checks.ts and fabric validate is that plus the deploy-shaped checks
5
+ tags: cli, validate, fabric
6
+ ---
7
+
8
+ # One project-shape check, called by both validators
9
+
10
+ `pikku workspace validate` and `pikku fabric validate` read like one command
11
+ with a flag. They were two implementations — 495 lines and 2026 — that walked
12
+ the same project and emitted sixteen identical findings. `functions-dir-missing`
13
+ existed character for character in both files. All sixteen already agreed on
14
+ severity, which is the only reason the duplication was invisible.
15
+
16
+ What the duplication cost was not the lines. It was that each validator carried
17
+ checks the other lacked for no reason anyone could name, and nobody noticed
18
+ because nobody read them side by side:
19
+
20
+ - fabric validate never checked that `packages/functions` declares zod v4, or
21
+ that it has a `package.json` at all — while it did check that six other
22
+ dependencies are present.
23
+ - workspace validate never checked the four scaffold flags beyond `console`,
24
+ though the console it requires is useless without `rpc`.
25
+ - fabric validate swallowed a JSON parse error and reported
26
+ `pikku-config-missing` for a file sitting right there.
27
+
28
+ Two of workspace validate's own checks were not merely absent from fabric — they
29
+ had never run anywhere:
30
+
31
+ - the auth checks were gated on a middleware instance with `definitionId ===
32
+ 'betterAuthSession'`, but the CLI wires `betterAuthStatelessSession` whenever
33
+ `session.cookieCache` is on, which is the configuration Fabric asks for. So
34
+ they skipped precisely the apps most likely to have auth.
35
+ - they looked for migrations in `packages/functions/db/`, and for tables named
36
+ `app_user` and `auth_verification_token`. `pikku db migrate` reads migrations
37
+ from `<root>/db/<engine>/`, and those two table names appear nowhere else in
38
+ pikku or in any template. Had the gate ever opened, the check would have been
39
+ wrong for every project it fired on.
40
+
41
+ `shared-checks.ts` is now the half that is true of any pikku project regardless
42
+ of where it deploys. Workspace validate is exactly that module; fabric validate
43
+ is that module plus the deploy-shaped checks — themes, frontends, the
44
+ Cloudflare adapter, the `.gitignore` contract, the frontend type-check.
45
+
46
+ Fabric's fix hints won where the two differed: they are multi-line and show the
47
+ config block to paste, and there was no reason workspace users were getting the
48
+ one-line version.
49
+
50
+ **What this rules out:** adding a check to one validator and not the other;
51
+ `pikku fabric validate` reporting a corrupt config as a missing one; asserting
52
+ auth table names no scaffold generates; and reading migrations from anywhere
53
+ other than where `pikku db migrate` reads them.
@@ -0,0 +1,48 @@
1
+ ---
2
+ type: decision
3
+ title: Scenarios, steps and personas live in files named for them, and validate errors when they do not
4
+ description: pikkuScenario/pikkuFeature/pikkuScenarioStep must be in *.scenario.ts, *.scenarios.ts or *.steps.ts, and definePersonas/runVirtualUser in *.virtual-user.ts or *.vu.ts — an error, because the mixing is only cheap to undo while it is one file
5
+ tags: cli, validate, scenarios, personas
6
+ ---
7
+
8
+ # Scenarios, steps and personas live in files named for them, and validate errors when they do not
9
+
10
+ A `pikkuScenario` declared beside the functions it exercises reads as more of
11
+ the same file. The wiring, the function, and the test of the function arrive in
12
+ one scroll, and the scenario is the part that gets lost — you cannot tell from
13
+ a directory listing which files ship and which files test.
14
+
15
+ So `validate` requires the declaration to be in a file named for what it is:
16
+
17
+ | declaration | file |
18
+ | --- | --- |
19
+ | `pikkuScenario`, `pikkuFeature` | `*.scenario.ts`, `*.scenarios.ts` |
20
+ | `pikkuScenarioStep` and its platform/addon variants | `*.steps.ts` (or the above) |
21
+ | `definePersonas`, `runVirtualUser` | `*.virtual-user.ts`, `*.vu.ts` |
22
+
23
+ Three suffixes for scenarios rather than one, because the split that matters is
24
+ scenarios apart from application code, not a particular spelling — `.steps.ts`
25
+ holds the steps, `.scenarios.ts` holds the scenarios that call them, and a
26
+ project that already made that split does not have to rename anything.
27
+
28
+ **Error, not warning.** The mixing is cheap to undo while it is one file and
29
+ expensive once it is thirty, and a warning is exactly the signal a project
30
+ learns to scroll past. This is the one place where the scenario checks are
31
+ strict: everything else `validate` says about scenarios (no personas declared,
32
+ no actor sign-in, no environments) is a warning, because those describe an
33
+ under-tested project rather than an unreadable one.
34
+
35
+ **Personas are the virtual-user file** because there is no `defineVirtualUsers`
36
+ to look for. That name was retired in favour of `definePersonas`: a virtual user
37
+ is derived rather than authored — the function meta becomes its catalogue, the
38
+ scenario meta becomes its intents, the declared personas become its identities.
39
+ The persona list is the only part a project writes, so it is the part that has
40
+ to be findable by filename.
41
+
42
+ Matching is on the call (`\bpikkuScenario\s*\(`), not the import, so an aliased
43
+ import cannot slip past and a mention in a comment or a string does not trip it.
44
+ Generated files are exempt — codegen puts things where it likes.
45
+
46
+ **What this rules out:** a scenario in a function file; a step in a wiring file;
47
+ personas in a general-purpose `personas.ts`; and reporting any of these as a
48
+ warning that a project can carry indefinitely.
@@ -0,0 +1,43 @@
1
+ ---
2
+ type: decision
3
+ title: Both validators check personas through one shared module, and a persona counts from either the meta or the source
4
+ description: workspace validate and fabric validate are separate implementations, so the persona/actor/knowledge checks live in persona-checks.ts and are called by both — and a persona is counted from the generated meta or a definePersonas() source scan, whichever answers first
5
+ tags: cli, validate, scenarios, personas
6
+ ---
7
+
8
+ # Both validators check personas through one shared module, and a persona counts from either the meta or the source
9
+
10
+ `pikku workspace validate` and `pikku fabric validate` read like one command
11
+ with a flag. They are not. They are two implementations that walk the same
12
+ project and duplicate about eighteen findings between them verbatim —
13
+ `functions-dir-missing` exists twice, character for character, in two files.
14
+ Fabric adds the deploy-shaped checks (themes, frontends, the Cloudflare
15
+ adapter, the `.gitignore` contract); workspace adds the local-development ones
16
+ (`dev.db`, the auth migrations).
17
+
18
+ So a check written into either one is a check half the projects never run. An
19
+ app that never deploys through fabric only ever sees workspace validate; an app
20
+ that does sees fabric validate in CI. The persona checks are the kind that only
21
+ pay off when nobody remembers to ask for them, which means they had to be in
22
+ both. Rather than duplicate them a nineteenth time, they live in
23
+ `persona-checks.ts` and both validators call `runPersonaChecks`. Merging the
24
+ two validators outright is the better fix and a much larger one; this does not
25
+ block it.
26
+
27
+ **A persona counts from either source, and either alone is wrong at a
28
+ predictable moment.** The generated `pikku-personas-meta.gen.json` is empty on
29
+ a fresh clone where codegen has not run, so trusting only it means a warning
30
+ that fires on every new checkout — and a validator that cries wolf gets muted,
31
+ after which it never says anything worth hearing. A source scan for
32
+ `definePersonas(` cannot see personas an addon contributed. Declared by either
33
+ counts as declared.
34
+
35
+ **Everything here is `warn`, never `error`.** A project with no personas is
36
+ under-tested, not broken, and `validate` gating a deploy on it would make the
37
+ first thing anyone does with the check be to remove it. `knowledge-empty` stays
38
+ at the `info` the shared `@pikku/knowledge` package already assigned it rather
39
+ than being raised to match — the severity belongs to that package.
40
+
41
+ **What this rules out:** duplicating the checks into both validators;
42
+ adding them to only one; deriving the persona count from the generated meta
43
+ alone; failing a build over a missing persona.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@pikku/core",
3
- "version": "0.12.74",
3
+ "version": "0.12.77",
4
4
  "description": "The Pikku runtime — functions, wirings, services, middleware and types",
5
5
  "author": "yasser.fadl@gmail.com",
6
6
  "license": "MIT",
@@ -8,7 +8,7 @@
8
8
  "main": "dist/index.js",
9
9
  "type": "module",
10
10
  "scripts": {
11
- "tsc": "tsc",
11
+ "tsc": "tsc && tsc -p tsconfig.type-tests.json",
12
12
  "build": "tsc -b",
13
13
  "ncu": "npx npm-check-updates",
14
14
  "release": "npm run build && npm test",
@@ -57,6 +57,7 @@
57
57
  "./services/v8-coverage": "./dist/services/v8-coverage-service.js",
58
58
  "./services/istanbul-coverage": "./dist/services/istanbul-coverage-service.js",
59
59
  "./services/local-content": "./dist/services/local-content.js",
60
+ "./services/local-content-request-handler": "./dist/services/local-content-request-handler.js",
60
61
  "./services/temporary-file-service": "./dist/services/temporary-file-service.js",
61
62
  "./crypto-utils": "./dist/crypto-utils.js",
62
63
  "./hmac": "./dist/utils/hmac.js",
@@ -0,0 +1,97 @@
1
+ import { describe, test } from 'node:test'
2
+ import assert from 'node:assert/strict'
3
+ import {
4
+ hashToken,
5
+ unsafeAsHashed,
6
+ unsafeAsSealed,
7
+ unsafeAsWrapped,
8
+ } from './column-form.js'
9
+ import type {
10
+ HashedValue,
11
+ SealedValue,
12
+ WrappedValue,
13
+ } from './data-classification.js'
14
+ import { deriveKEK, envelopeEncrypt, envelopeRewrap } from './crypto-utils.js'
15
+
16
+ const KEY_MATERIAL = 'x'.repeat(32)
17
+
18
+ describe('hashToken', () => {
19
+ test('is the sha256 of the input, lowercase hex', async () => {
20
+ // Known vector, so a refactor of the hex encoding cannot quietly change
21
+ // what every stored hash in every downstream database means.
22
+ assert.equal(
23
+ await hashToken('abc'),
24
+ 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad'
25
+ )
26
+ })
27
+
28
+ test('is stable across calls', async () => {
29
+ assert.equal(await hashToken('token'), await hashToken('token'))
30
+ })
31
+
32
+ test('separates different inputs', async () => {
33
+ assert.notEqual(await hashToken('a'), await hashToken('b'))
34
+ })
35
+ })
36
+
37
+ describe('the form brands', () => {
38
+ test('a plain string is not assignable to any of them', () => {
39
+ // @ts-expect-error plaintext cannot be written to a wrapped column
40
+ const wrapped: WrappedValue = 'not-ciphertext'
41
+ // @ts-expect-error plaintext cannot be written to a sealed column
42
+ const sealed: SealedValue = 'not-ciphertext'
43
+ // @ts-expect-error a raw token cannot be written where its hash belongs
44
+ const hashed: HashedValue = 'raw-token'
45
+ assert.ok(wrapped && sealed && hashed)
46
+ })
47
+
48
+ test('the three do not substitute for one another', () => {
49
+ const wrapped = unsafeAsWrapped('ct')
50
+ // @ts-expect-error a value fabric cannot read is not one it can
51
+ const asWrapped: WrappedValue = unsafeAsSealed('ct')
52
+ // @ts-expect-error a digest is not ciphertext
53
+ const asHashed: HashedValue = wrapped
54
+ assert.ok(asWrapped && asHashed)
55
+ })
56
+
57
+ test('but each still widens to string, so reads and query operands are unaffected', () => {
58
+ const asString: string = unsafeAsWrapped('ct')
59
+ assert.equal(asString, 'ct')
60
+ assert.equal(`${unsafeAsHashed('abc')}`, 'abc')
61
+ })
62
+ })
63
+
64
+ describe('the envelope primitives produce the brand', () => {
65
+ test('envelopeEncrypt returns ciphertext that is writable to a wrapped column', async () => {
66
+ const kek = await deriveKEK(KEY_MATERIAL, 'c2FsdA')
67
+ const { ciphertext, wrappedDEK } = await envelopeEncrypt(kek, 'value')
68
+ // No cast: this is the whole mechanism — the only way to satisfy the
69
+ // column type is to have actually encrypted something.
70
+ const forColumn: WrappedValue = ciphertext
71
+ const dekForColumn: WrappedValue = wrappedDEK
72
+ assert.ok(forColumn.length > 0 && dekForColumn.length > 0)
73
+ })
74
+
75
+ test('a rewrapped DEK is still writable, so rotation needs no escape hatch', async () => {
76
+ const oldKek = await deriveKEK(KEY_MATERIAL, 'c2FsdA')
77
+ const newKek = await deriveKEK('y'.repeat(32), 'c2FsdA')
78
+ const { wrappedDEK } = await envelopeEncrypt(oldKek, 'value')
79
+ const rewrapped: WrappedValue = await envelopeRewrap(
80
+ oldKek,
81
+ newKek,
82
+ wrappedDEK
83
+ )
84
+ assert.notEqual(rewrapped, wrappedDEK)
85
+ })
86
+
87
+ test('decrypting accepts a bare string, so reading a row back needs no cast', async () => {
88
+ const kek = await deriveKEK(KEY_MATERIAL, 'c2FsdA')
89
+ const { ciphertext, wrappedDEK } = await envelopeEncrypt(kek, 'value')
90
+ const { envelopeDecrypt } = await import('./crypto-utils.js')
91
+ const fromRow: string = ciphertext
92
+ assert.equal(
93
+ await envelopeDecrypt<string>(kek, fromRow, wrappedDEK),
94
+ 'value'
95
+ )
96
+ })
97
+ })
@@ -0,0 +1,58 @@
1
+ // knowledge: decisions/internals/core-column-form-is-an-axis-of-its-own.md
2
+ import type {
3
+ HashedValue,
4
+ SealedValue,
5
+ WrappedValue,
6
+ } from './data-classification.js'
7
+
8
+ /**
9
+ * SHA-256 of a credential, lowercase hex — the sole producer of `HashedValue`.
10
+ *
11
+ * Deliberately unpeppered. These are high-entropy random bearer tokens, not
12
+ * passwords: there is no candidate set to search, so a pepper would add a key
13
+ * to manage and a rotation story to own while buying nothing. A column holding
14
+ * a *low*-entropy secret is not a `hashed` column — it wants a password KDF and
15
+ * a different type entirely.
16
+ *
17
+ * Async because it runs on WebCrypto rather than `node:crypto`, so the same
18
+ * code path works in a Worker. Callers that hash a bearer token on every
19
+ * request are already async.
20
+ */
21
+ export const hashToken = async (raw: string): Promise<HashedValue> => {
22
+ const subtle = globalThis.crypto?.subtle
23
+ if (!subtle) {
24
+ throw new Error('WebCrypto not available')
25
+ }
26
+ const digest = await subtle.digest(
27
+ 'SHA-256',
28
+ new TextEncoder().encode(raw)
29
+ )
30
+ let hex = ''
31
+ for (const byte of new Uint8Array(digest)) {
32
+ hex += byte.toString(16).padStart(2, '0')
33
+ }
34
+ return hex as HashedValue
35
+ }
36
+
37
+ /**
38
+ * Assert that a string already in storage is ciphertext of the given form.
39
+ *
40
+ * The brands exist so that new writes must come from a real encrypt/hash call,
41
+ * but three paths legitimately hold such a value as a bare `string`: a
42
+ * migration backfilling rows written before the column declared its form, a
43
+ * test fixture, and a value arriving over the wire from a service that sealed
44
+ * it elsewhere. Each of those is a promise the caller is making, not something
45
+ * the type system can check — which is why these are named to be greppable and
46
+ * why there is no non-`unsafe` spelling. If one of these appears in ordinary
47
+ * request-handling code, that is the bug.
48
+ */
49
+ export const unsafeAsWrapped = (stored: string): WrappedValue =>
50
+ stored as WrappedValue
51
+
52
+ /** See `unsafeAsWrapped`. */
53
+ export const unsafeAsSealed = (stored: string): SealedValue =>
54
+ stored as SealedValue
55
+
56
+ /** See `unsafeAsWrapped`. */
57
+ export const unsafeAsHashed = (stored: string): HashedValue =>
58
+ stored as HashedValue
@@ -1,3 +1,4 @@
1
+ import type { WrappedValue } from './data-classification.js'
1
2
  import { WeakKeyMaterialError } from './errors/errors.js'
2
3
 
3
4
  const encoder = new TextEncoder()
@@ -373,11 +374,19 @@ export const deriveKEK = async (
373
374
  return deriveKey(passphrase, fromBase64Url(salt))
374
375
  }
375
376
 
377
+ /**
378
+ * The one place a `WrappedValue` is minted. Every branded return below routes
379
+ * through here, so the assertion "these bytes really are ciphertext" is made
380
+ * once and audited once instead of at each call site.
381
+ */
382
+ const asWrapped = (ciphertext: string): WrappedValue =>
383
+ ciphertext as WrappedValue
384
+
376
385
  export const wrapDEK = async (
377
386
  kek: CryptoKey,
378
387
  plaintextDEK: string
379
- ): Promise<string> => {
380
- return encryptWithCryptoKey(kek, plaintextDEK)
388
+ ): Promise<WrappedValue> => {
389
+ return asWrapped(await encryptWithCryptoKey(kek, plaintextDEK))
381
390
  }
382
391
 
383
392
  export const unwrapDEK = async (
@@ -408,8 +417,8 @@ const decryptWithDEK = async <T>(
408
417
  }
409
418
 
410
419
  export interface EnvelopeEncryptResult {
411
- ciphertext: string
412
- wrappedDEK: string
420
+ ciphertext: WrappedValue
421
+ wrappedDEK: WrappedValue
413
422
  }
414
423
 
415
424
  export const envelopeEncrypt = async (
@@ -417,11 +426,21 @@ export const envelopeEncrypt = async (
417
426
  value: unknown
418
427
  ): Promise<EnvelopeEncryptResult> => {
419
428
  const dek = await generateDEK()
420
- const ciphertext = await encryptWithDEK(dek, value)
429
+ const ciphertext = asWrapped(await encryptWithDEK(dek, value))
421
430
  const wrappedDEK = await wrapDEK(kek, dek)
422
431
  return { ciphertext, wrappedDEK }
423
432
  }
424
433
 
434
+ /**
435
+ * Note the inputs are plain `string`, not `WrappedValue`.
436
+ *
437
+ * The brand exists to stop plaintext being *written* to a wrapped column, and
438
+ * a `WrappedValue` is assignable to `string`, so a branded caller still passes
439
+ * without a cast. Demanding the brand here would buy nothing — feeding the
440
+ * wrong string in already fails at the AEAD tag — while forcing a cast into
441
+ * every path that reads ciphertext back out of a row, a parsed envelope, or the
442
+ * wire, which is exactly where casts are least reviewable.
443
+ */
425
444
  export const envelopeDecrypt = async <T>(
426
445
  kek: CryptoKey,
427
446
  ciphertext: string,
@@ -435,7 +454,7 @@ export const envelopeRewrap = async (
435
454
  oldKEK: CryptoKey,
436
455
  newKEK: CryptoKey,
437
456
  wrappedDEK: string
438
- ): Promise<string> => {
457
+ ): Promise<WrappedValue> => {
439
458
  const dek = await unwrapDEK(oldKEK, wrappedDEK)
440
459
  return wrapDEK(newKEK, dek)
441
460
  }