@pikku/core 0.12.74 → 0.12.78

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/CHANGELOG.md +397 -0
  2. package/dist/column-form.d.ts +32 -0
  3. package/dist/column-form.js +42 -0
  4. package/dist/crypto-utils.d.ts +15 -4
  5. package/dist/crypto-utils.js +18 -2
  6. package/dist/data-classification.d.ts +44 -0
  7. package/dist/function/functions.types.d.ts +23 -10
  8. package/dist/function/index.d.ts +1 -1
  9. package/dist/index.d.ts +6 -3
  10. package/dist/index.js +3 -1
  11. package/dist/middleware/auth-bearer.js +2 -1
  12. package/dist/middleware/remote-auth.js +1 -1
  13. package/dist/remote.js +1 -1
  14. package/dist/secret-value.d.ts +56 -0
  15. package/dist/secret-value.js +46 -0
  16. package/dist/services/audit-service.d.ts +74 -4
  17. package/dist/services/audit-service.js +7 -5
  18. package/dist/services/credential-wire-service.d.ts +5 -0
  19. package/dist/services/credential-wire-service.js +9 -1
  20. package/dist/services/email-service.d.ts +2 -1
  21. package/dist/services/in-memory-workflow-service.d.ts +1 -0
  22. package/dist/services/in-memory-workflow-service.js +20 -0
  23. package/dist/services/index.d.ts +3 -3
  24. package/dist/services/index.js +1 -1
  25. package/dist/services/local-content-request-handler.d.ts +29 -0
  26. package/dist/services/local-content-request-handler.js +176 -0
  27. package/dist/services/local-secrets.d.ts +4 -3
  28. package/dist/services/local-secrets.js +7 -3
  29. package/dist/services/logger.d.ts +22 -5
  30. package/dist/services/queue-webhook-service.js +1 -1
  31. package/dist/services/scoped-secret-service.d.ts +4 -3
  32. package/dist/services/secret-service.d.ts +8 -3
  33. package/dist/services/typed-secret-service.d.ts +5 -4
  34. package/dist/services/webhook-service.d.ts +2 -1
  35. package/dist/testing/service-tests.js +6 -6
  36. package/dist/types/core.types.d.ts +25 -4
  37. package/dist/wirings/ai-agent/ai-agent-agui.js +13 -1
  38. package/dist/wirings/ai-agent/ai-agent-prepare.js +7 -1
  39. package/dist/wirings/ai-agent/ai-agent-runner.js +14 -2
  40. package/dist/wirings/ai-agent/ai-agent-stream.js +27 -1
  41. package/dist/wirings/ai-agent/ai-agent.types.d.ts +40 -0
  42. package/dist/wirings/ai-agent/index.d.ts +1 -1
  43. package/dist/wirings/ai-agent/index.js +1 -1
  44. package/dist/wirings/ai-agent/voice-input.d.ts +20 -0
  45. package/dist/wirings/ai-agent/voice-input.js +44 -9
  46. package/dist/wirings/ai-agent/voice-output.d.ts +15 -0
  47. package/dist/wirings/ai-agent/voice-output.js +10 -1
  48. package/dist/wirings/cli/channel/cli-raw-client-runner.d.ts +21 -3
  49. package/dist/wirings/cli/channel/cli-raw-client-runner.js +13 -5
  50. package/dist/wirings/cli/channel/index.d.ts +1 -0
  51. package/dist/wirings/persona/define-personas.d.ts +4 -0
  52. package/dist/wirings/persona/define-personas.js +4 -0
  53. package/dist/wirings/persona/persona.types.d.ts +11 -0
  54. package/dist/wirings/queue/queue-identity.js +2 -1
  55. package/dist/wirings/queue/queue.types.d.ts +2 -1
  56. package/dist/wirings/queue/signed-queue-service.d.ts +2 -1
  57. package/dist/wirings/rpc/remote-addon-auth.d.ts +2 -1
  58. package/dist/wirings/rpc/remote-addon-auth.js +6 -2
  59. package/dist/wirings/virtual-user/index.d.ts +3 -0
  60. package/dist/wirings/virtual-user/index.js +2 -0
  61. package/dist/wirings/virtual-user/prepare-virtual-user-run.d.ts +54 -0
  62. package/dist/wirings/virtual-user/prepare-virtual-user-run.js +49 -0
  63. package/dist/wirings/virtual-user/virtual-user-run-store.d.ts +90 -0
  64. package/dist/wirings/virtual-user/virtual-user-run-store.js +1 -0
  65. package/dist/wirings/workflow/dsl/workflow-dsl.types.d.ts +14 -10
  66. package/dist/wirings/workflow/pikku-scenario-service.js +1 -2
  67. package/dist/wirings/workflow/pikku-workflow-service.d.ts +35 -0
  68. package/dist/wirings/workflow/pikku-workflow-service.js +51 -0
  69. package/dist/wirings/workflow/scenario-prose.js +1 -1
  70. package/dist/wirings/workflow/scenario-step.types.d.ts +13 -7
  71. package/dist/wirings/workflow/workflow.types.d.ts +7 -0
  72. package/knowledge/decisions/internals/a-virtual-user-run-is-not-a-workflow-and-not-a-queued-job.md +48 -0
  73. package/knowledge/decisions/internals/core-column-form-is-an-axis-of-its-own.md +84 -0
  74. package/knowledge/decisions/internals/core-data-classification-brand-is-an-optional-property.md +9 -2
  75. package/knowledge/decisions/internals/index.md +4 -0
  76. package/knowledge/decisions/internals/one-project-shape-check-two-validators.md +53 -0
  77. package/knowledge/decisions/internals/scenarios-live-in-files-named-for-them.md +48 -0
  78. package/knowledge/decisions/internals/validate-checks-personas-through-a-shared-module.md +43 -0
  79. package/package.json +3 -2
  80. package/src/column-form.test.ts +97 -0
  81. package/src/column-form.ts +58 -0
  82. package/src/crypto-utils.ts +25 -6
  83. package/src/data-classification.ts +44 -0
  84. package/src/function/functions.types.ts +47 -10
  85. package/src/function/index.ts +1 -0
  86. package/src/index.ts +24 -2
  87. package/src/middleware/auth-bearer.test.ts +3 -2
  88. package/src/middleware/auth-bearer.ts +2 -1
  89. package/src/middleware/remote-auth.test.ts +2 -1
  90. package/src/middleware/remote-auth.ts +1 -1
  91. package/src/remote.test.ts +2 -1
  92. package/src/remote.ts +1 -1
  93. package/src/secret-value.test.ts +204 -0
  94. package/src/secret-value.ts +111 -0
  95. package/src/services/audit-service.ts +87 -9
  96. package/src/services/credential-wire-service.ts +9 -1
  97. package/src/services/email-service.ts +3 -1
  98. package/src/services/in-memory-workflow-service.ts +29 -0
  99. package/src/services/index.ts +3 -3
  100. package/src/services/local-content-request-handler.test.ts +202 -0
  101. package/src/services/local-content-request-handler.ts +267 -0
  102. package/src/services/local-secrets.test.ts +20 -5
  103. package/src/services/local-secrets.ts +15 -7
  104. package/src/services/logger.ts +27 -7
  105. package/src/services/queue-webhook-service.test.ts +2 -1
  106. package/src/services/queue-webhook-service.ts +1 -1
  107. package/src/services/scoped-secret-service.ts +4 -3
  108. package/src/services/secret-service.ts +8 -3
  109. package/src/services/typed-secret-service.ts +11 -7
  110. package/src/services/webhook-service.ts +4 -1
  111. package/src/testing/service-tests.ts +6 -6
  112. package/src/types/core.types.ts +25 -4
  113. package/src/wirings/ai-agent/ai-agent-agui.test.ts +16 -0
  114. package/src/wirings/ai-agent/ai-agent-agui.ts +14 -1
  115. package/src/wirings/ai-agent/ai-agent-prepare.ts +7 -1
  116. package/src/wirings/ai-agent/ai-agent-runner.ts +18 -2
  117. package/src/wirings/ai-agent/ai-agent-stream.ts +32 -1
  118. package/src/wirings/ai-agent/ai-agent.types.ts +45 -1
  119. package/src/wirings/ai-agent/index.ts +2 -0
  120. package/src/wirings/ai-agent/voice-input.test.ts +65 -0
  121. package/src/wirings/ai-agent/voice-input.ts +48 -9
  122. package/src/wirings/ai-agent/voice-output.test.ts +91 -1
  123. package/src/wirings/ai-agent/voice-output.ts +28 -1
  124. package/src/wirings/cli/channel/cli-raw-client-runner.ts +39 -9
  125. package/src/wirings/cli/channel/index.ts +4 -0
  126. package/src/wirings/persona/define-personas.ts +4 -0
  127. package/src/wirings/persona/persona.types.ts +11 -0
  128. package/src/wirings/queue/queue-identity.test.ts +2 -1
  129. package/src/wirings/queue/queue-identity.ts +4 -1
  130. package/src/wirings/queue/queue.types.ts +6 -1
  131. package/src/wirings/queue/signed-queue-service.ts +2 -1
  132. package/src/wirings/rpc/remote-addon-auth.ts +8 -3
  133. package/src/wirings/rpc/rpc-runner.test.ts +6 -4
  134. package/src/wirings/virtual-user/index.ts +12 -0
  135. package/src/wirings/virtual-user/prepare-virtual-user-run.test.ts +115 -0
  136. package/src/wirings/virtual-user/prepare-virtual-user-run.ts +95 -0
  137. package/src/wirings/virtual-user/virtual-user-run-store.ts +98 -0
  138. package/src/wirings/workflow/dsl/workflow-dsl.types.ts +14 -16
  139. package/src/wirings/workflow/pikku-scenario-service.ts +1 -10
  140. package/src/wirings/workflow/pikku-workflow-service.ts +67 -0
  141. package/src/wirings/workflow/scenario-prose.test.ts +5 -7
  142. package/src/wirings/workflow/scenario-prose.ts +1 -1
  143. package/src/wirings/workflow/scenario-service.test.ts +0 -1
  144. package/src/wirings/workflow/scenario-step.test.ts +4 -5
  145. package/src/wirings/workflow/scenario-step.types.ts +13 -7
  146. package/src/wirings/workflow/scenario-surface.test.ts +6 -5
  147. package/src/wirings/workflow/workflow-stalled-recovery.test.ts +106 -0
  148. package/src/wirings/workflow/workflow.types.ts +7 -0
  149. package/tsconfig.tsbuildinfo +1 -1
  150. package/tsconfig.type-tests.json +12 -0
@@ -1,6 +1,11 @@
1
+ import type { SecretValue } from '../secret-value.js'
2
+
3
+ /** A record of secrets, each still wrapped. */
4
+ export type SecretValues<T> = { [K in keyof T]: SecretValue<T[K]> }
5
+
1
6
  export interface SecretService {
2
- /** Throws if the secret is not found. */
3
- getSecret<T = string>(key: string): Promise<T>
7
+ /** Throws if the secret is not found. Unwrap the result with `.reveal()`. */
8
+ getSecret<T = string>(key: string): Promise<SecretValue<T>>
4
9
  /** Answers for any key, including a disallowed one — it must not throw. */
5
10
  hasSecret(key: string): Promise<boolean>
6
11
  setSecret(key: string, value: unknown): Promise<void>
@@ -12,5 +17,5 @@ export interface SecretService {
12
17
  */
13
18
  getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(
14
19
  keys: (keyof T & string)[]
15
- ): Promise<Partial<T>>
20
+ ): Promise<Partial<SecretValues<T>>>
16
21
  }
@@ -1,4 +1,5 @@
1
- import type { SecretService } from './secret-service.js'
1
+ import type { SecretValue } from '../secret-value.js'
2
+ import type { SecretService, SecretValues } from './secret-service.js'
2
3
 
3
4
  export interface CredentialStatus {
4
5
  secretId: string
@@ -25,8 +26,10 @@ export class TypedSecretService<
25
26
  private credentialsMeta: Record<string, CredentialMeta>
26
27
  ) {}
27
28
 
28
- async getSecret<K extends keyof TMap & string>(key: K): Promise<TMap[K]>
29
- async getSecret<T = string>(key: string): Promise<T>
29
+ async getSecret<K extends keyof TMap & string>(
30
+ key: K
31
+ ): Promise<SecretValue<TMap[K]>>
32
+ async getSecret<T = string>(key: string): Promise<SecretValue<T>>
30
33
  async getSecret(key: string): Promise<unknown> {
31
34
  if (this.cache.has(key)) {
32
35
  return this.cache.get(key)
@@ -58,12 +61,12 @@ export class TypedSecretService<
58
61
 
59
62
  async getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(
60
63
  keys: (keyof T & string)[]
61
- ): Promise<Partial<T>> {
62
- const result: Partial<T> = {}
64
+ ): Promise<Partial<SecretValues<T>>> {
65
+ const result: Partial<SecretValues<T>> = {}
63
66
  const missing: (keyof T & string)[] = []
64
67
  for (const key of keys) {
65
68
  if (this.cache.has(key)) {
66
- result[key] = this.cache.get(key) as T[keyof T & string]
69
+ result[key] = this.cache.get(key) as SecretValues<T>[keyof T & string]
67
70
  } else {
68
71
  missing.push(key)
69
72
  }
@@ -72,7 +75,8 @@ export class TypedSecretService<
72
75
  const fetched = await this.secrets.getSecrets<T>(missing)
73
76
  for (const [key, value] of Object.entries(fetched)) {
74
77
  this.cache.set(key, value)
75
- result[key as keyof T & string] = value as T[keyof T & string]
78
+ result[key as keyof T & string] = value as SecretValues<T>[keyof T &
79
+ string]
76
80
  }
77
81
  }
78
82
  return result
@@ -1,5 +1,6 @@
1
1
  import { NotImplementedError } from '../errors/errors.js'
2
2
  import { hmacSha256Hex, timingSafeStringEqual } from '../utils/hmac.js'
3
+ import type { Safe } from '../secret-value.js'
3
4
 
4
5
  export interface SendWebhookInput {
5
6
  url: string
@@ -90,7 +91,9 @@ export const DEFAULT_WEBHOOK_SIGNATURE_HEADER = 'X-Pikku-Signature'
90
91
  export const DEFAULT_WEBHOOK_RETRIES = 3
91
92
 
92
93
  export abstract class WebhookService {
93
- abstract send(input: SendWebhookInput): Promise<SendWebhookResult>
94
+ abstract send<T extends SendWebhookInput>(
95
+ input: Safe<T>
96
+ ): Promise<SendWebhookResult>
94
97
 
95
98
  /** Produces the header value, `sha256=<hex>`, not the bare digest. */
96
99
  protected sign(secret: string, body: string): string {
@@ -831,17 +831,17 @@ export function defineServiceTests(config: ServiceTestConfig): void {
831
831
  token: string
832
832
  endpoint: string
833
833
  }>('api-key')
834
- assert.deepEqual(result, {
834
+ assert.deepEqual(result.reveal(), {
835
835
  token: 'sk-123',
836
836
  endpoint: 'https://api.example.com',
837
837
  })
838
838
  })
839
839
 
840
- test('getSecret returns raw string', async () => {
840
+ test('getSecret wraps the raw string rather than returning it', async () => {
841
841
  const service = await factory({ key: kek })
842
842
  await service.setSecret('string-secret', 'plain-value')
843
843
  const result = await service.getSecret('string-secret')
844
- assert.strictEqual(result, 'plain-value')
844
+ assert.strictEqual(result.reveal(), 'plain-value')
845
845
  })
846
846
 
847
847
  test('hasSecret returns true/false', async () => {
@@ -862,7 +862,7 @@ export function defineServiceTests(config: ServiceTestConfig): void {
862
862
  await service.setSecret('upsert-key', { v: 1 })
863
863
  await service.setSecret('upsert-key', { v: 2 })
864
864
  const result = await service.getSecret<{ v: number }>('upsert-key')
865
- assert.deepEqual(result, { v: 2 })
865
+ assert.deepEqual(result.reveal(), { v: 2 })
866
866
  })
867
867
 
868
868
  test('deleteSecret removes the key', async () => {
@@ -887,7 +887,7 @@ export function defineServiceTests(config: ServiceTestConfig): void {
887
887
  const before = await rotatedService.getSecret<{
888
888
  important: string
889
889
  }>('rotate-test')
890
- assert.deepEqual(before, { important: 'data' })
890
+ assert.deepEqual(before.reveal(), { important: 'data' })
891
891
 
892
892
  assert.ok(rotatedService.rotateKEK)
893
893
  const count = await rotatedService.rotateKEK!()
@@ -900,7 +900,7 @@ export function defineServiceTests(config: ServiceTestConfig): void {
900
900
  const after = await newOnlyService.getSecret<{
901
901
  important: string
902
902
  }>('rotate-test')
903
- assert.deepEqual(after, { important: 'data' })
903
+ assert.deepEqual(after.reveal(), { important: 'data' })
904
904
  })
905
905
 
906
906
  test('rotateKEK throws without previousKey', async () => {
@@ -44,6 +44,7 @@ import type { AIAgentRunnerService } from '../services/ai-agent-runner-service.j
44
44
  import type { AIEmbeddingService } from '../services/ai-embedding-service.js'
45
45
  import type { AIRunStateService } from '../services/ai-run-state-service.js'
46
46
  import type { AgentRunService } from '../wirings/ai-agent/ai-agent.types.js'
47
+ import type { VirtualUserRunStore } from '../wirings/virtual-user/virtual-user-run-store.js'
47
48
  import type { PikkuAIMiddlewareHooks } from '../wirings/ai-agent/ai-agent.types.js'
48
49
  import type { WorkflowRunService } from '../wirings/workflow/workflow.types.js'
49
50
  import type { CredentialService } from '../services/credential-service.js'
@@ -125,11 +126,12 @@ export type FunctionRuntimeMeta = {
125
126
  */
126
127
  auth?: boolean
127
128
  /**
128
- * The author's declaration that this function authorizes callers in its own
129
- * body. Carries no runtime effect — it records a gate codegen cannot see, so
130
- * an audit is not left to guess whether a sessionless function is open.
129
+ * The author's declaration that this function's permission check lives in its
130
+ * own body. Carries no runtime effect — it records a gate codegen cannot see,
131
+ * so an audit is not left to guess whether a sessionless function is open.
132
+ * Refused unless `allow.permissionsInBody` is set in `pikku.config.json`.
131
133
  */
132
- selfAuthenticated?: boolean
134
+ permissionsInBody?: boolean
133
135
  remote?: boolean
134
136
  /**
135
137
  * A step RPC: invoked by name only from a scenario run and refused
@@ -173,6 +175,19 @@ export type FunctionRuntimeMeta = {
173
175
  scenarioStepAddon?: string
174
176
  /** Scenario steps only: the prose a reporter renders, with `{placeholders}` filled from the step's recorded input. */
175
177
  scenarioStepTemplate?: string
178
+ /**
179
+ * The function's `audit` config, resolved — `audit: true` reads as
180
+ * `{ durability: 'best-effort' }`. Absent means the function records nothing:
181
+ * `auditLog.write()` from an unmarked function is dropped with a warning, so
182
+ * this is the only place a reader can see which functions have a trail at all
183
+ * without running them.
184
+ *
185
+ * Informational. The runner resolves audit from the live function config, not
186
+ * from here, so meta and runtime cannot disagree about whether audit is on.
187
+ */
188
+ audit?: {
189
+ durability: AuditDurability
190
+ }
176
191
  /** Keeps the full `SecretService`. Set by the inspector, read by the runner. */
177
192
  secretBroker?: boolean
178
193
  version?: number
@@ -323,6 +338,12 @@ export interface CoreSingletonServices<Config extends CoreConfig = CoreConfig> {
323
338
  */
324
339
  webhookService?: WebhookService
325
340
  metaService?: MetaService
341
+ /**
342
+ * Where virtual-user runs are recorded. A run is dispatched and answered for
343
+ * later, so this store is the only trace it leaves — see
344
+ * {@link VirtualUserRunStore}.
345
+ */
346
+ virtualUserRunStore?: VirtualUserRunStore
326
347
  /** V8 precise-coverage collector (`pikku dev --coverage` only) */
327
348
  coverageService?: CoverageService
328
349
  audit?: AuditService
@@ -590,6 +590,22 @@ describe('wrapChannelWithAGUI — speech', () => {
590
590
  assert.ok(find(events, 'CUSTOM', 'pikku:audio-done'))
591
591
  })
592
592
 
593
+ it('forwards the transcript as a CUSTOM event', () => {
594
+ const { channel, events } = makeChannel()
595
+ const wrapped = wrapChannelWithAGUI(channel)
596
+
597
+ wrapped.send({
598
+ type: 'transcript',
599
+ text: 'what the user said',
600
+ } as AIStreamEvent)
601
+
602
+ // The client sent audio, so this is the only way it learns what its own
603
+ // turn was. Dropped, the user's message renders as an empty bubble.
604
+ const custom = find(events, 'CUSTOM', 'pikku:transcript')
605
+ assert.ok(custom)
606
+ assert.deepEqual(custom.value, { text: 'what the user said' })
607
+ })
608
+
593
609
  it('leaves an open text message open, so speech cannot split a reply', () => {
594
610
  const { channel, events } = makeChannel()
595
611
  const wrapped = wrapChannelWithAGUI(channel)
@@ -381,7 +381,11 @@ export function wrapChannelWithAGUI(
381
381
  send({
382
382
  type: 'CUSTOM',
383
383
  name: 'pikku:audio-delta',
384
- value: { data: event.data, format: event.format },
384
+ value: {
385
+ data: event.data,
386
+ format: event.format,
387
+ ...(event.text === undefined ? {} : { text: event.text }),
388
+ },
385
389
  })
386
390
  break
387
391
  }
@@ -390,6 +394,15 @@ export function wrapChannelWithAGUI(
390
394
  send({ type: 'CUSTOM', name: 'pikku:audio-done', value: {} })
391
395
  break
392
396
  }
397
+
398
+ case 'transcript': {
399
+ send({
400
+ type: 'CUSTOM',
401
+ name: 'pikku:transcript',
402
+ value: { text: event.text },
403
+ })
404
+ break
405
+ }
393
406
  }
394
407
  },
395
408
  }
@@ -959,7 +959,13 @@ export async function prepareAgentRun(
959
959
 
960
960
  const userContent: AIMessage['content'] = input.attachments?.length
961
961
  ? [
962
- { type: 'text' as const, text: input.message },
962
+ // Omitted when there is nothing to say. An attachment on its own is a
963
+ // real turn — a spoken one carries audio and no text at all — and an
964
+ // empty text part alongside it is a part providers are entitled to
965
+ // reject, for a caller who never wrote one.
966
+ ...(input.message
967
+ ? [{ type: 'text' as const, text: input.message }]
968
+ : []),
963
969
  ...input.attachments.map(
964
970
  (a) =>
965
971
  ({
@@ -144,6 +144,9 @@ export async function runAIAgent(
144
144
  ...(agent.aiMiddleware ?? []),
145
145
  ]
146
146
 
147
+ // One bag per run, shared by every middleware — see PikkuAIMiddlewareHooks.
148
+ const sharedNotes: Record<string, unknown> = {}
149
+
147
150
  let modifiedMessages = runnerParams.messages
148
151
  let modifiedInstructions = runnerParams.instructions
149
152
  for (const mw of aiMiddlewares) {
@@ -151,6 +154,7 @@ export async function runAIAgent(
151
154
  const result = await mw.modifyInput(singletonServices, {
152
155
  messages: modifiedMessages,
153
156
  instructions: modifiedInstructions,
157
+ shared: sharedNotes,
154
158
  })
155
159
  modifiedMessages = result.messages
156
160
  modifiedInstructions = result.instructions
@@ -159,6 +163,14 @@ export async function runAIAgent(
159
163
  runnerParams.messages = modifiedMessages
160
164
  runnerParams.instructions = modifiedInstructions
161
165
 
166
+ // History records what the model was asked, which for a spoken turn is the
167
+ // transcript rather than the base64 audio that arrived — see the same note on
168
+ // the streaming path. Identity-checked, because a middleware may legitimately
169
+ // replace the message list with something unrelated to this turn.
170
+ const lastModified = modifiedMessages[modifiedMessages.length - 1]
171
+ const persistedUserMessage =
172
+ lastModified?.id === userMessage.id ? lastModified : userMessage
173
+
162
174
  const runId = await aiRunState.createRun({
163
175
  agentName,
164
176
  threadId,
@@ -275,7 +287,7 @@ export async function runAIAgent(
275
287
  threadId,
276
288
  input.resourceId,
277
289
  memoryConfig,
278
- userMessage,
290
+ persistedUserMessage,
279
291
  { text: '', steps: completedStepsForSave }
280
292
  )
281
293
 
@@ -351,7 +363,7 @@ export async function runAIAgent(
351
363
  threadId,
352
364
  input.resourceId,
353
365
  memoryConfig,
354
- userMessage,
366
+ persistedUserMessage,
355
367
  {
356
368
  ...result,
357
369
  text: outputText,
@@ -608,6 +620,9 @@ async function continueAfterToolResultSync(
608
620
  }),
609
621
  ...(agent.aiMiddleware ?? []),
610
622
  ]
623
+ // One bag per run, shared by every middleware — see PikkuAIMiddlewareHooks.
624
+ const sharedNotes: Record<string, unknown> = {}
625
+
611
626
  let modifiedMessages = trimmedMessages
612
627
  let modifiedInstructions = instructions
613
628
  for (const mw of aiMiddlewares) {
@@ -615,6 +630,7 @@ async function continueAfterToolResultSync(
615
630
  const result = await mw.modifyInput(singletonServices, {
616
631
  messages: modifiedMessages,
617
632
  instructions: modifiedInstructions,
633
+ shared: sharedNotes,
618
634
  })
619
635
  modifiedMessages = result.messages
620
636
  modifiedInstructions = result.instructions
@@ -12,6 +12,7 @@ import type {
12
12
  import { pikkuState, getSingletonServices } from '../../pikku-state.js'
13
13
  import { AIProviderNotConfiguredError } from '../../errors/errors.js'
14
14
  import { randomUUID } from './ai-agent-utils.js'
15
+ import { SPOKEN_TRANSCRIPT } from './voice-input.js'
15
16
  import {
16
17
  combineChannelMiddleware,
17
18
  wrapChannelWithMiddleware,
@@ -672,6 +673,9 @@ export async function streamAIAgent(
672
673
  ...(agent.aiMiddleware ?? []),
673
674
  ]
674
675
 
676
+ // One bag per run, shared by every middleware — see PikkuAIMiddlewareHooks.
677
+ const sharedNotes: Record<string, unknown> = {}
678
+
675
679
  let modifiedMessages = runnerParams.messages
676
680
  let modifiedInstructions = runnerParams.instructions
677
681
  for (const mw of aiMiddlewares) {
@@ -679,6 +683,7 @@ export async function streamAIAgent(
679
683
  const result = await mw.modifyInput(singletonServices, {
680
684
  messages: modifiedMessages,
681
685
  instructions: modifiedInstructions,
686
+ shared: sharedNotes,
682
687
  })
683
688
  modifiedMessages = result.messages
684
689
  modifiedInstructions = result.instructions
@@ -687,6 +692,26 @@ export async function streamAIAgent(
687
692
  runnerParams.messages = modifiedMessages
688
693
  runnerParams.instructions = modifiedInstructions
689
694
 
695
+ // Sent on the raw channel, ahead of the run. A voice client sent audio and so
696
+ // has no idea what it said; until this arrives its own message is a blank
697
+ // bubble. Ahead of the run rather than alongside it because the answer starts
698
+ // streaming within a few hundred milliseconds, and a question that appears
699
+ // after its answer reads as the wrong question.
700
+ const transcript = sharedNotes[SPOKEN_TRANSCRIPT]
701
+ if (typeof transcript === 'string') {
702
+ channel.send({ type: 'transcript', text: transcript })
703
+ }
704
+
705
+ // What goes into thread history is what the model was actually asked. For a
706
+ // spoken turn that is not what arrived over the wire: the wire carried a
707
+ // base64 audio blob, and persisting it would write megabytes of unreadable
708
+ // data into the history while losing the only readable record of the turn.
709
+ // Identity-checked rather than assumed — a middleware is free to rewrite the
710
+ // message list into something with no relation to the turn.
711
+ const lastModified = modifiedMessages[modifiedMessages.length - 1]
712
+ const persistedUserMessage =
713
+ lastModified?.id === userMessage.id ? lastModified : userMessage
714
+
690
715
  const runId = await aiRunState.createRun({
691
716
  agentName,
692
717
  threadId,
@@ -706,7 +731,7 @@ export async function streamAIAgent(
706
731
  runnerParams.tools = trackToolExecution(runnerParams.tools, interruptHandle)
707
732
 
708
733
  if (storage) {
709
- await storage.saveMessages(threadId, [userMessage])
734
+ await storage.saveMessages(threadId, [persistedUserMessage])
710
735
  }
711
736
 
712
737
  const streamMiddleware = aiMiddlewares
@@ -720,6 +745,7 @@ export async function streamAIAgent(
720
745
  event,
721
746
  allEvents,
722
747
  state,
748
+ shared: sharedNotes,
723
749
  // Sends downstream directly, so a hook can hand back the fast event
724
750
  // now and push the slow one when it is ready.
725
751
  emit: next,
@@ -1288,6 +1314,9 @@ async function continueAfterToolResult(
1288
1314
  }),
1289
1315
  ...(agent.aiMiddleware ?? []),
1290
1316
  ]
1317
+ // One bag per run, shared by every middleware — see PikkuAIMiddlewareHooks.
1318
+ const sharedNotes: Record<string, unknown> = {}
1319
+
1291
1320
  let modifiedMessages = trimmedMessages
1292
1321
  let modifiedInstructions = instructions
1293
1322
  for (const mw of aiMiddlewares) {
@@ -1295,6 +1324,7 @@ async function continueAfterToolResult(
1295
1324
  const result = await mw.modifyInput(singletonServices, {
1296
1325
  messages: modifiedMessages,
1297
1326
  instructions: modifiedInstructions,
1327
+ shared: sharedNotes,
1298
1328
  })
1299
1329
  modifiedMessages = result.messages
1300
1330
  modifiedInstructions = result.instructions
@@ -1318,6 +1348,7 @@ async function continueAfterToolResult(
1318
1348
  event,
1319
1349
  allEvents,
1320
1350
  state,
1351
+ shared: sharedNotes,
1321
1352
  // Sends downstream directly, so a hook can hand back the fast event
1322
1353
  // now and push the slow one when it is ready.
1323
1354
  emit: next,
@@ -171,7 +171,27 @@ export interface PikkuAIMiddlewareHooks<
171
171
  > {
172
172
  modifyInput?: (
173
173
  services: Services,
174
- ctx: { messages: AIMessage[]; instructions: string }
174
+ ctx: {
175
+ messages: AIMessage[]
176
+ instructions: string
177
+ /**
178
+ * Notes about this run that every middleware can read and write, as
179
+ * opposed to {@link modifyOutputStream}'s `state`, which is private to
180
+ * one middleware.
181
+ *
182
+ * It exists because middlewares transform the turn for each other, and
183
+ * the transformation can destroy what a later one needed to know. Voice
184
+ * is the case in point: `voiceInput` replaces the user's audio with its
185
+ * transcript, so by the time anything downstream looks, the turn is
186
+ * indistinguishable from one that was typed — and whether it was typed is
187
+ * exactly what `voiceOutput` needs in order to decide whether to answer
188
+ * aloud. Only the middleware that consumed the audio can still say so.
189
+ *
190
+ * Namespace what you put here (`voice:spokenTurn`); it is one bag for
191
+ * everybody. Cleared between runs.
192
+ */
193
+ shared: Record<string, unknown>
194
+ }
175
195
  ) =>
176
196
  | Promise<{ messages: AIMessage[]; instructions: string }>
177
197
  | { messages: AIMessage[]; instructions: string }
@@ -181,7 +201,10 @@ export interface PikkuAIMiddlewareHooks<
181
201
  ctx: {
182
202
  event: AIStreamEvent
183
203
  allEvents: readonly AIStreamEvent[]
204
+ /** Private to this middleware, for this run. Cross-middleware facts go in `shared`. */
184
205
  state: State
206
+ /** Per-run notes shared with every middleware — see {@link modifyInput}. */
207
+ shared: Record<string, unknown>
185
208
  /**
186
209
  * Push an event into the stream *after* this call has returned.
187
210
  *
@@ -418,10 +441,31 @@ export type AIStreamEvent =
418
441
  type: 'audio-delta'
419
442
  data: string
420
443
  format: string
444
+ /**
445
+ * The sentence this audio says.
446
+ *
447
+ * Carried alongside the bytes because a client that gets talked over has
448
+ * to tell the model what the user actually heard, and playback position
449
+ * is the only place that is knowable. Without it a barge-in can stop the
450
+ * sound but not report it, and the next turn is answered as though the
451
+ * whole reply had landed.
452
+ */
453
+ text?: string
421
454
  agent?: string
422
455
  session?: string
423
456
  }
424
457
  | { type: 'audio-done'; agent?: string; session?: string }
458
+ | {
459
+ /**
460
+ * What the user was heard to say, sent once at the start of a spoken
461
+ * turn. Only the server knows this — the client sent audio — and a chat
462
+ * surface needs it to show the user's own message.
463
+ */
464
+ type: 'transcript'
465
+ text: string
466
+ agent?: string
467
+ session?: string
468
+ }
425
469
  | {
426
470
  type: 'data'
427
471
  name: string
@@ -16,6 +16,8 @@ export {
16
16
  voiceInput,
17
17
  readsAsNonSpeech,
18
18
  NoSpeechDetectedError,
19
+ SPOKEN_TURN,
20
+ SPOKEN_TRANSCRIPT,
19
21
  } from './voice-input.js'
20
22
  export {
21
23
  voiceOutput,
@@ -5,6 +5,8 @@ import {
5
5
  voiceInput,
6
6
  readsAsNonSpeech,
7
7
  NoSpeechDetectedError,
8
+ SPOKEN_TURN,
9
+ SPOKEN_TRANSCRIPT,
8
10
  } from './voice-input.js'
9
11
  import type { AIContentPart, AIMessage } from './ai-agent.types.js'
10
12
 
@@ -39,6 +41,7 @@ describe('voiceInput', () => {
39
41
  const result = await mw.modifyInput!(services as any, {
40
42
  messages: [audioMessage()],
41
43
  instructions: 'sys',
44
+ shared: {},
42
45
  })
43
46
 
44
47
  const last = result.messages[result.messages.length - 1]!
@@ -67,6 +70,7 @@ describe('voiceInput', () => {
67
70
  const result = await mw.modifyInput!(services as any, {
68
71
  messages,
69
72
  instructions: 'sys',
73
+ shared: {},
70
74
  })
71
75
 
72
76
  assert.equal(result.messages[0]!.content, 'just text')
@@ -81,6 +85,7 @@ describe('voiceInput', () => {
81
85
  mw.modifyInput!(services as any, {
82
86
  messages: [audioMessage()],
83
87
  instructions: 'sys',
88
+ shared: {},
84
89
  }),
85
90
  /voiceInput requires a transcription model/
86
91
  )
@@ -105,6 +110,7 @@ describe('voiceInput', () => {
105
110
  mw.modifyInput!({ aiAgentRunner: silent } as any, {
106
111
  messages: [audioMessage()],
107
112
  instructions: 'sys',
113
+ shared: {},
108
114
  }),
109
115
  NoSpeechDetectedError
110
116
  )
@@ -127,6 +133,7 @@ describe('voiceInput', () => {
127
133
  const result = await mw.modifyInput!({ aiAgentRunner: spaced } as any, {
128
134
  messages: [audioMessage()],
129
135
  instructions: 'sys',
136
+ shared: {},
130
137
  })
131
138
 
132
139
  const parts = result.messages.at(-1)!.content as AIContentPart[]
@@ -152,4 +159,62 @@ describe('readsAsNonSpeech', () => {
152
159
  assert.equal(readsAsNonSpeech({ text: 'yes' }), false)
153
160
  assert.equal(readsAsNonSpeech({ text: 'Thank you.' }), false)
154
161
  })
162
+ test('records that the turn was spoken, for voiceOutput to read', async () => {
163
+ // The note has to be written here: the transcript this middleware puts in
164
+ // the message's place is indistinguishable from something typed, so no
165
+ // later middleware can tell what it started as.
166
+ const mw = voiceInput({ model: 'mock/whisper' })
167
+ const shared: Record<string, unknown> = {}
168
+
169
+ await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
170
+ messages: [audioMessage()],
171
+ instructions: 'sys',
172
+ shared,
173
+ })
174
+
175
+ assert.equal(shared[SPOKEN_TURN], true)
176
+ })
177
+
178
+ test('records a typed turn as not spoken, rather than leaving it unsaid', async () => {
179
+ const mw = voiceInput({ model: 'mock/whisper' })
180
+ const shared: Record<string, unknown> = {}
181
+
182
+ await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
183
+ messages: [{ role: 'user', content: 'typed' } as any],
184
+ instructions: 'sys',
185
+ shared,
186
+ })
187
+
188
+ // Explicitly false, not absent: absent is what a caller with no voice input
189
+ // at all looks like, and that one should still be spoken to.
190
+ assert.equal(shared[SPOKEN_TURN], false)
191
+ })
192
+
193
+ test('records what was heard, so the client can show the user their own turn', async () => {
194
+ const mw = voiceInput({ model: 'mock/whisper' })
195
+ const shared: Record<string, unknown> = {}
196
+
197
+ await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
198
+ messages: [audioMessage()],
199
+ instructions: 'sys',
200
+ shared,
201
+ })
202
+
203
+ assert.equal(shared[SPOKEN_TRANSCRIPT], 'the transcribed spoken words')
204
+ })
205
+
206
+ test('a typed turn leaves no transcript behind', async () => {
207
+ // Absence is the signal: the stream only announces a transcript when there
208
+ // was one, and a typed message is already on the client's screen.
209
+ const mw = voiceInput({ model: 'mock/whisper' })
210
+ const shared: Record<string, unknown> = {}
211
+
212
+ await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
213
+ messages: [{ role: 'user', content: 'typed' } as any],
214
+ instructions: 'sys',
215
+ shared,
216
+ })
217
+
218
+ assert.equal(SPOKEN_TRANSCRIPT in shared, false)
219
+ })
155
220
  })