@pikku/core 0.12.74 → 0.12.78
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +397 -0
- package/dist/column-form.d.ts +32 -0
- package/dist/column-form.js +42 -0
- package/dist/crypto-utils.d.ts +15 -4
- package/dist/crypto-utils.js +18 -2
- package/dist/data-classification.d.ts +44 -0
- package/dist/function/functions.types.d.ts +23 -10
- package/dist/function/index.d.ts +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.js +3 -1
- package/dist/middleware/auth-bearer.js +2 -1
- package/dist/middleware/remote-auth.js +1 -1
- package/dist/remote.js +1 -1
- package/dist/secret-value.d.ts +56 -0
- package/dist/secret-value.js +46 -0
- package/dist/services/audit-service.d.ts +74 -4
- package/dist/services/audit-service.js +7 -5
- package/dist/services/credential-wire-service.d.ts +5 -0
- package/dist/services/credential-wire-service.js +9 -1
- package/dist/services/email-service.d.ts +2 -1
- package/dist/services/in-memory-workflow-service.d.ts +1 -0
- package/dist/services/in-memory-workflow-service.js +20 -0
- package/dist/services/index.d.ts +3 -3
- package/dist/services/index.js +1 -1
- package/dist/services/local-content-request-handler.d.ts +29 -0
- package/dist/services/local-content-request-handler.js +176 -0
- package/dist/services/local-secrets.d.ts +4 -3
- package/dist/services/local-secrets.js +7 -3
- package/dist/services/logger.d.ts +22 -5
- package/dist/services/queue-webhook-service.js +1 -1
- package/dist/services/scoped-secret-service.d.ts +4 -3
- package/dist/services/secret-service.d.ts +8 -3
- package/dist/services/typed-secret-service.d.ts +5 -4
- package/dist/services/webhook-service.d.ts +2 -1
- package/dist/testing/service-tests.js +6 -6
- package/dist/types/core.types.d.ts +25 -4
- package/dist/wirings/ai-agent/ai-agent-agui.js +13 -1
- package/dist/wirings/ai-agent/ai-agent-prepare.js +7 -1
- package/dist/wirings/ai-agent/ai-agent-runner.js +14 -2
- package/dist/wirings/ai-agent/ai-agent-stream.js +27 -1
- package/dist/wirings/ai-agent/ai-agent.types.d.ts +40 -0
- package/dist/wirings/ai-agent/index.d.ts +1 -1
- package/dist/wirings/ai-agent/index.js +1 -1
- package/dist/wirings/ai-agent/voice-input.d.ts +20 -0
- package/dist/wirings/ai-agent/voice-input.js +44 -9
- package/dist/wirings/ai-agent/voice-output.d.ts +15 -0
- package/dist/wirings/ai-agent/voice-output.js +10 -1
- package/dist/wirings/cli/channel/cli-raw-client-runner.d.ts +21 -3
- package/dist/wirings/cli/channel/cli-raw-client-runner.js +13 -5
- package/dist/wirings/cli/channel/index.d.ts +1 -0
- package/dist/wirings/persona/define-personas.d.ts +4 -0
- package/dist/wirings/persona/define-personas.js +4 -0
- package/dist/wirings/persona/persona.types.d.ts +11 -0
- package/dist/wirings/queue/queue-identity.js +2 -1
- package/dist/wirings/queue/queue.types.d.ts +2 -1
- package/dist/wirings/queue/signed-queue-service.d.ts +2 -1
- package/dist/wirings/rpc/remote-addon-auth.d.ts +2 -1
- package/dist/wirings/rpc/remote-addon-auth.js +6 -2
- package/dist/wirings/virtual-user/index.d.ts +3 -0
- package/dist/wirings/virtual-user/index.js +2 -0
- package/dist/wirings/virtual-user/prepare-virtual-user-run.d.ts +54 -0
- package/dist/wirings/virtual-user/prepare-virtual-user-run.js +49 -0
- package/dist/wirings/virtual-user/virtual-user-run-store.d.ts +90 -0
- package/dist/wirings/virtual-user/virtual-user-run-store.js +1 -0
- package/dist/wirings/workflow/dsl/workflow-dsl.types.d.ts +14 -10
- package/dist/wirings/workflow/pikku-scenario-service.js +1 -2
- package/dist/wirings/workflow/pikku-workflow-service.d.ts +35 -0
- package/dist/wirings/workflow/pikku-workflow-service.js +51 -0
- package/dist/wirings/workflow/scenario-prose.js +1 -1
- package/dist/wirings/workflow/scenario-step.types.d.ts +13 -7
- package/dist/wirings/workflow/workflow.types.d.ts +7 -0
- package/knowledge/decisions/internals/a-virtual-user-run-is-not-a-workflow-and-not-a-queued-job.md +48 -0
- package/knowledge/decisions/internals/core-column-form-is-an-axis-of-its-own.md +84 -0
- package/knowledge/decisions/internals/core-data-classification-brand-is-an-optional-property.md +9 -2
- package/knowledge/decisions/internals/index.md +4 -0
- package/knowledge/decisions/internals/one-project-shape-check-two-validators.md +53 -0
- package/knowledge/decisions/internals/scenarios-live-in-files-named-for-them.md +48 -0
- package/knowledge/decisions/internals/validate-checks-personas-through-a-shared-module.md +43 -0
- package/package.json +3 -2
- package/src/column-form.test.ts +97 -0
- package/src/column-form.ts +58 -0
- package/src/crypto-utils.ts +25 -6
- package/src/data-classification.ts +44 -0
- package/src/function/functions.types.ts +47 -10
- package/src/function/index.ts +1 -0
- package/src/index.ts +24 -2
- package/src/middleware/auth-bearer.test.ts +3 -2
- package/src/middleware/auth-bearer.ts +2 -1
- package/src/middleware/remote-auth.test.ts +2 -1
- package/src/middleware/remote-auth.ts +1 -1
- package/src/remote.test.ts +2 -1
- package/src/remote.ts +1 -1
- package/src/secret-value.test.ts +204 -0
- package/src/secret-value.ts +111 -0
- package/src/services/audit-service.ts +87 -9
- package/src/services/credential-wire-service.ts +9 -1
- package/src/services/email-service.ts +3 -1
- package/src/services/in-memory-workflow-service.ts +29 -0
- package/src/services/index.ts +3 -3
- package/src/services/local-content-request-handler.test.ts +202 -0
- package/src/services/local-content-request-handler.ts +267 -0
- package/src/services/local-secrets.test.ts +20 -5
- package/src/services/local-secrets.ts +15 -7
- package/src/services/logger.ts +27 -7
- package/src/services/queue-webhook-service.test.ts +2 -1
- package/src/services/queue-webhook-service.ts +1 -1
- package/src/services/scoped-secret-service.ts +4 -3
- package/src/services/secret-service.ts +8 -3
- package/src/services/typed-secret-service.ts +11 -7
- package/src/services/webhook-service.ts +4 -1
- package/src/testing/service-tests.ts +6 -6
- package/src/types/core.types.ts +25 -4
- package/src/wirings/ai-agent/ai-agent-agui.test.ts +16 -0
- package/src/wirings/ai-agent/ai-agent-agui.ts +14 -1
- package/src/wirings/ai-agent/ai-agent-prepare.ts +7 -1
- package/src/wirings/ai-agent/ai-agent-runner.ts +18 -2
- package/src/wirings/ai-agent/ai-agent-stream.ts +32 -1
- package/src/wirings/ai-agent/ai-agent.types.ts +45 -1
- package/src/wirings/ai-agent/index.ts +2 -0
- package/src/wirings/ai-agent/voice-input.test.ts +65 -0
- package/src/wirings/ai-agent/voice-input.ts +48 -9
- package/src/wirings/ai-agent/voice-output.test.ts +91 -1
- package/src/wirings/ai-agent/voice-output.ts +28 -1
- package/src/wirings/cli/channel/cli-raw-client-runner.ts +39 -9
- package/src/wirings/cli/channel/index.ts +4 -0
- package/src/wirings/persona/define-personas.ts +4 -0
- package/src/wirings/persona/persona.types.ts +11 -0
- package/src/wirings/queue/queue-identity.test.ts +2 -1
- package/src/wirings/queue/queue-identity.ts +4 -1
- package/src/wirings/queue/queue.types.ts +6 -1
- package/src/wirings/queue/signed-queue-service.ts +2 -1
- package/src/wirings/rpc/remote-addon-auth.ts +8 -3
- package/src/wirings/rpc/rpc-runner.test.ts +6 -4
- package/src/wirings/virtual-user/index.ts +12 -0
- package/src/wirings/virtual-user/prepare-virtual-user-run.test.ts +115 -0
- package/src/wirings/virtual-user/prepare-virtual-user-run.ts +95 -0
- package/src/wirings/virtual-user/virtual-user-run-store.ts +98 -0
- package/src/wirings/workflow/dsl/workflow-dsl.types.ts +14 -16
- package/src/wirings/workflow/pikku-scenario-service.ts +1 -10
- package/src/wirings/workflow/pikku-workflow-service.ts +67 -0
- package/src/wirings/workflow/scenario-prose.test.ts +5 -7
- package/src/wirings/workflow/scenario-prose.ts +1 -1
- package/src/wirings/workflow/scenario-service.test.ts +0 -1
- package/src/wirings/workflow/scenario-step.test.ts +4 -5
- package/src/wirings/workflow/scenario-step.types.ts +13 -7
- package/src/wirings/workflow/scenario-surface.test.ts +6 -5
- package/src/wirings/workflow/workflow-stalled-recovery.test.ts +106 -0
- package/src/wirings/workflow/workflow.types.ts +7 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/tsconfig.type-tests.json +12 -0
|
@@ -1,6 +1,11 @@
|
|
|
1
|
+
import type { SecretValue } from '../secret-value.js'
|
|
2
|
+
|
|
3
|
+
/** A record of secrets, each still wrapped. */
|
|
4
|
+
export type SecretValues<T> = { [K in keyof T]: SecretValue<T[K]> }
|
|
5
|
+
|
|
1
6
|
export interface SecretService {
|
|
2
|
-
/** Throws if the secret is not found. */
|
|
3
|
-
getSecret<T = string>(key: string): Promise<T
|
|
7
|
+
/** Throws if the secret is not found. Unwrap the result with `.reveal()`. */
|
|
8
|
+
getSecret<T = string>(key: string): Promise<SecretValue<T>>
|
|
4
9
|
/** Answers for any key, including a disallowed one — it must not throw. */
|
|
5
10
|
hasSecret(key: string): Promise<boolean>
|
|
6
11
|
setSecret(key: string, value: unknown): Promise<void>
|
|
@@ -12,5 +17,5 @@ export interface SecretService {
|
|
|
12
17
|
*/
|
|
13
18
|
getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(
|
|
14
19
|
keys: (keyof T & string)[]
|
|
15
|
-
): Promise<Partial<T
|
|
20
|
+
): Promise<Partial<SecretValues<T>>>
|
|
16
21
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { SecretValue } from '../secret-value.js'
|
|
2
|
+
import type { SecretService, SecretValues } from './secret-service.js'
|
|
2
3
|
|
|
3
4
|
export interface CredentialStatus {
|
|
4
5
|
secretId: string
|
|
@@ -25,8 +26,10 @@ export class TypedSecretService<
|
|
|
25
26
|
private credentialsMeta: Record<string, CredentialMeta>
|
|
26
27
|
) {}
|
|
27
28
|
|
|
28
|
-
async getSecret<K extends keyof TMap & string>(
|
|
29
|
-
|
|
29
|
+
async getSecret<K extends keyof TMap & string>(
|
|
30
|
+
key: K
|
|
31
|
+
): Promise<SecretValue<TMap[K]>>
|
|
32
|
+
async getSecret<T = string>(key: string): Promise<SecretValue<T>>
|
|
30
33
|
async getSecret(key: string): Promise<unknown> {
|
|
31
34
|
if (this.cache.has(key)) {
|
|
32
35
|
return this.cache.get(key)
|
|
@@ -58,12 +61,12 @@ export class TypedSecretService<
|
|
|
58
61
|
|
|
59
62
|
async getSecrets<T extends Record<string, unknown> = Record<string, unknown>>(
|
|
60
63
|
keys: (keyof T & string)[]
|
|
61
|
-
): Promise<Partial<T
|
|
62
|
-
const result: Partial<T
|
|
64
|
+
): Promise<Partial<SecretValues<T>>> {
|
|
65
|
+
const result: Partial<SecretValues<T>> = {}
|
|
63
66
|
const missing: (keyof T & string)[] = []
|
|
64
67
|
for (const key of keys) {
|
|
65
68
|
if (this.cache.has(key)) {
|
|
66
|
-
result[key] = this.cache.get(key) as T[keyof T & string]
|
|
69
|
+
result[key] = this.cache.get(key) as SecretValues<T>[keyof T & string]
|
|
67
70
|
} else {
|
|
68
71
|
missing.push(key)
|
|
69
72
|
}
|
|
@@ -72,7 +75,8 @@ export class TypedSecretService<
|
|
|
72
75
|
const fetched = await this.secrets.getSecrets<T>(missing)
|
|
73
76
|
for (const [key, value] of Object.entries(fetched)) {
|
|
74
77
|
this.cache.set(key, value)
|
|
75
|
-
result[key as keyof T & string] = value as T[keyof T &
|
|
78
|
+
result[key as keyof T & string] = value as SecretValues<T>[keyof T &
|
|
79
|
+
string]
|
|
76
80
|
}
|
|
77
81
|
}
|
|
78
82
|
return result
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { NotImplementedError } from '../errors/errors.js'
|
|
2
2
|
import { hmacSha256Hex, timingSafeStringEqual } from '../utils/hmac.js'
|
|
3
|
+
import type { Safe } from '../secret-value.js'
|
|
3
4
|
|
|
4
5
|
export interface SendWebhookInput {
|
|
5
6
|
url: string
|
|
@@ -90,7 +91,9 @@ export const DEFAULT_WEBHOOK_SIGNATURE_HEADER = 'X-Pikku-Signature'
|
|
|
90
91
|
export const DEFAULT_WEBHOOK_RETRIES = 3
|
|
91
92
|
|
|
92
93
|
export abstract class WebhookService {
|
|
93
|
-
abstract send
|
|
94
|
+
abstract send<T extends SendWebhookInput>(
|
|
95
|
+
input: Safe<T>
|
|
96
|
+
): Promise<SendWebhookResult>
|
|
94
97
|
|
|
95
98
|
/** Produces the header value, `sha256=<hex>`, not the bare digest. */
|
|
96
99
|
protected sign(secret: string, body: string): string {
|
|
@@ -831,17 +831,17 @@ export function defineServiceTests(config: ServiceTestConfig): void {
|
|
|
831
831
|
token: string
|
|
832
832
|
endpoint: string
|
|
833
833
|
}>('api-key')
|
|
834
|
-
assert.deepEqual(result, {
|
|
834
|
+
assert.deepEqual(result.reveal(), {
|
|
835
835
|
token: 'sk-123',
|
|
836
836
|
endpoint: 'https://api.example.com',
|
|
837
837
|
})
|
|
838
838
|
})
|
|
839
839
|
|
|
840
|
-
test('getSecret
|
|
840
|
+
test('getSecret wraps the raw string rather than returning it', async () => {
|
|
841
841
|
const service = await factory({ key: kek })
|
|
842
842
|
await service.setSecret('string-secret', 'plain-value')
|
|
843
843
|
const result = await service.getSecret('string-secret')
|
|
844
|
-
assert.strictEqual(result, 'plain-value')
|
|
844
|
+
assert.strictEqual(result.reveal(), 'plain-value')
|
|
845
845
|
})
|
|
846
846
|
|
|
847
847
|
test('hasSecret returns true/false', async () => {
|
|
@@ -862,7 +862,7 @@ export function defineServiceTests(config: ServiceTestConfig): void {
|
|
|
862
862
|
await service.setSecret('upsert-key', { v: 1 })
|
|
863
863
|
await service.setSecret('upsert-key', { v: 2 })
|
|
864
864
|
const result = await service.getSecret<{ v: number }>('upsert-key')
|
|
865
|
-
assert.deepEqual(result, { v: 2 })
|
|
865
|
+
assert.deepEqual(result.reveal(), { v: 2 })
|
|
866
866
|
})
|
|
867
867
|
|
|
868
868
|
test('deleteSecret removes the key', async () => {
|
|
@@ -887,7 +887,7 @@ export function defineServiceTests(config: ServiceTestConfig): void {
|
|
|
887
887
|
const before = await rotatedService.getSecret<{
|
|
888
888
|
important: string
|
|
889
889
|
}>('rotate-test')
|
|
890
|
-
assert.deepEqual(before, { important: 'data' })
|
|
890
|
+
assert.deepEqual(before.reveal(), { important: 'data' })
|
|
891
891
|
|
|
892
892
|
assert.ok(rotatedService.rotateKEK)
|
|
893
893
|
const count = await rotatedService.rotateKEK!()
|
|
@@ -900,7 +900,7 @@ export function defineServiceTests(config: ServiceTestConfig): void {
|
|
|
900
900
|
const after = await newOnlyService.getSecret<{
|
|
901
901
|
important: string
|
|
902
902
|
}>('rotate-test')
|
|
903
|
-
assert.deepEqual(after, { important: 'data' })
|
|
903
|
+
assert.deepEqual(after.reveal(), { important: 'data' })
|
|
904
904
|
})
|
|
905
905
|
|
|
906
906
|
test('rotateKEK throws without previousKey', async () => {
|
package/src/types/core.types.ts
CHANGED
|
@@ -44,6 +44,7 @@ import type { AIAgentRunnerService } from '../services/ai-agent-runner-service.j
|
|
|
44
44
|
import type { AIEmbeddingService } from '../services/ai-embedding-service.js'
|
|
45
45
|
import type { AIRunStateService } from '../services/ai-run-state-service.js'
|
|
46
46
|
import type { AgentRunService } from '../wirings/ai-agent/ai-agent.types.js'
|
|
47
|
+
import type { VirtualUserRunStore } from '../wirings/virtual-user/virtual-user-run-store.js'
|
|
47
48
|
import type { PikkuAIMiddlewareHooks } from '../wirings/ai-agent/ai-agent.types.js'
|
|
48
49
|
import type { WorkflowRunService } from '../wirings/workflow/workflow.types.js'
|
|
49
50
|
import type { CredentialService } from '../services/credential-service.js'
|
|
@@ -125,11 +126,12 @@ export type FunctionRuntimeMeta = {
|
|
|
125
126
|
*/
|
|
126
127
|
auth?: boolean
|
|
127
128
|
/**
|
|
128
|
-
* The author's declaration that this function
|
|
129
|
-
* body. Carries no runtime effect — it records a gate codegen cannot see,
|
|
130
|
-
* an audit is not left to guess whether a sessionless function is open.
|
|
129
|
+
* The author's declaration that this function's permission check lives in its
|
|
130
|
+
* own body. Carries no runtime effect — it records a gate codegen cannot see,
|
|
131
|
+
* so an audit is not left to guess whether a sessionless function is open.
|
|
132
|
+
* Refused unless `allow.permissionsInBody` is set in `pikku.config.json`.
|
|
131
133
|
*/
|
|
132
|
-
|
|
134
|
+
permissionsInBody?: boolean
|
|
133
135
|
remote?: boolean
|
|
134
136
|
/**
|
|
135
137
|
* A step RPC: invoked by name only from a scenario run and refused
|
|
@@ -173,6 +175,19 @@ export type FunctionRuntimeMeta = {
|
|
|
173
175
|
scenarioStepAddon?: string
|
|
174
176
|
/** Scenario steps only: the prose a reporter renders, with `{placeholders}` filled from the step's recorded input. */
|
|
175
177
|
scenarioStepTemplate?: string
|
|
178
|
+
/**
|
|
179
|
+
* The function's `audit` config, resolved — `audit: true` reads as
|
|
180
|
+
* `{ durability: 'best-effort' }`. Absent means the function records nothing:
|
|
181
|
+
* `auditLog.write()` from an unmarked function is dropped with a warning, so
|
|
182
|
+
* this is the only place a reader can see which functions have a trail at all
|
|
183
|
+
* without running them.
|
|
184
|
+
*
|
|
185
|
+
* Informational. The runner resolves audit from the live function config, not
|
|
186
|
+
* from here, so meta and runtime cannot disagree about whether audit is on.
|
|
187
|
+
*/
|
|
188
|
+
audit?: {
|
|
189
|
+
durability: AuditDurability
|
|
190
|
+
}
|
|
176
191
|
/** Keeps the full `SecretService`. Set by the inspector, read by the runner. */
|
|
177
192
|
secretBroker?: boolean
|
|
178
193
|
version?: number
|
|
@@ -323,6 +338,12 @@ export interface CoreSingletonServices<Config extends CoreConfig = CoreConfig> {
|
|
|
323
338
|
*/
|
|
324
339
|
webhookService?: WebhookService
|
|
325
340
|
metaService?: MetaService
|
|
341
|
+
/**
|
|
342
|
+
* Where virtual-user runs are recorded. A run is dispatched and answered for
|
|
343
|
+
* later, so this store is the only trace it leaves — see
|
|
344
|
+
* {@link VirtualUserRunStore}.
|
|
345
|
+
*/
|
|
346
|
+
virtualUserRunStore?: VirtualUserRunStore
|
|
326
347
|
/** V8 precise-coverage collector (`pikku dev --coverage` only) */
|
|
327
348
|
coverageService?: CoverageService
|
|
328
349
|
audit?: AuditService
|
|
@@ -590,6 +590,22 @@ describe('wrapChannelWithAGUI — speech', () => {
|
|
|
590
590
|
assert.ok(find(events, 'CUSTOM', 'pikku:audio-done'))
|
|
591
591
|
})
|
|
592
592
|
|
|
593
|
+
it('forwards the transcript as a CUSTOM event', () => {
|
|
594
|
+
const { channel, events } = makeChannel()
|
|
595
|
+
const wrapped = wrapChannelWithAGUI(channel)
|
|
596
|
+
|
|
597
|
+
wrapped.send({
|
|
598
|
+
type: 'transcript',
|
|
599
|
+
text: 'what the user said',
|
|
600
|
+
} as AIStreamEvent)
|
|
601
|
+
|
|
602
|
+
// The client sent audio, so this is the only way it learns what its own
|
|
603
|
+
// turn was. Dropped, the user's message renders as an empty bubble.
|
|
604
|
+
const custom = find(events, 'CUSTOM', 'pikku:transcript')
|
|
605
|
+
assert.ok(custom)
|
|
606
|
+
assert.deepEqual(custom.value, { text: 'what the user said' })
|
|
607
|
+
})
|
|
608
|
+
|
|
593
609
|
it('leaves an open text message open, so speech cannot split a reply', () => {
|
|
594
610
|
const { channel, events } = makeChannel()
|
|
595
611
|
const wrapped = wrapChannelWithAGUI(channel)
|
|
@@ -381,7 +381,11 @@ export function wrapChannelWithAGUI(
|
|
|
381
381
|
send({
|
|
382
382
|
type: 'CUSTOM',
|
|
383
383
|
name: 'pikku:audio-delta',
|
|
384
|
-
value: {
|
|
384
|
+
value: {
|
|
385
|
+
data: event.data,
|
|
386
|
+
format: event.format,
|
|
387
|
+
...(event.text === undefined ? {} : { text: event.text }),
|
|
388
|
+
},
|
|
385
389
|
})
|
|
386
390
|
break
|
|
387
391
|
}
|
|
@@ -390,6 +394,15 @@ export function wrapChannelWithAGUI(
|
|
|
390
394
|
send({ type: 'CUSTOM', name: 'pikku:audio-done', value: {} })
|
|
391
395
|
break
|
|
392
396
|
}
|
|
397
|
+
|
|
398
|
+
case 'transcript': {
|
|
399
|
+
send({
|
|
400
|
+
type: 'CUSTOM',
|
|
401
|
+
name: 'pikku:transcript',
|
|
402
|
+
value: { text: event.text },
|
|
403
|
+
})
|
|
404
|
+
break
|
|
405
|
+
}
|
|
393
406
|
}
|
|
394
407
|
},
|
|
395
408
|
}
|
|
@@ -959,7 +959,13 @@ export async function prepareAgentRun(
|
|
|
959
959
|
|
|
960
960
|
const userContent: AIMessage['content'] = input.attachments?.length
|
|
961
961
|
? [
|
|
962
|
-
|
|
962
|
+
// Omitted when there is nothing to say. An attachment on its own is a
|
|
963
|
+
// real turn — a spoken one carries audio and no text at all — and an
|
|
964
|
+
// empty text part alongside it is a part providers are entitled to
|
|
965
|
+
// reject, for a caller who never wrote one.
|
|
966
|
+
...(input.message
|
|
967
|
+
? [{ type: 'text' as const, text: input.message }]
|
|
968
|
+
: []),
|
|
963
969
|
...input.attachments.map(
|
|
964
970
|
(a) =>
|
|
965
971
|
({
|
|
@@ -144,6 +144,9 @@ export async function runAIAgent(
|
|
|
144
144
|
...(agent.aiMiddleware ?? []),
|
|
145
145
|
]
|
|
146
146
|
|
|
147
|
+
// One bag per run, shared by every middleware — see PikkuAIMiddlewareHooks.
|
|
148
|
+
const sharedNotes: Record<string, unknown> = {}
|
|
149
|
+
|
|
147
150
|
let modifiedMessages = runnerParams.messages
|
|
148
151
|
let modifiedInstructions = runnerParams.instructions
|
|
149
152
|
for (const mw of aiMiddlewares) {
|
|
@@ -151,6 +154,7 @@ export async function runAIAgent(
|
|
|
151
154
|
const result = await mw.modifyInput(singletonServices, {
|
|
152
155
|
messages: modifiedMessages,
|
|
153
156
|
instructions: modifiedInstructions,
|
|
157
|
+
shared: sharedNotes,
|
|
154
158
|
})
|
|
155
159
|
modifiedMessages = result.messages
|
|
156
160
|
modifiedInstructions = result.instructions
|
|
@@ -159,6 +163,14 @@ export async function runAIAgent(
|
|
|
159
163
|
runnerParams.messages = modifiedMessages
|
|
160
164
|
runnerParams.instructions = modifiedInstructions
|
|
161
165
|
|
|
166
|
+
// History records what the model was asked, which for a spoken turn is the
|
|
167
|
+
// transcript rather than the base64 audio that arrived — see the same note on
|
|
168
|
+
// the streaming path. Identity-checked, because a middleware may legitimately
|
|
169
|
+
// replace the message list with something unrelated to this turn.
|
|
170
|
+
const lastModified = modifiedMessages[modifiedMessages.length - 1]
|
|
171
|
+
const persistedUserMessage =
|
|
172
|
+
lastModified?.id === userMessage.id ? lastModified : userMessage
|
|
173
|
+
|
|
162
174
|
const runId = await aiRunState.createRun({
|
|
163
175
|
agentName,
|
|
164
176
|
threadId,
|
|
@@ -275,7 +287,7 @@ export async function runAIAgent(
|
|
|
275
287
|
threadId,
|
|
276
288
|
input.resourceId,
|
|
277
289
|
memoryConfig,
|
|
278
|
-
|
|
290
|
+
persistedUserMessage,
|
|
279
291
|
{ text: '', steps: completedStepsForSave }
|
|
280
292
|
)
|
|
281
293
|
|
|
@@ -351,7 +363,7 @@ export async function runAIAgent(
|
|
|
351
363
|
threadId,
|
|
352
364
|
input.resourceId,
|
|
353
365
|
memoryConfig,
|
|
354
|
-
|
|
366
|
+
persistedUserMessage,
|
|
355
367
|
{
|
|
356
368
|
...result,
|
|
357
369
|
text: outputText,
|
|
@@ -608,6 +620,9 @@ async function continueAfterToolResultSync(
|
|
|
608
620
|
}),
|
|
609
621
|
...(agent.aiMiddleware ?? []),
|
|
610
622
|
]
|
|
623
|
+
// One bag per run, shared by every middleware — see PikkuAIMiddlewareHooks.
|
|
624
|
+
const sharedNotes: Record<string, unknown> = {}
|
|
625
|
+
|
|
611
626
|
let modifiedMessages = trimmedMessages
|
|
612
627
|
let modifiedInstructions = instructions
|
|
613
628
|
for (const mw of aiMiddlewares) {
|
|
@@ -615,6 +630,7 @@ async function continueAfterToolResultSync(
|
|
|
615
630
|
const result = await mw.modifyInput(singletonServices, {
|
|
616
631
|
messages: modifiedMessages,
|
|
617
632
|
instructions: modifiedInstructions,
|
|
633
|
+
shared: sharedNotes,
|
|
618
634
|
})
|
|
619
635
|
modifiedMessages = result.messages
|
|
620
636
|
modifiedInstructions = result.instructions
|
|
@@ -12,6 +12,7 @@ import type {
|
|
|
12
12
|
import { pikkuState, getSingletonServices } from '../../pikku-state.js'
|
|
13
13
|
import { AIProviderNotConfiguredError } from '../../errors/errors.js'
|
|
14
14
|
import { randomUUID } from './ai-agent-utils.js'
|
|
15
|
+
import { SPOKEN_TRANSCRIPT } from './voice-input.js'
|
|
15
16
|
import {
|
|
16
17
|
combineChannelMiddleware,
|
|
17
18
|
wrapChannelWithMiddleware,
|
|
@@ -672,6 +673,9 @@ export async function streamAIAgent(
|
|
|
672
673
|
...(agent.aiMiddleware ?? []),
|
|
673
674
|
]
|
|
674
675
|
|
|
676
|
+
// One bag per run, shared by every middleware — see PikkuAIMiddlewareHooks.
|
|
677
|
+
const sharedNotes: Record<string, unknown> = {}
|
|
678
|
+
|
|
675
679
|
let modifiedMessages = runnerParams.messages
|
|
676
680
|
let modifiedInstructions = runnerParams.instructions
|
|
677
681
|
for (const mw of aiMiddlewares) {
|
|
@@ -679,6 +683,7 @@ export async function streamAIAgent(
|
|
|
679
683
|
const result = await mw.modifyInput(singletonServices, {
|
|
680
684
|
messages: modifiedMessages,
|
|
681
685
|
instructions: modifiedInstructions,
|
|
686
|
+
shared: sharedNotes,
|
|
682
687
|
})
|
|
683
688
|
modifiedMessages = result.messages
|
|
684
689
|
modifiedInstructions = result.instructions
|
|
@@ -687,6 +692,26 @@ export async function streamAIAgent(
|
|
|
687
692
|
runnerParams.messages = modifiedMessages
|
|
688
693
|
runnerParams.instructions = modifiedInstructions
|
|
689
694
|
|
|
695
|
+
// Sent on the raw channel, ahead of the run. A voice client sent audio and so
|
|
696
|
+
// has no idea what it said; until this arrives its own message is a blank
|
|
697
|
+
// bubble. Ahead of the run rather than alongside it because the answer starts
|
|
698
|
+
// streaming within a few hundred milliseconds, and a question that appears
|
|
699
|
+
// after its answer reads as the wrong question.
|
|
700
|
+
const transcript = sharedNotes[SPOKEN_TRANSCRIPT]
|
|
701
|
+
if (typeof transcript === 'string') {
|
|
702
|
+
channel.send({ type: 'transcript', text: transcript })
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
// What goes into thread history is what the model was actually asked. For a
|
|
706
|
+
// spoken turn that is not what arrived over the wire: the wire carried a
|
|
707
|
+
// base64 audio blob, and persisting it would write megabytes of unreadable
|
|
708
|
+
// data into the history while losing the only readable record of the turn.
|
|
709
|
+
// Identity-checked rather than assumed — a middleware is free to rewrite the
|
|
710
|
+
// message list into something with no relation to the turn.
|
|
711
|
+
const lastModified = modifiedMessages[modifiedMessages.length - 1]
|
|
712
|
+
const persistedUserMessage =
|
|
713
|
+
lastModified?.id === userMessage.id ? lastModified : userMessage
|
|
714
|
+
|
|
690
715
|
const runId = await aiRunState.createRun({
|
|
691
716
|
agentName,
|
|
692
717
|
threadId,
|
|
@@ -706,7 +731,7 @@ export async function streamAIAgent(
|
|
|
706
731
|
runnerParams.tools = trackToolExecution(runnerParams.tools, interruptHandle)
|
|
707
732
|
|
|
708
733
|
if (storage) {
|
|
709
|
-
await storage.saveMessages(threadId, [
|
|
734
|
+
await storage.saveMessages(threadId, [persistedUserMessage])
|
|
710
735
|
}
|
|
711
736
|
|
|
712
737
|
const streamMiddleware = aiMiddlewares
|
|
@@ -720,6 +745,7 @@ export async function streamAIAgent(
|
|
|
720
745
|
event,
|
|
721
746
|
allEvents,
|
|
722
747
|
state,
|
|
748
|
+
shared: sharedNotes,
|
|
723
749
|
// Sends downstream directly, so a hook can hand back the fast event
|
|
724
750
|
// now and push the slow one when it is ready.
|
|
725
751
|
emit: next,
|
|
@@ -1288,6 +1314,9 @@ async function continueAfterToolResult(
|
|
|
1288
1314
|
}),
|
|
1289
1315
|
...(agent.aiMiddleware ?? []),
|
|
1290
1316
|
]
|
|
1317
|
+
// One bag per run, shared by every middleware — see PikkuAIMiddlewareHooks.
|
|
1318
|
+
const sharedNotes: Record<string, unknown> = {}
|
|
1319
|
+
|
|
1291
1320
|
let modifiedMessages = trimmedMessages
|
|
1292
1321
|
let modifiedInstructions = instructions
|
|
1293
1322
|
for (const mw of aiMiddlewares) {
|
|
@@ -1295,6 +1324,7 @@ async function continueAfterToolResult(
|
|
|
1295
1324
|
const result = await mw.modifyInput(singletonServices, {
|
|
1296
1325
|
messages: modifiedMessages,
|
|
1297
1326
|
instructions: modifiedInstructions,
|
|
1327
|
+
shared: sharedNotes,
|
|
1298
1328
|
})
|
|
1299
1329
|
modifiedMessages = result.messages
|
|
1300
1330
|
modifiedInstructions = result.instructions
|
|
@@ -1318,6 +1348,7 @@ async function continueAfterToolResult(
|
|
|
1318
1348
|
event,
|
|
1319
1349
|
allEvents,
|
|
1320
1350
|
state,
|
|
1351
|
+
shared: sharedNotes,
|
|
1321
1352
|
// Sends downstream directly, so a hook can hand back the fast event
|
|
1322
1353
|
// now and push the slow one when it is ready.
|
|
1323
1354
|
emit: next,
|
|
@@ -171,7 +171,27 @@ export interface PikkuAIMiddlewareHooks<
|
|
|
171
171
|
> {
|
|
172
172
|
modifyInput?: (
|
|
173
173
|
services: Services,
|
|
174
|
-
ctx: {
|
|
174
|
+
ctx: {
|
|
175
|
+
messages: AIMessage[]
|
|
176
|
+
instructions: string
|
|
177
|
+
/**
|
|
178
|
+
* Notes about this run that every middleware can read and write, as
|
|
179
|
+
* opposed to {@link modifyOutputStream}'s `state`, which is private to
|
|
180
|
+
* one middleware.
|
|
181
|
+
*
|
|
182
|
+
* It exists because middlewares transform the turn for each other, and
|
|
183
|
+
* the transformation can destroy what a later one needed to know. Voice
|
|
184
|
+
* is the case in point: `voiceInput` replaces the user's audio with its
|
|
185
|
+
* transcript, so by the time anything downstream looks, the turn is
|
|
186
|
+
* indistinguishable from one that was typed — and whether it was typed is
|
|
187
|
+
* exactly what `voiceOutput` needs in order to decide whether to answer
|
|
188
|
+
* aloud. Only the middleware that consumed the audio can still say so.
|
|
189
|
+
*
|
|
190
|
+
* Namespace what you put here (`voice:spokenTurn`); it is one bag for
|
|
191
|
+
* everybody. Cleared between runs.
|
|
192
|
+
*/
|
|
193
|
+
shared: Record<string, unknown>
|
|
194
|
+
}
|
|
175
195
|
) =>
|
|
176
196
|
| Promise<{ messages: AIMessage[]; instructions: string }>
|
|
177
197
|
| { messages: AIMessage[]; instructions: string }
|
|
@@ -181,7 +201,10 @@ export interface PikkuAIMiddlewareHooks<
|
|
|
181
201
|
ctx: {
|
|
182
202
|
event: AIStreamEvent
|
|
183
203
|
allEvents: readonly AIStreamEvent[]
|
|
204
|
+
/** Private to this middleware, for this run. Cross-middleware facts go in `shared`. */
|
|
184
205
|
state: State
|
|
206
|
+
/** Per-run notes shared with every middleware — see {@link modifyInput}. */
|
|
207
|
+
shared: Record<string, unknown>
|
|
185
208
|
/**
|
|
186
209
|
* Push an event into the stream *after* this call has returned.
|
|
187
210
|
*
|
|
@@ -418,10 +441,31 @@ export type AIStreamEvent =
|
|
|
418
441
|
type: 'audio-delta'
|
|
419
442
|
data: string
|
|
420
443
|
format: string
|
|
444
|
+
/**
|
|
445
|
+
* The sentence this audio says.
|
|
446
|
+
*
|
|
447
|
+
* Carried alongside the bytes because a client that gets talked over has
|
|
448
|
+
* to tell the model what the user actually heard, and playback position
|
|
449
|
+
* is the only place that is knowable. Without it a barge-in can stop the
|
|
450
|
+
* sound but not report it, and the next turn is answered as though the
|
|
451
|
+
* whole reply had landed.
|
|
452
|
+
*/
|
|
453
|
+
text?: string
|
|
421
454
|
agent?: string
|
|
422
455
|
session?: string
|
|
423
456
|
}
|
|
424
457
|
| { type: 'audio-done'; agent?: string; session?: string }
|
|
458
|
+
| {
|
|
459
|
+
/**
|
|
460
|
+
* What the user was heard to say, sent once at the start of a spoken
|
|
461
|
+
* turn. Only the server knows this — the client sent audio — and a chat
|
|
462
|
+
* surface needs it to show the user's own message.
|
|
463
|
+
*/
|
|
464
|
+
type: 'transcript'
|
|
465
|
+
text: string
|
|
466
|
+
agent?: string
|
|
467
|
+
session?: string
|
|
468
|
+
}
|
|
425
469
|
| {
|
|
426
470
|
type: 'data'
|
|
427
471
|
name: string
|
|
@@ -5,6 +5,8 @@ import {
|
|
|
5
5
|
voiceInput,
|
|
6
6
|
readsAsNonSpeech,
|
|
7
7
|
NoSpeechDetectedError,
|
|
8
|
+
SPOKEN_TURN,
|
|
9
|
+
SPOKEN_TRANSCRIPT,
|
|
8
10
|
} from './voice-input.js'
|
|
9
11
|
import type { AIContentPart, AIMessage } from './ai-agent.types.js'
|
|
10
12
|
|
|
@@ -39,6 +41,7 @@ describe('voiceInput', () => {
|
|
|
39
41
|
const result = await mw.modifyInput!(services as any, {
|
|
40
42
|
messages: [audioMessage()],
|
|
41
43
|
instructions: 'sys',
|
|
44
|
+
shared: {},
|
|
42
45
|
})
|
|
43
46
|
|
|
44
47
|
const last = result.messages[result.messages.length - 1]!
|
|
@@ -67,6 +70,7 @@ describe('voiceInput', () => {
|
|
|
67
70
|
const result = await mw.modifyInput!(services as any, {
|
|
68
71
|
messages,
|
|
69
72
|
instructions: 'sys',
|
|
73
|
+
shared: {},
|
|
70
74
|
})
|
|
71
75
|
|
|
72
76
|
assert.equal(result.messages[0]!.content, 'just text')
|
|
@@ -81,6 +85,7 @@ describe('voiceInput', () => {
|
|
|
81
85
|
mw.modifyInput!(services as any, {
|
|
82
86
|
messages: [audioMessage()],
|
|
83
87
|
instructions: 'sys',
|
|
88
|
+
shared: {},
|
|
84
89
|
}),
|
|
85
90
|
/voiceInput requires a transcription model/
|
|
86
91
|
)
|
|
@@ -105,6 +110,7 @@ describe('voiceInput', () => {
|
|
|
105
110
|
mw.modifyInput!({ aiAgentRunner: silent } as any, {
|
|
106
111
|
messages: [audioMessage()],
|
|
107
112
|
instructions: 'sys',
|
|
113
|
+
shared: {},
|
|
108
114
|
}),
|
|
109
115
|
NoSpeechDetectedError
|
|
110
116
|
)
|
|
@@ -127,6 +133,7 @@ describe('voiceInput', () => {
|
|
|
127
133
|
const result = await mw.modifyInput!({ aiAgentRunner: spaced } as any, {
|
|
128
134
|
messages: [audioMessage()],
|
|
129
135
|
instructions: 'sys',
|
|
136
|
+
shared: {},
|
|
130
137
|
})
|
|
131
138
|
|
|
132
139
|
const parts = result.messages.at(-1)!.content as AIContentPart[]
|
|
@@ -152,4 +159,62 @@ describe('readsAsNonSpeech', () => {
|
|
|
152
159
|
assert.equal(readsAsNonSpeech({ text: 'yes' }), false)
|
|
153
160
|
assert.equal(readsAsNonSpeech({ text: 'Thank you.' }), false)
|
|
154
161
|
})
|
|
162
|
+
test('records that the turn was spoken, for voiceOutput to read', async () => {
|
|
163
|
+
// The note has to be written here: the transcript this middleware puts in
|
|
164
|
+
// the message's place is indistinguishable from something typed, so no
|
|
165
|
+
// later middleware can tell what it started as.
|
|
166
|
+
const mw = voiceInput({ model: 'mock/whisper' })
|
|
167
|
+
const shared: Record<string, unknown> = {}
|
|
168
|
+
|
|
169
|
+
await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
|
|
170
|
+
messages: [audioMessage()],
|
|
171
|
+
instructions: 'sys',
|
|
172
|
+
shared,
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
assert.equal(shared[SPOKEN_TURN], true)
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
test('records a typed turn as not spoken, rather than leaving it unsaid', async () => {
|
|
179
|
+
const mw = voiceInput({ model: 'mock/whisper' })
|
|
180
|
+
const shared: Record<string, unknown> = {}
|
|
181
|
+
|
|
182
|
+
await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
|
|
183
|
+
messages: [{ role: 'user', content: 'typed' } as any],
|
|
184
|
+
instructions: 'sys',
|
|
185
|
+
shared,
|
|
186
|
+
})
|
|
187
|
+
|
|
188
|
+
// Explicitly false, not absent: absent is what a caller with no voice input
|
|
189
|
+
// at all looks like, and that one should still be spoken to.
|
|
190
|
+
assert.equal(shared[SPOKEN_TURN], false)
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
test('records what was heard, so the client can show the user their own turn', async () => {
|
|
194
|
+
const mw = voiceInput({ model: 'mock/whisper' })
|
|
195
|
+
const shared: Record<string, unknown> = {}
|
|
196
|
+
|
|
197
|
+
await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
|
|
198
|
+
messages: [audioMessage()],
|
|
199
|
+
instructions: 'sys',
|
|
200
|
+
shared,
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
assert.equal(shared[SPOKEN_TRANSCRIPT], 'the transcribed spoken words')
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
test('a typed turn leaves no transcript behind', async () => {
|
|
207
|
+
// Absence is the signal: the stream only announces a transcript when there
|
|
208
|
+
// was one, and a typed message is already on the client's screen.
|
|
209
|
+
const mw = voiceInput({ model: 'mock/whisper' })
|
|
210
|
+
const shared: Record<string, unknown> = {}
|
|
211
|
+
|
|
212
|
+
await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
|
|
213
|
+
messages: [{ role: 'user', content: 'typed' } as any],
|
|
214
|
+
instructions: 'sys',
|
|
215
|
+
shared,
|
|
216
|
+
})
|
|
217
|
+
|
|
218
|
+
assert.equal(SPOKEN_TRANSCRIPT in shared, false)
|
|
219
|
+
})
|
|
155
220
|
})
|