@pikku/core 0.12.74 → 0.12.77
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +378 -0
- package/dist/column-form.d.ts +32 -0
- package/dist/column-form.js +42 -0
- package/dist/crypto-utils.d.ts +15 -4
- package/dist/crypto-utils.js +18 -2
- package/dist/data-classification.d.ts +44 -0
- package/dist/function/functions.types.d.ts +23 -10
- package/dist/function/index.d.ts +1 -1
- package/dist/index.d.ts +6 -3
- package/dist/index.js +3 -1
- package/dist/middleware/auth-bearer.js +2 -1
- package/dist/middleware/remote-auth.js +1 -1
- package/dist/remote.js +1 -1
- package/dist/secret-value.d.ts +56 -0
- package/dist/secret-value.js +46 -0
- package/dist/services/audit-service.d.ts +74 -4
- package/dist/services/audit-service.js +7 -5
- package/dist/services/credential-wire-service.d.ts +5 -0
- package/dist/services/credential-wire-service.js +9 -1
- package/dist/services/email-service.d.ts +2 -1
- package/dist/services/index.d.ts +3 -3
- package/dist/services/index.js +1 -1
- package/dist/services/local-content-request-handler.d.ts +29 -0
- package/dist/services/local-content-request-handler.js +176 -0
- package/dist/services/local-secrets.d.ts +4 -3
- package/dist/services/local-secrets.js +7 -3
- package/dist/services/logger.d.ts +22 -5
- package/dist/services/queue-webhook-service.js +1 -1
- package/dist/services/scoped-secret-service.d.ts +4 -3
- package/dist/services/secret-service.d.ts +8 -3
- package/dist/services/typed-secret-service.d.ts +5 -4
- package/dist/services/webhook-service.d.ts +2 -1
- package/dist/testing/service-tests.js +6 -6
- package/dist/types/core.types.d.ts +25 -4
- package/dist/wirings/ai-agent/ai-agent-agui.js +13 -1
- package/dist/wirings/ai-agent/ai-agent-prepare.js +7 -1
- package/dist/wirings/ai-agent/ai-agent-runner.js +14 -2
- package/dist/wirings/ai-agent/ai-agent-stream.js +27 -1
- package/dist/wirings/ai-agent/ai-agent.types.d.ts +40 -0
- package/dist/wirings/ai-agent/index.d.ts +1 -1
- package/dist/wirings/ai-agent/index.js +1 -1
- package/dist/wirings/ai-agent/voice-input.d.ts +20 -0
- package/dist/wirings/ai-agent/voice-input.js +44 -9
- package/dist/wirings/ai-agent/voice-output.d.ts +15 -0
- package/dist/wirings/ai-agent/voice-output.js +10 -1
- package/dist/wirings/cli/channel/cli-raw-client-runner.d.ts +21 -3
- package/dist/wirings/cli/channel/cli-raw-client-runner.js +13 -5
- package/dist/wirings/cli/channel/index.d.ts +1 -0
- package/dist/wirings/persona/define-personas.d.ts +4 -0
- package/dist/wirings/persona/define-personas.js +4 -0
- package/dist/wirings/persona/persona.types.d.ts +11 -0
- package/dist/wirings/queue/queue-identity.js +2 -1
- package/dist/wirings/queue/queue.types.d.ts +2 -1
- package/dist/wirings/queue/signed-queue-service.d.ts +2 -1
- package/dist/wirings/rpc/remote-addon-auth.d.ts +2 -1
- package/dist/wirings/rpc/remote-addon-auth.js +6 -2
- package/dist/wirings/virtual-user/index.d.ts +3 -0
- package/dist/wirings/virtual-user/index.js +2 -0
- package/dist/wirings/virtual-user/prepare-virtual-user-run.d.ts +54 -0
- package/dist/wirings/virtual-user/prepare-virtual-user-run.js +49 -0
- package/dist/wirings/virtual-user/virtual-user-run-store.d.ts +90 -0
- package/dist/wirings/virtual-user/virtual-user-run-store.js +1 -0
- package/dist/wirings/workflow/dsl/workflow-dsl.types.d.ts +14 -10
- package/dist/wirings/workflow/pikku-scenario-service.js +1 -2
- package/dist/wirings/workflow/scenario-prose.js +1 -1
- package/dist/wirings/workflow/scenario-step.types.d.ts +13 -7
- package/dist/wirings/workflow/workflow.types.d.ts +7 -0
- package/knowledge/decisions/internals/a-virtual-user-run-is-not-a-workflow-and-not-a-queued-job.md +48 -0
- package/knowledge/decisions/internals/core-column-form-is-an-axis-of-its-own.md +84 -0
- package/knowledge/decisions/internals/core-data-classification-brand-is-an-optional-property.md +9 -2
- package/knowledge/decisions/internals/index.md +4 -0
- package/knowledge/decisions/internals/one-project-shape-check-two-validators.md +53 -0
- package/knowledge/decisions/internals/scenarios-live-in-files-named-for-them.md +48 -0
- package/knowledge/decisions/internals/validate-checks-personas-through-a-shared-module.md +43 -0
- package/package.json +3 -2
- package/src/column-form.test.ts +97 -0
- package/src/column-form.ts +58 -0
- package/src/crypto-utils.ts +25 -6
- package/src/data-classification.ts +44 -0
- package/src/function/functions.types.ts +47 -10
- package/src/function/index.ts +1 -0
- package/src/index.ts +24 -2
- package/src/middleware/auth-bearer.test.ts +3 -2
- package/src/middleware/auth-bearer.ts +2 -1
- package/src/middleware/remote-auth.test.ts +2 -1
- package/src/middleware/remote-auth.ts +1 -1
- package/src/remote.test.ts +2 -1
- package/src/remote.ts +1 -1
- package/src/secret-value.test.ts +204 -0
- package/src/secret-value.ts +111 -0
- package/src/services/audit-service.ts +87 -9
- package/src/services/credential-wire-service.ts +9 -1
- package/src/services/email-service.ts +3 -1
- package/src/services/index.ts +3 -3
- package/src/services/local-content-request-handler.test.ts +202 -0
- package/src/services/local-content-request-handler.ts +267 -0
- package/src/services/local-secrets.test.ts +20 -5
- package/src/services/local-secrets.ts +15 -7
- package/src/services/logger.ts +27 -7
- package/src/services/queue-webhook-service.test.ts +2 -1
- package/src/services/queue-webhook-service.ts +1 -1
- package/src/services/scoped-secret-service.ts +4 -3
- package/src/services/secret-service.ts +8 -3
- package/src/services/typed-secret-service.ts +11 -7
- package/src/services/webhook-service.ts +4 -1
- package/src/testing/service-tests.ts +6 -6
- package/src/types/core.types.ts +25 -4
- package/src/wirings/ai-agent/ai-agent-agui.test.ts +16 -0
- package/src/wirings/ai-agent/ai-agent-agui.ts +14 -1
- package/src/wirings/ai-agent/ai-agent-prepare.ts +7 -1
- package/src/wirings/ai-agent/ai-agent-runner.ts +18 -2
- package/src/wirings/ai-agent/ai-agent-stream.ts +32 -1
- package/src/wirings/ai-agent/ai-agent.types.ts +45 -1
- package/src/wirings/ai-agent/index.ts +2 -0
- package/src/wirings/ai-agent/voice-input.test.ts +65 -0
- package/src/wirings/ai-agent/voice-input.ts +48 -9
- package/src/wirings/ai-agent/voice-output.test.ts +91 -1
- package/src/wirings/ai-agent/voice-output.ts +28 -1
- package/src/wirings/cli/channel/cli-raw-client-runner.ts +39 -9
- package/src/wirings/cli/channel/index.ts +4 -0
- package/src/wirings/persona/define-personas.ts +4 -0
- package/src/wirings/persona/persona.types.ts +11 -0
- package/src/wirings/queue/queue-identity.test.ts +2 -1
- package/src/wirings/queue/queue-identity.ts +4 -1
- package/src/wirings/queue/queue.types.ts +6 -1
- package/src/wirings/queue/signed-queue-service.ts +2 -1
- package/src/wirings/rpc/remote-addon-auth.ts +8 -3
- package/src/wirings/rpc/rpc-runner.test.ts +6 -4
- package/src/wirings/virtual-user/index.ts +12 -0
- package/src/wirings/virtual-user/prepare-virtual-user-run.test.ts +115 -0
- package/src/wirings/virtual-user/prepare-virtual-user-run.ts +95 -0
- package/src/wirings/virtual-user/virtual-user-run-store.ts +98 -0
- package/src/wirings/workflow/dsl/workflow-dsl.types.ts +14 -16
- package/src/wirings/workflow/pikku-scenario-service.ts +1 -10
- package/src/wirings/workflow/scenario-prose.test.ts +5 -7
- package/src/wirings/workflow/scenario-prose.ts +1 -1
- package/src/wirings/workflow/scenario-service.test.ts +0 -1
- package/src/wirings/workflow/scenario-step.test.ts +4 -5
- package/src/wirings/workflow/scenario-step.types.ts +13 -7
- package/src/wirings/workflow/scenario-surface.test.ts +6 -5
- package/src/wirings/workflow/workflow.types.ts +7 -0
- package/tsconfig.tsbuildinfo +1 -1
- package/tsconfig.type-tests.json +12 -0
|
@@ -171,7 +171,27 @@ export interface PikkuAIMiddlewareHooks<
|
|
|
171
171
|
> {
|
|
172
172
|
modifyInput?: (
|
|
173
173
|
services: Services,
|
|
174
|
-
ctx: {
|
|
174
|
+
ctx: {
|
|
175
|
+
messages: AIMessage[]
|
|
176
|
+
instructions: string
|
|
177
|
+
/**
|
|
178
|
+
* Notes about this run that every middleware can read and write, as
|
|
179
|
+
* opposed to {@link modifyOutputStream}'s `state`, which is private to
|
|
180
|
+
* one middleware.
|
|
181
|
+
*
|
|
182
|
+
* It exists because middlewares transform the turn for each other, and
|
|
183
|
+
* the transformation can destroy what a later one needed to know. Voice
|
|
184
|
+
* is the case in point: `voiceInput` replaces the user's audio with its
|
|
185
|
+
* transcript, so by the time anything downstream looks, the turn is
|
|
186
|
+
* indistinguishable from one that was typed — and whether it was typed is
|
|
187
|
+
* exactly what `voiceOutput` needs in order to decide whether to answer
|
|
188
|
+
* aloud. Only the middleware that consumed the audio can still say so.
|
|
189
|
+
*
|
|
190
|
+
* Namespace what you put here (`voice:spokenTurn`); it is one bag for
|
|
191
|
+
* everybody. Cleared between runs.
|
|
192
|
+
*/
|
|
193
|
+
shared: Record<string, unknown>
|
|
194
|
+
}
|
|
175
195
|
) =>
|
|
176
196
|
| Promise<{ messages: AIMessage[]; instructions: string }>
|
|
177
197
|
| { messages: AIMessage[]; instructions: string }
|
|
@@ -181,7 +201,10 @@ export interface PikkuAIMiddlewareHooks<
|
|
|
181
201
|
ctx: {
|
|
182
202
|
event: AIStreamEvent
|
|
183
203
|
allEvents: readonly AIStreamEvent[]
|
|
204
|
+
/** Private to this middleware, for this run. Cross-middleware facts go in `shared`. */
|
|
184
205
|
state: State
|
|
206
|
+
/** Per-run notes shared with every middleware — see {@link modifyInput}. */
|
|
207
|
+
shared: Record<string, unknown>
|
|
185
208
|
/**
|
|
186
209
|
* Push an event into the stream *after* this call has returned.
|
|
187
210
|
*
|
|
@@ -418,10 +441,31 @@ export type AIStreamEvent =
|
|
|
418
441
|
type: 'audio-delta'
|
|
419
442
|
data: string
|
|
420
443
|
format: string
|
|
444
|
+
/**
|
|
445
|
+
* The sentence this audio says.
|
|
446
|
+
*
|
|
447
|
+
* Carried alongside the bytes because a client that gets talked over has
|
|
448
|
+
* to tell the model what the user actually heard, and playback position
|
|
449
|
+
* is the only place that is knowable. Without it a barge-in can stop the
|
|
450
|
+
* sound but not report it, and the next turn is answered as though the
|
|
451
|
+
* whole reply had landed.
|
|
452
|
+
*/
|
|
453
|
+
text?: string
|
|
421
454
|
agent?: string
|
|
422
455
|
session?: string
|
|
423
456
|
}
|
|
424
457
|
| { type: 'audio-done'; agent?: string; session?: string }
|
|
458
|
+
| {
|
|
459
|
+
/**
|
|
460
|
+
* What the user was heard to say, sent once at the start of a spoken
|
|
461
|
+
* turn. Only the server knows this — the client sent audio — and a chat
|
|
462
|
+
* surface needs it to show the user's own message.
|
|
463
|
+
*/
|
|
464
|
+
type: 'transcript'
|
|
465
|
+
text: string
|
|
466
|
+
agent?: string
|
|
467
|
+
session?: string
|
|
468
|
+
}
|
|
425
469
|
| {
|
|
426
470
|
type: 'data'
|
|
427
471
|
name: string
|
|
@@ -5,6 +5,8 @@ import {
|
|
|
5
5
|
voiceInput,
|
|
6
6
|
readsAsNonSpeech,
|
|
7
7
|
NoSpeechDetectedError,
|
|
8
|
+
SPOKEN_TURN,
|
|
9
|
+
SPOKEN_TRANSCRIPT,
|
|
8
10
|
} from './voice-input.js'
|
|
9
11
|
import type { AIContentPart, AIMessage } from './ai-agent.types.js'
|
|
10
12
|
|
|
@@ -39,6 +41,7 @@ describe('voiceInput', () => {
|
|
|
39
41
|
const result = await mw.modifyInput!(services as any, {
|
|
40
42
|
messages: [audioMessage()],
|
|
41
43
|
instructions: 'sys',
|
|
44
|
+
shared: {},
|
|
42
45
|
})
|
|
43
46
|
|
|
44
47
|
const last = result.messages[result.messages.length - 1]!
|
|
@@ -67,6 +70,7 @@ describe('voiceInput', () => {
|
|
|
67
70
|
const result = await mw.modifyInput!(services as any, {
|
|
68
71
|
messages,
|
|
69
72
|
instructions: 'sys',
|
|
73
|
+
shared: {},
|
|
70
74
|
})
|
|
71
75
|
|
|
72
76
|
assert.equal(result.messages[0]!.content, 'just text')
|
|
@@ -81,6 +85,7 @@ describe('voiceInput', () => {
|
|
|
81
85
|
mw.modifyInput!(services as any, {
|
|
82
86
|
messages: [audioMessage()],
|
|
83
87
|
instructions: 'sys',
|
|
88
|
+
shared: {},
|
|
84
89
|
}),
|
|
85
90
|
/voiceInput requires a transcription model/
|
|
86
91
|
)
|
|
@@ -105,6 +110,7 @@ describe('voiceInput', () => {
|
|
|
105
110
|
mw.modifyInput!({ aiAgentRunner: silent } as any, {
|
|
106
111
|
messages: [audioMessage()],
|
|
107
112
|
instructions: 'sys',
|
|
113
|
+
shared: {},
|
|
108
114
|
}),
|
|
109
115
|
NoSpeechDetectedError
|
|
110
116
|
)
|
|
@@ -127,6 +133,7 @@ describe('voiceInput', () => {
|
|
|
127
133
|
const result = await mw.modifyInput!({ aiAgentRunner: spaced } as any, {
|
|
128
134
|
messages: [audioMessage()],
|
|
129
135
|
instructions: 'sys',
|
|
136
|
+
shared: {},
|
|
130
137
|
})
|
|
131
138
|
|
|
132
139
|
const parts = result.messages.at(-1)!.content as AIContentPart[]
|
|
@@ -152,4 +159,62 @@ describe('readsAsNonSpeech', () => {
|
|
|
152
159
|
assert.equal(readsAsNonSpeech({ text: 'yes' }), false)
|
|
153
160
|
assert.equal(readsAsNonSpeech({ text: 'Thank you.' }), false)
|
|
154
161
|
})
|
|
162
|
+
test('records that the turn was spoken, for voiceOutput to read', async () => {
|
|
163
|
+
// The note has to be written here: the transcript this middleware puts in
|
|
164
|
+
// the message's place is indistinguishable from something typed, so no
|
|
165
|
+
// later middleware can tell what it started as.
|
|
166
|
+
const mw = voiceInput({ model: 'mock/whisper' })
|
|
167
|
+
const shared: Record<string, unknown> = {}
|
|
168
|
+
|
|
169
|
+
await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
|
|
170
|
+
messages: [audioMessage()],
|
|
171
|
+
instructions: 'sys',
|
|
172
|
+
shared,
|
|
173
|
+
})
|
|
174
|
+
|
|
175
|
+
assert.equal(shared[SPOKEN_TURN], true)
|
|
176
|
+
})
|
|
177
|
+
|
|
178
|
+
test('records a typed turn as not spoken, rather than leaving it unsaid', async () => {
|
|
179
|
+
const mw = voiceInput({ model: 'mock/whisper' })
|
|
180
|
+
const shared: Record<string, unknown> = {}
|
|
181
|
+
|
|
182
|
+
await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
|
|
183
|
+
messages: [{ role: 'user', content: 'typed' } as any],
|
|
184
|
+
instructions: 'sys',
|
|
185
|
+
shared,
|
|
186
|
+
})
|
|
187
|
+
|
|
188
|
+
// Explicitly false, not absent: absent is what a caller with no voice input
|
|
189
|
+
// at all looks like, and that one should still be spoken to.
|
|
190
|
+
assert.equal(shared[SPOKEN_TURN], false)
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
test('records what was heard, so the client can show the user their own turn', async () => {
|
|
194
|
+
const mw = voiceInput({ model: 'mock/whisper' })
|
|
195
|
+
const shared: Record<string, unknown> = {}
|
|
196
|
+
|
|
197
|
+
await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
|
|
198
|
+
messages: [audioMessage()],
|
|
199
|
+
instructions: 'sys',
|
|
200
|
+
shared,
|
|
201
|
+
})
|
|
202
|
+
|
|
203
|
+
assert.equal(shared[SPOKEN_TRANSCRIPT], 'the transcribed spoken words')
|
|
204
|
+
})
|
|
205
|
+
|
|
206
|
+
test('a typed turn leaves no transcript behind', async () => {
|
|
207
|
+
// Absence is the signal: the stream only announces a transcript when there
|
|
208
|
+
// was one, and a typed message is already on the client's screen.
|
|
209
|
+
const mw = voiceInput({ model: 'mock/whisper' })
|
|
210
|
+
const shared: Record<string, unknown> = {}
|
|
211
|
+
|
|
212
|
+
await mw.modifyInput!({ aiAgentRunner: new ThisDependentRunner() } as any, {
|
|
213
|
+
messages: [{ role: 'user', content: 'typed' } as any],
|
|
214
|
+
instructions: 'sys',
|
|
215
|
+
shared,
|
|
216
|
+
})
|
|
217
|
+
|
|
218
|
+
assert.equal(SPOKEN_TRANSCRIPT in shared, false)
|
|
219
|
+
})
|
|
155
220
|
})
|
|
@@ -70,13 +70,35 @@ async function fetchAsUint8Array(
|
|
|
70
70
|
return new Uint8Array(buffer)
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Shared-notes key recording whether the user's turn arrived as audio.
|
|
75
|
+
*
|
|
76
|
+
* Written here because this is the last point at which it is knowable: the
|
|
77
|
+
* transcript that replaces the audio is indistinguishable from something typed.
|
|
78
|
+
* Read by `voiceOutput` to decide whether to answer aloud.
|
|
79
|
+
*
|
|
80
|
+
* Absent — rather than `false` — when this middleware is not wired at all, which
|
|
81
|
+
* is what lets `voiceOutput` still speak for a caller that has no voice input.
|
|
82
|
+
*/
|
|
83
|
+
export const SPOKEN_TURN = 'voice:spokenTurn'
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Shared-notes key holding what the user was heard to say, when they spoke.
|
|
87
|
+
*
|
|
88
|
+
* Absent on a typed turn. Read by the stream wiring, which forwards it to the
|
|
89
|
+
* client as a `transcript` event — a voice client has no idea what its own
|
|
90
|
+
* audio said, and without this the user's turn shows up in the UI as an empty
|
|
91
|
+
* bubble followed by an answer to a question they cannot see.
|
|
92
|
+
*/
|
|
93
|
+
export const SPOKEN_TRANSCRIPT = 'voice:transcript'
|
|
94
|
+
|
|
73
95
|
export const voiceInput = (config?: {
|
|
74
96
|
language?: string
|
|
75
97
|
model?: string
|
|
76
98
|
allowedAudioHosts?: string[]
|
|
77
99
|
}) =>
|
|
78
100
|
pikkuAIMiddleware({
|
|
79
|
-
modifyInput: async (services, { messages, instructions }) => {
|
|
101
|
+
modifyInput: async (services, { messages, instructions, shared }) => {
|
|
80
102
|
const aiAgentRunner = (
|
|
81
103
|
services as {
|
|
82
104
|
aiAgentRunner?: AIAgentRunnerService
|
|
@@ -85,19 +107,22 @@ export const voiceInput = (config?: {
|
|
|
85
107
|
if (!aiAgentRunner?.transcribe) return { messages, instructions }
|
|
86
108
|
|
|
87
109
|
const last = messages[messages.length - 1]
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
const parts = last.content as AIContentPart[]
|
|
93
|
-
if (!parts) return { messages, instructions }
|
|
110
|
+
const parts =
|
|
111
|
+
last?.role === 'user' && typeof last.content !== 'string'
|
|
112
|
+
? (last.content as AIContentPart[] | undefined)
|
|
113
|
+
: undefined
|
|
94
114
|
|
|
95
|
-
const hasAudio = parts
|
|
115
|
+
const hasAudio = !!parts?.some(
|
|
96
116
|
(p) => p.type === 'file' && !!p.mediaType?.startsWith('audio/')
|
|
97
117
|
)
|
|
98
|
-
|
|
118
|
+
// Recorded on every turn this middleware sees, spoken or not, so that a
|
|
119
|
+
// typed turn is an explicit `false` downstream rather than a silence that
|
|
120
|
+
// could equally mean nobody was asked.
|
|
121
|
+
shared[SPOKEN_TURN] = hasAudio
|
|
122
|
+
if (!hasAudio || !parts) return { messages, instructions }
|
|
99
123
|
|
|
100
124
|
const updatedContent: AIContentPart[] = []
|
|
125
|
+
const heard: string[] = []
|
|
101
126
|
for (const p of parts) {
|
|
102
127
|
if (!(p.type === 'file' && p.mediaType?.startsWith('audio/'))) {
|
|
103
128
|
updatedContent.push(p)
|
|
@@ -111,6 +136,9 @@ export const voiceInput = (config?: {
|
|
|
111
136
|
const audioData = p.data
|
|
112
137
|
? base64ToUint8Array(p.data)
|
|
113
138
|
: await fetchAsUint8Array(p.url!, config.allowedAudioHosts)
|
|
139
|
+
if (audioData.byteLength > MAX_AUDIO_SIZE) {
|
|
140
|
+
throw new Error('Audio file exceeds maximum size')
|
|
141
|
+
}
|
|
114
142
|
const result = await aiAgentRunner.transcribe({
|
|
115
143
|
model: config.model,
|
|
116
144
|
audio: audioData,
|
|
@@ -128,6 +156,7 @@ export const voiceInput = (config?: {
|
|
|
128
156
|
// model will answer, and what it answers is a guess about what it could
|
|
129
157
|
// not hear.
|
|
130
158
|
if (readsAsNonSpeech(result)) continue
|
|
159
|
+
heard.push(result.text)
|
|
131
160
|
updatedContent.push({ type: 'text' as const, text: result.text })
|
|
132
161
|
}
|
|
133
162
|
|
|
@@ -135,6 +164,16 @@ export const voiceInput = (config?: {
|
|
|
135
164
|
// to send, and a message with no content is not a question.
|
|
136
165
|
if (updatedContent.length === 0) throw new NoSpeechDetectedError()
|
|
137
166
|
|
|
167
|
+
// Only when something was actually heard. A turn can carry audio that all
|
|
168
|
+
// reads as non-speech and still have content — an image with a silent
|
|
169
|
+
// caption clip — which leaves nothing above to throw. Recording `''` here
|
|
170
|
+
// would send a transcript event saying the user said nothing, and a client
|
|
171
|
+
// that tells "not transcribed yet" from "transcribed" by the key being
|
|
172
|
+
// absent would render the turn as an empty bubble rather than a pending one.
|
|
173
|
+
if (heard.length > 0) {
|
|
174
|
+
shared[SPOKEN_TRANSCRIPT] = heard.join(' ')
|
|
175
|
+
}
|
|
176
|
+
|
|
138
177
|
return {
|
|
139
178
|
messages: [
|
|
140
179
|
...messages.slice(0, -1),
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
voiceForText,
|
|
8
8
|
} from './voice-output.js'
|
|
9
9
|
import type { AIStreamEvent } from './ai-agent.types.js'
|
|
10
|
+
import { SPOKEN_TURN } from './voice-input.js'
|
|
10
11
|
|
|
11
12
|
const KOKORO_SCRIPTS = ['latin', 'devanagari', 'han', 'kana']
|
|
12
13
|
|
|
@@ -33,7 +34,8 @@ const deferred = () => {
|
|
|
33
34
|
*/
|
|
34
35
|
const createStream = (
|
|
35
36
|
mw: ReturnType<typeof voiceOutput>,
|
|
36
|
-
services: unknown
|
|
37
|
+
services: unknown,
|
|
38
|
+
shared: Record<string, unknown> = {}
|
|
37
39
|
) => {
|
|
38
40
|
const state: Record<string, unknown> = {}
|
|
39
41
|
const allEvents: AIStreamEvent[] = []
|
|
@@ -54,6 +56,7 @@ const createStream = (
|
|
|
54
56
|
event,
|
|
55
57
|
allEvents,
|
|
56
58
|
state,
|
|
59
|
+
shared,
|
|
57
60
|
emit: next,
|
|
58
61
|
})
|
|
59
62
|
if (result == null) return
|
|
@@ -314,6 +317,93 @@ describe('voiceOutput', () => {
|
|
|
314
317
|
])
|
|
315
318
|
})
|
|
316
319
|
|
|
320
|
+
test('a typed turn is answered in text only', async () => {
|
|
321
|
+
// The cost case. One agent serves both kinds of caller, so a reply to
|
|
322
|
+
// something typed must not quietly synthesize speech nobody asked to hear
|
|
323
|
+
// — it is billed per sentence, on every turn.
|
|
324
|
+
const asked: string[] = []
|
|
325
|
+
const services = {
|
|
326
|
+
aiAgentRunner: {
|
|
327
|
+
generateSpeech: async ({ text }: { text: string }) => {
|
|
328
|
+
asked.push(text)
|
|
329
|
+
return speechFor(text)
|
|
330
|
+
},
|
|
331
|
+
},
|
|
332
|
+
}
|
|
333
|
+
const stream = createStream(voiceOutput({ model: 'mock/tts' }), services, {
|
|
334
|
+
[SPOKEN_TURN]: false,
|
|
335
|
+
})
|
|
336
|
+
|
|
337
|
+
await stream.send({ type: 'text-delta', text: 'Added it.' })
|
|
338
|
+
await stream.send({ type: 'done' })
|
|
339
|
+
|
|
340
|
+
assert.deepEqual(asked, [])
|
|
341
|
+
assert.deepEqual(stream.types(), ['text-delta', 'done'])
|
|
342
|
+
})
|
|
343
|
+
|
|
344
|
+
test('a spoken turn is answered aloud', async () => {
|
|
345
|
+
const asked: string[] = []
|
|
346
|
+
const services = {
|
|
347
|
+
aiAgentRunner: {
|
|
348
|
+
generateSpeech: async ({ text }: { text: string }) => {
|
|
349
|
+
asked.push(text)
|
|
350
|
+
return speechFor(text)
|
|
351
|
+
},
|
|
352
|
+
},
|
|
353
|
+
}
|
|
354
|
+
const stream = createStream(voiceOutput({ model: 'mock/tts' }), services, {
|
|
355
|
+
[SPOKEN_TURN]: true,
|
|
356
|
+
})
|
|
357
|
+
|
|
358
|
+
await stream.send({ type: 'text-delta', text: 'Added it.' })
|
|
359
|
+
await stream.send({ type: 'done' })
|
|
360
|
+
|
|
361
|
+
assert.deepEqual(asked, ['Added it.'])
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
test('`always` speaks a typed turn too', async () => {
|
|
365
|
+
// Read-aloud modes exist, and for them speech is not a reply to speech.
|
|
366
|
+
const asked: string[] = []
|
|
367
|
+
const services = {
|
|
368
|
+
aiAgentRunner: {
|
|
369
|
+
generateSpeech: async ({ text }: { text: string }) => {
|
|
370
|
+
asked.push(text)
|
|
371
|
+
return speechFor(text)
|
|
372
|
+
},
|
|
373
|
+
},
|
|
374
|
+
}
|
|
375
|
+
const stream = createStream(
|
|
376
|
+
voiceOutput({ model: 'mock/tts', always: true }),
|
|
377
|
+
services,
|
|
378
|
+
{ [SPOKEN_TURN]: false }
|
|
379
|
+
)
|
|
380
|
+
|
|
381
|
+
await stream.send({ type: 'text-delta', text: 'Added it.' })
|
|
382
|
+
await stream.send({ type: 'done' })
|
|
383
|
+
|
|
384
|
+
assert.deepEqual(asked, ['Added it.'])
|
|
385
|
+
})
|
|
386
|
+
|
|
387
|
+
test('speaks when nothing reported how the turn arrived', async () => {
|
|
388
|
+
// voiceOutput without voiceInput: nobody set the note, so there is no
|
|
389
|
+
// evidence the turn was typed and the pre-existing behaviour stands.
|
|
390
|
+
const asked: string[] = []
|
|
391
|
+
const services = {
|
|
392
|
+
aiAgentRunner: {
|
|
393
|
+
generateSpeech: async ({ text }: { text: string }) => {
|
|
394
|
+
asked.push(text)
|
|
395
|
+
return speechFor(text)
|
|
396
|
+
},
|
|
397
|
+
},
|
|
398
|
+
}
|
|
399
|
+
const stream = createStream(voiceOutput({ model: 'mock/tts' }), services)
|
|
400
|
+
|
|
401
|
+
await stream.send({ type: 'text-delta', text: 'Added it.' })
|
|
402
|
+
await stream.send({ type: 'done' })
|
|
403
|
+
|
|
404
|
+
assert.deepEqual(asked, ['Added it.'])
|
|
405
|
+
})
|
|
406
|
+
|
|
317
407
|
test('a script with no voice mapped is still refused, not mis-voiced', async () => {
|
|
318
408
|
const asked: string[] = []
|
|
319
409
|
const services = {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AIAgentRunnerService } from '../../services/ai-agent-runner-service.js'
|
|
2
2
|
import { pikkuAIMiddleware } from '../../types/core.types.js'
|
|
3
|
+
import { SPOKEN_TURN } from './voice-input.js'
|
|
3
4
|
|
|
4
5
|
type VoiceOutputState = {
|
|
5
6
|
textBuffer?: string
|
|
@@ -178,15 +179,40 @@ export const voiceOutput = (config?: {
|
|
|
178
179
|
* the right default for a provider that genuinely is multilingual.
|
|
179
180
|
*/
|
|
180
181
|
speakableScripts?: SpeakableScripts
|
|
182
|
+
/**
|
|
183
|
+
* Speak every reply, including answers to turns that were typed.
|
|
184
|
+
*
|
|
185
|
+
* The default is to speak only what was spoken to. An agent is usually one
|
|
186
|
+
* agent serving both kinds of caller — the same thread is typed at from a
|
|
187
|
+
* desk and talked to from a phone — and synthesizing a reply nobody is
|
|
188
|
+
* listening to costs real money on every sentence of every turn. Whether the
|
|
189
|
+
* user is in a call is not something the client needs to declare: sending
|
|
190
|
+
* audio is what saying so looks like.
|
|
191
|
+
*
|
|
192
|
+
* Set this for the cases where speech is not a reply to speech at all — a
|
|
193
|
+
* read-aloud accessibility mode, a kiosk, anything whose whole output is
|
|
194
|
+
* meant to be heard.
|
|
195
|
+
*/
|
|
196
|
+
always?: boolean
|
|
181
197
|
}) =>
|
|
182
198
|
pikkuAIMiddleware<VoiceOutputState>({
|
|
183
|
-
modifyOutputStream: async (
|
|
199
|
+
modifyOutputStream: async (
|
|
200
|
+
services,
|
|
201
|
+
{ event, state, shared, emit, signal }
|
|
202
|
+
) => {
|
|
184
203
|
const { aiAgentRunner, logger } = services as {
|
|
185
204
|
aiAgentRunner?: AIAgentRunnerService
|
|
186
205
|
logger?: { error: (message: string) => void }
|
|
187
206
|
}
|
|
188
207
|
if (!aiAgentRunner?.generateSpeech) return event
|
|
189
208
|
|
|
209
|
+
// Only an explicit `false` silences the reply. `voiceInput` sets this on
|
|
210
|
+
// every turn it handles, so `false` means a real user really typed; the
|
|
211
|
+
// key being absent means nothing reported either way — no voice input is
|
|
212
|
+
// wired — and that caller's replies are spoken exactly as they were before
|
|
213
|
+
// this option existed.
|
|
214
|
+
if (!config?.always && shared[SPOKEN_TURN] === false) return event
|
|
215
|
+
|
|
190
216
|
/**
|
|
191
217
|
* Queue a finished sentence for speech.
|
|
192
218
|
*
|
|
@@ -261,6 +287,7 @@ export const voiceOutput = (config?: {
|
|
|
261
287
|
type: 'audio-delta',
|
|
262
288
|
data: bufferToBase64(result.audio.bytes),
|
|
263
289
|
format: result.audio.format,
|
|
290
|
+
text,
|
|
264
291
|
})
|
|
265
292
|
})
|
|
266
293
|
}
|
|
@@ -7,18 +7,48 @@ import type {
|
|
|
7
7
|
Capabilities,
|
|
8
8
|
} from '../../channel/channel-rpc.js'
|
|
9
9
|
import { approverForMode, takeApprovalFlags } from './cli-approval.js'
|
|
10
|
-
import type {
|
|
10
|
+
import type { Logger } from '../../../services/logger.js'
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Everything a renderer gets when the command ran on the server: a logger, and
|
|
14
|
+
* nothing else. There is no container on this side of the socket to resolve a
|
|
15
|
+
* service from, so this is the whole of it rather than a subset of
|
|
16
|
+
* `CoreSingletonServices` — which requires `config`, `variables` and `secrets`
|
|
17
|
+
* the client does not have and cannot invent.
|
|
18
|
+
*/
|
|
19
|
+
export type ClientCLIRenderServices = { logger: Logger }
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A renderer that runs on the client of a CLI-over-channel connection.
|
|
23
|
+
*
|
|
24
|
+
* Deliberately not `CorePikkuCLIRender`: that type's `Services` parameter is
|
|
25
|
+
* constrained to `CoreSingletonServices`, so the honest logger-only shape
|
|
26
|
+
* cannot be spelled through it.
|
|
27
|
+
*/
|
|
28
|
+
export type CorePikkuCLIClientRender<Data> = (
|
|
29
|
+
services: ClientCLIRenderServices,
|
|
30
|
+
data: Data,
|
|
31
|
+
session?: undefined
|
|
32
|
+
) => void | Promise<void>
|
|
11
33
|
|
|
12
34
|
/** The normal case once the server owns the command tree. */
|
|
13
|
-
const defaultJSONRenderer:
|
|
35
|
+
const defaultJSONRenderer: CorePikkuCLIClientRender<any> = (_services, data) => {
|
|
14
36
|
console.log(JSON.stringify(data))
|
|
15
37
|
}
|
|
16
38
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
39
|
+
// `console` is the whole logger a CLI wants — its output is the command's output.
|
|
40
|
+
// `setLevel` is a no-op rather than an omission because `Logger` requires it and
|
|
41
|
+
// there is no level to set: a renderer writes what the user asked to see.
|
|
42
|
+
const clientServices: ClientCLIRenderServices = {
|
|
43
|
+
logger: {
|
|
44
|
+
info: (...args: any[]) => console.info(...(args as [any])),
|
|
45
|
+
warn: (...args: any[]) => console.warn(...(args as [any])),
|
|
46
|
+
error: (...args: any[]) => console.error(...(args as [any])),
|
|
47
|
+
debug: (...args: any[]) => console.debug(...(args as [any])),
|
|
48
|
+
trace: (...args: any[]) => console.trace(...(args as [any])),
|
|
49
|
+
setLevel: () => {},
|
|
50
|
+
},
|
|
51
|
+
}
|
|
22
52
|
|
|
23
53
|
/**
|
|
24
54
|
* Runs a CLI command entirely on the server. Unlike `executeCLIViaChannel`,
|
|
@@ -40,8 +70,8 @@ export async function executeRawCLIViaChannel({
|
|
|
40
70
|
}: {
|
|
41
71
|
pikkuWS: any // CorePikkuWebsocket instance
|
|
42
72
|
args?: string[]
|
|
43
|
-
renderers?: Record<string,
|
|
44
|
-
defaultRenderer?:
|
|
73
|
+
renderers?: Record<string, CorePikkuCLIClientRender<any>>
|
|
74
|
+
defaultRenderer?: CorePikkuCLIClientRender<any>
|
|
45
75
|
capabilities?: Capabilities
|
|
46
76
|
approve?: ApprovalRequester
|
|
47
77
|
}): Promise<number> {
|
|
@@ -2,6 +2,10 @@ export { executeCLIViaChannel } from './cli-channel-runner.js'
|
|
|
2
2
|
export { handleRawCLI } from './cli-raw-channel-runner.js'
|
|
3
3
|
export type { RawCLIFrame, RawCLIResult } from './cli-raw-channel-runner.js'
|
|
4
4
|
export { executeRawCLIViaChannel } from './cli-raw-client-runner.js'
|
|
5
|
+
export type {
|
|
6
|
+
ClientCLIRenderServices,
|
|
7
|
+
CorePikkuCLIClientRender,
|
|
8
|
+
} from './cli-raw-client-runner.js'
|
|
5
9
|
export {
|
|
6
10
|
APPROVAL_FLAGS,
|
|
7
11
|
approverForMode,
|
|
@@ -11,6 +11,10 @@ import type { CorePersonas } from './persona.types.js'
|
|
|
11
11
|
* running: a persona that exists only to be acted upon (the account an admin
|
|
12
12
|
* bans, the colleague a document is shared with) is seeded and never run.
|
|
13
13
|
*
|
|
14
|
+
* Exactly one `definePersonas(...)` is allowed per codebase, so there is one
|
|
15
|
+
* place to read the cast from and one place to add to. A second call — even in
|
|
16
|
+
* the same file — fails the build.
|
|
17
|
+
*
|
|
14
18
|
* @example
|
|
15
19
|
* ```typescript
|
|
16
20
|
* definePersonas({
|
|
@@ -38,6 +38,16 @@ export type CorePersona = {
|
|
|
38
38
|
name: string
|
|
39
39
|
jobTitle?: string
|
|
40
40
|
description?: string
|
|
41
|
+
/**
|
|
42
|
+
* A picture of this person, as any URL a browser can load: a file your app
|
|
43
|
+
* serves, an asset on a CDN, a Gravatar link.
|
|
44
|
+
*
|
|
45
|
+
* Optional because it is decoration. Omitted, the console draws a colour and
|
|
46
|
+
* an icon derived from the persona's id, which is stable, needs no network and
|
|
47
|
+
* is more legible than the identicon a synthetic address would resolve to —
|
|
48
|
+
* which is also why nothing is derived here.
|
|
49
|
+
*/
|
|
50
|
+
avatarUrl?: string
|
|
41
51
|
/**
|
|
42
52
|
* The roles this person holds. Only system roles may be named — a custom
|
|
43
53
|
* role can be deleted from the console, so a persona pinned to one silently
|
|
@@ -112,6 +122,7 @@ export type PersonaMeta = {
|
|
|
112
122
|
name: string
|
|
113
123
|
jobTitle?: string
|
|
114
124
|
description?: string
|
|
125
|
+
avatarUrl?: string
|
|
115
126
|
roles: string[]
|
|
116
127
|
personality?: string
|
|
117
128
|
goals: string[]
|
|
@@ -21,6 +21,7 @@ import type {
|
|
|
21
21
|
QueueJob,
|
|
22
22
|
QueueService,
|
|
23
23
|
} from './queue.types.js'
|
|
24
|
+
import { createSecretValue } from '../../secret-value.js'
|
|
24
25
|
|
|
25
26
|
const QUEUE_NAME = 'identity-queue'
|
|
26
27
|
const SECRET = 'a'.repeat(48)
|
|
@@ -49,7 +50,7 @@ const createSecretService = (secrets: Record<string, string>) =>
|
|
|
49
50
|
if (value === undefined) {
|
|
50
51
|
throw new Error(`Secret not found: ${key}`)
|
|
51
52
|
}
|
|
52
|
-
return value
|
|
53
|
+
return createSecretValue(value)
|
|
53
54
|
},
|
|
54
55
|
hasSecret: async (key: string) => secrets[key] !== undefined,
|
|
55
56
|
setSecret: async () => {},
|
|
@@ -59,7 +59,10 @@ export const getQueueIdentitySecret = async (
|
|
|
59
59
|
secrets?: SecretService
|
|
60
60
|
): Promise<string | undefined> => {
|
|
61
61
|
try {
|
|
62
|
-
return (
|
|
62
|
+
return (
|
|
63
|
+
(await secrets?.getSecret(QUEUE_IDENTITY_SECRET_NAME))?.reveal() ||
|
|
64
|
+
undefined
|
|
65
|
+
)
|
|
63
66
|
} catch {
|
|
64
67
|
return undefined
|
|
65
68
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { CommonWireMeta } from '../../types/core.types.js'
|
|
2
2
|
import type { CorePikkuFunctionConfig } from '../../function/functions.types.js'
|
|
3
3
|
import type { QueueConfigMapping } from './validate-worker-config.js'
|
|
4
|
+
import type { Safe } from '../../secret-value.js'
|
|
4
5
|
|
|
5
6
|
export interface PikkuWorkerConfig {
|
|
6
7
|
name?: string
|
|
@@ -107,7 +108,11 @@ export interface JobOptions {
|
|
|
107
108
|
export interface QueueService {
|
|
108
109
|
readonly supportsResults: boolean
|
|
109
110
|
|
|
110
|
-
add<T>(
|
|
111
|
+
add<T>(
|
|
112
|
+
queueName: string,
|
|
113
|
+
data: Safe<T>,
|
|
114
|
+
options?: JobOptions
|
|
115
|
+
): Promise<string>
|
|
111
116
|
|
|
112
117
|
getJob<T, R>(queueName: string, jobId: string): Promise<QueueJob<T, R> | null>
|
|
113
118
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Safe } from '../../secret-value.js'
|
|
1
2
|
import type { Logger } from '../../services/logger.js'
|
|
2
3
|
import type { SecretService } from '../../services/secret-service.js'
|
|
3
4
|
import {
|
|
@@ -24,7 +25,7 @@ export class SignedQueueService implements QueueService {
|
|
|
24
25
|
|
|
25
26
|
public async add<T>(
|
|
26
27
|
queueName: string,
|
|
27
|
-
data: T
|
|
28
|
+
data: Safe<T>,
|
|
28
29
|
options?: JobOptions
|
|
29
30
|
): Promise<string> {
|
|
30
31
|
const pikkuUserId = options?.pikkuUserId
|