@axium/ai 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/LICENSE.md +157 -0
  2. package/README.md +2 -2
  3. package/db.json +114 -0
  4. package/dist/client/cli.d.ts +1 -0
  5. package/dist/client/cli.js +224 -0
  6. package/dist/client/config.d.ts +196 -0
  7. package/dist/client/config.js +55 -0
  8. package/dist/client/hooks.d.ts +3 -0
  9. package/dist/client/hooks.js +32 -0
  10. package/dist/client/local.d.ts +17 -0
  11. package/dist/client/local.js +112 -0
  12. package/dist/common.d.ts +1318 -0
  13. package/dist/common.js +569 -0
  14. package/dist/hooks.d.ts +13 -0
  15. package/dist/hooks.js +34 -0
  16. package/dist/hub.d.ts +4 -0
  17. package/dist/hub.js +75 -0
  18. package/dist/index.d.ts +1 -0
  19. package/dist/index.js +1 -0
  20. package/dist/integrations/notes.d.ts +1 -0
  21. package/dist/integrations/notes.js +104 -0
  22. package/dist/integrations/tasks.d.ts +1 -0
  23. package/dist/integrations/tasks.js +157 -0
  24. package/dist/loop.d.ts +75 -0
  25. package/dist/loop.js +239 -0
  26. package/dist/memory.d.ts +107 -0
  27. package/dist/memory.js +193 -0
  28. package/dist/node/chat.d.ts +50 -0
  29. package/dist/node/chat.js +60 -0
  30. package/dist/node/llama.d.ts +115 -0
  31. package/dist/node/llama.js +385 -0
  32. package/dist/node/markdown.d.ts +6 -0
  33. package/dist/node/markdown.js +285 -0
  34. package/dist/node/models.d.ts +34 -0
  35. package/dist/node/models.js +287 -0
  36. package/dist/node/web.d.ts +1 -0
  37. package/dist/node/web.js +203 -0
  38. package/dist/server/api/conversations.d.ts +1 -0
  39. package/dist/server/api/conversations.js +216 -0
  40. package/dist/server/api/gateway.d.ts +1 -0
  41. package/dist/server/api/gateway.js +113 -0
  42. package/dist/server/api/memories.d.ts +1 -0
  43. package/dist/server/api/memories.js +129 -0
  44. package/dist/server/api/models.d.ts +1 -0
  45. package/dist/server/api/models.js +161 -0
  46. package/dist/server/api/temporary.d.ts +1 -0
  47. package/dist/server/api/temporary.js +50 -0
  48. package/dist/server/api/timers.d.ts +1 -0
  49. package/dist/server/api/timers.js +27 -0
  50. package/dist/server/api/voice.d.ts +1 -0
  51. package/dist/server/api/voice.js +188 -0
  52. package/dist/server/generation.d.ts +38 -0
  53. package/dist/server/generation.js +299 -0
  54. package/dist/server/memory.d.ts +7 -0
  55. package/dist/server/memory.js +218 -0
  56. package/dist/server/models.d.ts +13 -0
  57. package/dist/server/models.js +59 -0
  58. package/dist/server/preferences.d.ts +3 -0
  59. package/dist/server/preferences.js +12 -0
  60. package/dist/server/timers.d.ts +7 -0
  61. package/dist/server/timers.js +66 -0
  62. package/dist/speech.d.ts +17 -0
  63. package/dist/speech.js +69 -0
  64. package/dist/timers.d.ts +1 -0
  65. package/dist/timers.js +67 -0
  66. package/dist/tools.d.ts +70 -0
  67. package/dist/tools.js +84 -0
  68. package/lib/ChatMessage.svelte +331 -0
  69. package/lib/Composer.svelte +234 -0
  70. package/lib/Conversation.svelte +433 -0
  71. package/lib/ConversationMenu.svelte +18 -0
  72. package/lib/ModelActions.svelte +71 -0
  73. package/lib/ModelOptions.svelte +101 -0
  74. package/lib/ModelStatus.svelte +49 -0
  75. package/lib/ToolCall.svelte +171 -0
  76. package/lib/VoiceMode.svelte +113 -0
  77. package/lib/actions.ts +21 -0
  78. package/lib/index.ts +7 -0
  79. package/lib/live.svelte.ts +113 -0
  80. package/lib/markdown.ts +40 -0
  81. package/lib/preferences.ts +13 -0
  82. package/lib/transition.ts +24 -0
  83. package/lib/tsconfig.json +14 -0
  84. package/lib/voice.svelte.ts +476 -0
  85. package/locales/en.json +316 -0
  86. package/package.json +139 -5
  87. package/routes/ai/+layout.svelte +177 -0
  88. package/routes/ai/+layout.ts +26 -0
  89. package/routes/ai/+page.svelte +109 -0
  90. package/routes/ai/TimerAlerts.svelte +51 -0
  91. package/routes/ai/Timers.svelte +121 -0
  92. package/routes/ai/[id]/+page.svelte +18 -0
  93. package/routes/ai/[id]/+page.ts +14 -0
  94. package/routes/ai/memory/+page.svelte +313 -0
  95. package/routes/ai/memory/+page.ts +10 -0
  96. package/routes/ai/memory/MemoryForm.svelte +111 -0
  97. package/routes/ai/models/+page.svelte +199 -0
  98. package/routes/ai/models/+page.ts +5 -0
  99. package/routes/ai/models/AddModel.svelte +240 -0
  100. package/routes/ai/models/MemoryMeter.svelte +120 -0
  101. package/routes/ai/models/Roles.svelte +159 -0
  102. package/routes/ai/models/[id]/+page.svelte +331 -0
  103. package/routes/ai/models/[id]/+page.ts +6 -0
  104. package/routes/ai/timers.svelte.ts +94 -0
  105. package/routes/tsconfig.json +14 -0
@@ -0,0 +1,476 @@
1
+ import * as ai from '@axium/ai/common';
2
+ import { SpeechBuffer } from '@axium/ai/speech';
3
+ import { logWarnings, prefix, token } from '@axium/client/requests';
4
+ import features from '@axium/core/features';
5
+
6
+ /** What voice features the instance's models allow */
7
+ export const available = $state({ dictation: false, speech: false });
8
+
9
+ /** Updates what is {@link available} from the instance's models */
10
+ export function useModels({ models, roles }: Pick<ai.Models, 'models' | 'roles'>): void {
11
+ const enabled = features('ai-voice');
12
+ available.dictation = enabled && !!navigator.mediaDevices && !!ai.resolveRole(models, roles, 'stt');
13
+ available.speech = enabled && !!ai.resolveRole(models, roles, 'tts');
14
+ }
15
+
16
+ async function request(path: string, init: RequestInit): Promise<Response> {
17
+ const headers = new Headers(init.headers);
18
+ if (token) headers.set('Authorization', 'Bearer ' + token);
19
+
20
+ const response = await fetch(prefix + path, { ...init, headers });
21
+ logWarnings(response);
22
+ if (response.ok) return response;
23
+
24
+ const body = await response.json().catch(() => null);
25
+ throw new Error(body?.message ?? response.statusText);
26
+ }
27
+
28
+ /** Samples per second of recordings sent for transcription */
29
+ const recordingRate = 16_000;
30
+
31
+ /** Mono 16-bit PCM in a WAV container */
32
+ function encodeWAV(samples: Float32Array, rate: number): Blob {
33
+ const view = new DataView(new ArrayBuffer(44 + samples.length * 2));
34
+ const writeText = (offset: number, text: string) => {
35
+ for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i));
36
+ };
37
+
38
+ writeText(0, 'RIFF');
39
+ view.setUint32(4, 36 + samples.length * 2, true);
40
+ writeText(8, 'WAVEfmt ');
41
+ view.setUint32(16, 16, true);
42
+ view.setUint16(20, 1, true);
43
+ view.setUint16(22, 1, true);
44
+ view.setUint32(24, rate, true);
45
+ view.setUint32(28, rate * 2, true);
46
+ view.setUint16(32, 2, true);
47
+ view.setUint16(34, 16, true);
48
+ writeText(36, 'data');
49
+ view.setUint32(40, samples.length * 2, true);
50
+
51
+ for (const [index, sample] of samples.entries()) {
52
+ const clamped = Math.max(-1, Math.min(1, sample));
53
+ view.setInt16(44 + index * 2, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
54
+ }
55
+
56
+ return new Blob([view], { type: 'audio/wav' });
57
+ }
58
+
59
+ /** Converts a recording in any format the browser can decode */
60
+ async function toWAV(recording: Blob): Promise<Blob> {
61
+ const context = new AudioContext();
62
+ try {
63
+ const decoded = await context.decodeAudioData(await recording.arrayBuffer());
64
+ const offline = new OfflineAudioContext(1, Math.ceil(decoded.duration * recordingRate), recordingRate);
65
+ const source = offline.createBufferSource();
66
+ source.buffer = decoded;
67
+ source.connect(offline.destination);
68
+ source.start();
69
+ return encodeWAV((await offline.startRendering()).getChannelData(0), recordingRate);
70
+ } finally {
71
+ void context.close();
72
+ }
73
+ }
74
+
75
+ export interface Recording {
76
+ /** Ends the recording and gives it as WAV */
77
+ stop(): Promise<Blob>;
78
+ /** Ends the recording and discards it */
79
+ cancel(): void;
80
+ }
81
+
82
+ export interface RecordOptions {
83
+ /** Called once the speaker pauses after talking */
84
+ onpause?(): void;
85
+ /** Called when nobody talks for a while after recording starts */
86
+ onsilence?(): void;
87
+ /** Called about 60 times a second with the input level, from 0 to 1 */
88
+ onlevel?(level: number): void;
89
+ }
90
+
91
+ /** How long a pause ends a recording, in milliseconds */
92
+ const pauseLength = 1200;
93
+
94
+ /** How long to wait for someone to talk, in milliseconds */
95
+ const silenceLength = 8000;
96
+
97
+ /** Watches a stream's input level to tell speech from background noise */
98
+ function detectSpeech(stream: MediaStream, options: RecordOptions): () => void {
99
+ const context = new AudioContext();
100
+ const analyser = context.createAnalyser();
101
+ analyser.fftSize = 1024;
102
+ context.createMediaStreamSource(stream).connect(analyser);
103
+
104
+ const samples = new Float32Array(analyser.fftSize);
105
+ const started = performance.now();
106
+ let noise = 0.01,
107
+ spoke = false,
108
+ lastSpeech = started,
109
+ frame = 0;
110
+
111
+ function check(): void {
112
+ analyser.getFloatTimeDomainData(samples);
113
+ const level = Math.sqrt(samples.reduce((sum, sample) => sum + sample * sample, 0) / samples.length);
114
+ const now = performance.now();
115
+ options.onlevel?.(Math.min(1, level * 8));
116
+
117
+ if (now - started < 300) noise = Math.max(noise, level);
118
+ else if (level > Math.max(0.02, noise * 2.5)) {
119
+ spoke = true;
120
+ lastSpeech = now;
121
+ }
122
+
123
+ if (spoke && now - lastSpeech > pauseLength) return options.onpause?.();
124
+ if (!spoke && now - started > silenceLength) return options.onsilence?.();
125
+ frame = requestAnimationFrame(check);
126
+ }
127
+ frame = requestAnimationFrame(check);
128
+
129
+ return () => {
130
+ cancelAnimationFrame(frame);
131
+ void context.close();
132
+ };
133
+ }
134
+
135
+ /**
136
+ * Starts recording from the microphone, asking for permission when needed.
137
+ * @throws when the microphone can't be used
138
+ */
139
+ export async function record(options: RecordOptions = {}): Promise<Recording> {
140
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: { echoCancellation: true, noiseSuppression: true } });
141
+ const recorder = new MediaRecorder(stream);
142
+ const chunks: Blob[] = [];
143
+ recorder.addEventListener('dataavailable', event => chunks.push(event.data));
144
+ recorder.start();
145
+
146
+ const stopDetecting = options.onpause || options.onsilence || options.onlevel ? detectSpeech(stream, options) : undefined;
147
+
148
+ function end(): Promise<void> {
149
+ stopDetecting?.();
150
+ const { promise, resolve } = Promise.withResolvers<void>();
151
+ recorder.addEventListener('stop', () => resolve(), { once: true });
152
+ if (recorder.state != 'inactive') recorder.stop();
153
+ else resolve();
154
+ for (const track of stream.getTracks()) track.stop();
155
+ return promise;
156
+ }
157
+
158
+ return {
159
+ async stop() {
160
+ await end();
161
+ return await toWAV(new Blob(chunks, { type: recorder.mimeType }));
162
+ },
163
+ cancel: () => void end(),
164
+ };
165
+ }
166
+
167
+ export async function transcribe(audio: Blob): Promise<string> {
168
+ const response = await request('ai/transcription', {
169
+ method: 'POST',
170
+ body: audio,
171
+ headers: { 'Content-Type': audio.type, Accept: 'application/json' },
172
+ });
173
+ const { text } = ai.Transcription.parse(await response.json());
174
+ return text;
175
+ }
176
+
177
+ async function synthesize(text: string, signal: AbortSignal): Promise<string> {
178
+ const response = await request('ai/speech', {
179
+ method: 'POST',
180
+ body: JSON.stringify({ text }),
181
+ headers: { 'Content-Type': 'application/json' },
182
+ signal,
183
+ });
184
+ return URL.createObjectURL(await response.blob());
185
+ }
186
+
187
+ function play(url: string, signal: AbortSignal): Promise<void> {
188
+ const { promise, resolve, reject } = Promise.withResolvers<void>();
189
+ const audio = new Audio(url);
190
+
191
+ const stop = () => {
192
+ audio.pause();
193
+ resolve();
194
+ };
195
+ signal.addEventListener('abort', stop, { once: true });
196
+
197
+ audio.addEventListener('ended', () => resolve(), { once: true });
198
+ audio.addEventListener('error', () => reject(new Error('Could not play the audio')), { once: true });
199
+ audio.play().catch(reject);
200
+
201
+ return promise.finally(() => {
202
+ signal.removeEventListener('abort', stop);
203
+ URL.revokeObjectURL(url);
204
+ });
205
+ }
206
+
207
+ /** Speaks Markdown as it arrives, synthesizing each part while the one before it plays */
208
+ class Speaker {
209
+ protected buffer = new SpeechBuffer();
210
+ protected parts: string[] = [];
211
+ protected ended = false;
212
+ protected controller = new AbortController();
213
+ protected wake?: () => void;
214
+ /** Settles once everything is spoken or the speaker is stopped; rejects when speech can't be made or played */
215
+ readonly done: Promise<void>;
216
+
217
+ constructor(public readonly id: string) {
218
+ this.done = this.run();
219
+ }
220
+
221
+ push(text: string): void {
222
+ this.add(this.buffer.push(text));
223
+ }
224
+
225
+ /** Marks the text as complete, so what is left is spoken */
226
+ end(): void {
227
+ this.add(this.buffer.end());
228
+ this.ended = true;
229
+ this.wake?.();
230
+ }
231
+
232
+ stop(): void {
233
+ this.controller.abort();
234
+ this.wake?.();
235
+ }
236
+
237
+ protected add(parts: string[]): void {
238
+ if (!parts.length) return;
239
+ this.parts.push(...parts);
240
+ this.wake?.();
241
+ }
242
+
243
+ /** Starts synthesizing the next part, waiting for one when none is ready */
244
+ protected async next(): Promise<Promise<string> | undefined> {
245
+ const { signal } = this.controller;
246
+ while (!this.parts.length) {
247
+ if (this.ended || signal.aborted) return;
248
+ await new Promise<void>(resolve => (this.wake = resolve));
249
+ }
250
+ if (signal.aborted) return;
251
+
252
+ const audio = synthesize(this.parts.shift()!, signal);
253
+ audio.catch(() => {});
254
+ return audio;
255
+ }
256
+
257
+ protected async run(): Promise<void> {
258
+ const { signal } = this.controller;
259
+ try {
260
+ let upcoming = await this.next();
261
+ while (upcoming && !signal.aborted) {
262
+ const playing = play(await upcoming, signal);
263
+ playing.catch(() => {});
264
+ upcoming = await this.next();
265
+ await playing;
266
+ }
267
+ } catch (error) {
268
+ if (!signal.aborted) throw error;
269
+ }
270
+ }
271
+ }
272
+
273
+ let speaker = $state<Speaker>();
274
+
275
+ /** The ID of what is being spoken, e.g. a message's */
276
+ export function speaking(): string | undefined {
277
+ return speaker?.id;
278
+ }
279
+
280
+ export function stopSpeaking(): void {
281
+ speaker?.stop();
282
+ speaker = undefined;
283
+ }
284
+
285
+ function startSpeaking(id: string): Speaker {
286
+ stopSpeaking();
287
+ const started = new Speaker(id);
288
+ speaker = started;
289
+ void started.done
290
+ .catch(() => {})
291
+ .finally(() => {
292
+ if (speaker == started) speaker = undefined;
293
+ });
294
+ return started;
295
+ }
296
+
297
+ /**
298
+ * Speaks Markdown aloud, replacing anything already being spoken.
299
+ * @throws when the speech can't be made or played
300
+ */
301
+ export async function speak(id: string, markdown: string): Promise<void> {
302
+ const started = startSpeaking(id);
303
+ started.push(markdown);
304
+ started.end();
305
+ await started.done;
306
+ }
307
+
308
+ export type Phase = 'idle' | 'listening' | 'transcribing' | 'responding' | 'speaking';
309
+
310
+ /** Conversation mode: talking with the assistant hands-free */
311
+ export const conversation = $state({
312
+ active: false,
313
+ phase: 'idle' as Phase,
314
+ /** The microphone's level while listening, from 0 to 1 */
315
+ level: 0,
316
+ error: null as string | null,
317
+ });
318
+
319
+ /** Sends what the user said; set by the composer on the page */
320
+ type Sender = (text: string) => Promise<void>;
321
+
322
+ let sender: Sender | undefined;
323
+
324
+ /**
325
+ * Sets how conversation mode sends what the user said.
326
+ * @returns a function that unsets it
327
+ */
328
+ export function useSender(send: Sender): () => void {
329
+ sender = send;
330
+ return () => {
331
+ if (sender == send) sender = undefined;
332
+ };
333
+ }
334
+
335
+ /** IDs of the user messages sent from this page by voice, whose replies are spoken */
336
+ const voiceTurns = new Set<string>();
337
+
338
+ /** Marks a sent user message as spoken, so its reply is read aloud in conversation mode */
339
+ export function addTurn(messageId: string): void {
340
+ voiceTurns.add(messageId);
341
+ }
342
+
343
+ export function isTurn(messageId: string): boolean {
344
+ return voiceTurns.has(messageId);
345
+ }
346
+
347
+ let recording: Recording | undefined;
348
+
349
+ /** Text of the current turn's replies that the speaker already has */
350
+ let followed = { turn: '', length: 0, ended: false };
351
+
352
+ function fail(error: unknown): void {
353
+ conversation.error = error instanceof Error ? error.message : String(error);
354
+ conversation.phase = 'idle';
355
+ }
356
+
357
+ async function finishListening(): Promise<void> {
358
+ if (!recording || conversation.phase != 'listening') return;
359
+ const current = recording;
360
+ recording = undefined;
361
+ conversation.level = 0;
362
+ conversation.phase = 'transcribing';
363
+
364
+ try {
365
+ const text = await transcribe(await current.stop());
366
+ if (!conversation.active) return;
367
+ if (!text) {
368
+ conversation.phase = 'idle';
369
+ return;
370
+ }
371
+ if (!sender) throw new Error('Nothing can send messages on this page');
372
+ conversation.phase = 'responding';
373
+ await sender(text);
374
+ } catch (error) {
375
+ fail(error);
376
+ }
377
+ }
378
+
379
+ /** Starts listening, interrupting anything being spoken */
380
+ export async function listen(): Promise<void> {
381
+ stopSpeaking();
382
+ recording?.cancel();
383
+ conversation.error = null;
384
+ conversation.phase = 'listening';
385
+
386
+ try {
387
+ recording = await record({
388
+ onpause: () => void finishListening(),
389
+ onsilence() {
390
+ recording?.cancel();
391
+ recording = undefined;
392
+ conversation.level = 0;
393
+ conversation.phase = 'idle';
394
+ },
395
+ onlevel: level => (conversation.level = level),
396
+ });
397
+ } catch (error) {
398
+ fail(error);
399
+ }
400
+ }
401
+
402
+ /** What the main button does: start listening, send what was said, or interrupt the reply */
403
+ export function toggle(): void {
404
+ if (conversation.phase == 'listening') void finishListening();
405
+ else if (conversation.phase != 'transcribing') void listen();
406
+ }
407
+
408
+ /** Loads the voice models ahead of use, ignoring failures */
409
+ export function prepare(): void {
410
+ void request('ai/voice/prepare', {
411
+ method: 'POST',
412
+ body: '{}',
413
+ headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
414
+ }).catch(() => {});
415
+ }
416
+
417
+ export function startConversation(): void {
418
+ conversation.active = true;
419
+ prepare();
420
+ void listen();
421
+ }
422
+
423
+ export function endConversation(): void {
424
+ conversation.active = false;
425
+ conversation.phase = 'idle';
426
+ conversation.level = 0;
427
+ recording?.cancel();
428
+ recording = undefined;
429
+ stopSpeaking();
430
+ }
431
+
432
+ /**
433
+ * Whether the messages after a user's message are a finished reply:
434
+ * not between tool steps, but done, paused for approval, stopped or failed.
435
+ */
436
+ export function isSettled(after: ai.Message[]): boolean {
437
+ const last = after.at(-1);
438
+ if (!last || after.some(message => message.status == 'pending')) return false;
439
+ if (after.some(message => message.status == 'awaiting')) return true;
440
+ if (last.status == 'cancelled' || last.status == 'error') return last.role == 'assistant' || last.status == 'cancelled';
441
+ return last.role == 'assistant' && !last.toolCalls?.length;
442
+ }
443
+
444
+ /**
445
+ * Speaks the replies to a voice turn as they stream; call it whenever they change.
446
+ * @param turn The ID of the user message the replies answer
447
+ * @param text The replies' text so far, joined in order
448
+ * @param complete Whether the replies are finished
449
+ */
450
+ export function follow(turn: string, text: string, complete: boolean): void {
451
+ if (!conversation.active || !voiceTurns.has(turn)) return;
452
+
453
+ if (followed.turn != turn) {
454
+ followed = { turn, length: 0, ended: false };
455
+ startSpeaking(turn);
456
+ }
457
+ if (followed.ended || speaker?.id != turn) return;
458
+
459
+ const current = speaker;
460
+ if (text.length > followed.length) {
461
+ conversation.phase = 'speaking';
462
+ current.push(text.slice(followed.length));
463
+ followed.length = text.length;
464
+ }
465
+ if (!complete) return;
466
+
467
+ followed.ended = true;
468
+ current.end();
469
+ void current.done.then(
470
+ () => {
471
+ const waiting = conversation.phase == 'speaking' || conversation.phase == 'responding';
472
+ if (conversation.active && waiting) void listen();
473
+ },
474
+ error => fail(error)
475
+ );
476
+ }