kimaki 0.0.3 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/bundle.js DELETED
@@ -1,3124 +0,0 @@
1
- #!/usr/bin/env node
2
- var __defProp = Object.defineProperty;
3
- var __export = (target, all) => {
4
- for (var name in all)
5
- __defProp(target, name, {
6
- get: all[name],
7
- enumerable: true,
8
- configurable: true,
9
- set: (newValue) => all[name] = () => newValue
10
- });
11
- };
12
- var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
13
-
14
- // src/video-to-ascii.ts
15
- var exports_video_to_ascii = {};
16
- __export(exports_video_to_ascii, {
17
- extractFrames: () => extractFrames,
18
- convertImageToAscii: () => convertImageToAscii
19
- });
20
- import fs2 from "node:fs";
21
- import sharp from "sharp";
22
- import { exec } from "node:child_process";
23
- import { promisify } from "node:util";
24
- function stripAnsi(str) {
25
- return str.replace(/\x1b\[[0-9;]*m/g, "");
26
- }
27
- async function extractFrames({
28
- videoPath,
29
- outputDir,
30
- fps
31
- }) {
32
- if (!fs2.existsSync(outputDir)) {
33
- fs2.mkdirSync(outputDir, { recursive: true });
34
- }
35
- console.log(`Extracting frames at ${fps} fps...`);
36
- const command = `ffmpeg -i "${videoPath}" -vf fps=${fps} "${outputDir}/frame_%04d.png" -y`;
37
- try {
38
- const { stdout, stderr } = await execAsync(command);
39
- console.log("Frames extracted successfully");
40
- } catch (error) {
41
- console.error("Error extracting frames:", error.message);
42
- throw error;
43
- }
44
- }
45
- async function convertImageToAscii({
46
- imagePath,
47
- cols,
48
- rows,
49
- keepAspectRatio = true,
50
- colored = false
51
- }) {
52
- try {
53
- const metadata = await sharp(imagePath).metadata();
54
- const originalWidth = metadata.width || 100;
55
- const originalHeight = metadata.height || 100;
56
- let finalCols = cols;
57
- let finalRows = rows;
58
- let resizeOptions = {};
59
- if (keepAspectRatio) {
60
- const aspectRatio = originalWidth / originalHeight;
61
- const targetAspectRatio = cols * 2 / rows;
62
- if (aspectRatio > targetAspectRatio) {
63
- finalRows = Math.floor(cols / aspectRatio * 0.5);
64
- } else {
65
- finalCols = Math.floor(rows * aspectRatio * 2);
66
- }
67
- resizeOptions = { fit: "fill" };
68
- } else {
69
- const aspectRatio = originalWidth / originalHeight;
70
- const targetAspectRatio = cols * 2 / rows;
71
- if (aspectRatio > targetAspectRatio) {
72
- const zoomHeight = Math.ceil(cols / aspectRatio * 0.5);
73
- resizeOptions = {
74
- width: cols,
75
- height: zoomHeight,
76
- fit: "cover",
77
- position: "center"
78
- };
79
- } else {
80
- const zoomWidth = Math.ceil(rows * aspectRatio * 2);
81
- resizeOptions = {
82
- width: zoomWidth,
83
- height: rows,
84
- fit: "cover",
85
- position: "center"
86
- };
87
- }
88
- finalCols = cols;
89
- finalRows = rows;
90
- }
91
- if (colored) {
92
- const { data, info } = await sharp(imagePath).resize(finalCols, finalRows, resizeOptions).raw().toBuffer({ resolveWithObject: true });
93
- const asciiArt = [];
94
- const channels = info.channels;
95
- const actualCols = info.width;
96
- const actualRows = info.height;
97
- for (let y = 0;y < actualRows; y++) {
98
- let row = "";
99
- for (let x = 0;x < actualCols; x++) {
100
- const pixelIndex = (y * actualCols + x) * channels;
101
- const r = data[pixelIndex] || 0;
102
- const g = data[pixelIndex + 1] || 0;
103
- const b = data[pixelIndex + 2] || 0;
104
- const brightness = r * 0.299 + g * 0.587 + b * 0.114;
105
- const charIndex = Math.floor(brightness / 255 * (ASCII_CHARS.length - 1));
106
- const char = ASCII_CHARS[charIndex] || " ";
107
- const brightnessFactor = brightness / 255;
108
- const adjustedR = Math.floor(r * brightnessFactor * 0.8);
109
- const adjustedG = Math.floor(g * brightnessFactor * 0.8);
110
- const adjustedB = Math.floor(b * brightnessFactor * 0.8);
111
- row += `\x1B[38;2;${adjustedR};${adjustedG};${adjustedB}m${char}\x1B[0m`;
112
- }
113
- if (!keepAspectRatio) {
114
- const visibleLength = stripAnsi(row).length;
115
- if (visibleLength < cols) {
116
- row += " ".repeat(cols - visibleLength);
117
- }
118
- }
119
- asciiArt.push(row);
120
- }
121
- if (!keepAspectRatio) {
122
- while (asciiArt.length < rows) {
123
- asciiArt.push(" ".repeat(cols));
124
- }
125
- }
126
- return asciiArt.join(`
127
- `);
128
- } else {
129
- const { data: resized, info } = await sharp(imagePath).resize(finalCols, finalRows, resizeOptions).greyscale().raw().toBuffer({ resolveWithObject: true });
130
- const actualCols = info.width;
131
- const actualRows = info.height;
132
- const asciiArt = [];
133
- for (let y = 0;y < actualRows; y++) {
134
- let row = "";
135
- for (let x = 0;x < actualCols; x++) {
136
- const pixelIndex = y * actualCols + x;
137
- const brightness = resized[pixelIndex] || 0;
138
- const charIndex = Math.floor(brightness / 255 * (ASCII_CHARS.length - 1));
139
- row += ASCII_CHARS[charIndex] || " ";
140
- }
141
- if (!keepAspectRatio) {
142
- const visibleLength = stripAnsi(row).length;
143
- if (visibleLength < cols) {
144
- row += " ".repeat(cols - visibleLength);
145
- }
146
- }
147
- asciiArt.push(row);
148
- }
149
- if (!keepAspectRatio) {
150
- while (asciiArt.length < rows) {
151
- asciiArt.push(" ".repeat(cols));
152
- }
153
- }
154
- return asciiArt.join(`
155
- `);
156
- }
157
- } catch (error) {
158
- console.error(`Error converting ${imagePath}:`, error.message);
159
- throw error;
160
- }
161
- }
162
- var execAsync, ASCII_CHARS = " .'`^\",:;Il!i><~+_-?][}{1)(|/tfjrxnuvczXYUJCLQ0OZmwqpdbkhao*#MW&8%B@$";
163
- var init_video_to_ascii = __esm(() => {
164
- execAsync = promisify(exec);
165
- });
166
-
167
- // ../liveapi/src/utils.ts
168
- async function closeAllAudioContexts() {
169
- for (const [id, ctx] of map) {
170
- if (ctx.state !== "closed") {
171
- await ctx.close();
172
- }
173
- }
174
- map.clear();
175
- }
176
- function base64ToArrayBuffer(base64) {
177
- var binaryString = atob(base64);
178
- var bytes = new Uint8Array(binaryString.length);
179
- for (let i = 0;i < binaryString.length; i++) {
180
- bytes[i] = binaryString.charCodeAt(i);
181
- }
182
- return bytes.buffer;
183
- }
184
- var map, audioContext;
185
- var init_utils = __esm(() => {
186
- map = new Map;
187
- audioContext = (() => {
188
- const didInteract = new Promise((res) => {
189
- if (typeof window === "undefined") {
190
- res(true);
191
- }
192
- addEventListener("pointerdown", res, { once: true });
193
- addEventListener("keydown", res, { once: true });
194
- });
195
- return async (options) => {
196
- try {
197
- const a = new Audio;
198
- a.src = "data:audio/wav;base64,UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA";
199
- await a.play();
200
- if (options?.id && map.has(options.id)) {
201
- const ctx2 = map.get(options.id);
202
- if (ctx2) {
203
- return ctx2;
204
- }
205
- }
206
- const ctx = new globalThis.AudioContext(options);
207
- if (options?.id) {
208
- map.set(options.id, ctx);
209
- }
210
- return ctx;
211
- } catch (e) {
212
- await didInteract;
213
- if (options?.id && map.has(options.id)) {
214
- const ctx2 = map.get(options.id);
215
- if (ctx2) {
216
- return ctx2;
217
- }
218
- }
219
- const ctx = new globalThis.AudioContext(options);
220
- if (options?.id) {
221
- map.set(options.id, ctx);
222
- }
223
- return ctx;
224
- }
225
- };
226
- })();
227
- });
228
-
229
- // ../liveapi/src/worklets/audio-processing.ts
230
- var AudioRecordingWorklet = `
231
- class AudioProcessingWorklet extends AudioWorkletProcessor {
232
-
233
- // send and clear buffer every 2048 samples,
234
- // which at 16khz is about 8 times a second
235
- buffer = new Int16Array(2048);
236
-
237
- // current write index
238
- bufferWriteIndex = 0;
239
-
240
- constructor() {
241
- super();
242
- this.hasAudio = false;
243
- }
244
-
245
- /**
246
- * @param inputs Float32Array[][] [input#][channel#][sample#] so to access first inputs 1st channel inputs[0][0]
247
- * @param outputs Float32Array[][]
248
- */
249
- process(inputs) {
250
- if (inputs[0].length) {
251
- const channel0 = inputs[0][0];
252
- this.processChunk(channel0);
253
- }
254
- return true;
255
- }
256
-
257
- sendAndClearBuffer(){
258
- this.port.postMessage({
259
- event: "chunk",
260
- data: {
261
- int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer,
262
- },
263
- });
264
- this.bufferWriteIndex = 0;
265
- }
266
-
267
- processChunk(float32Array) {
268
- const l = float32Array.length;
269
-
270
- for (let i = 0; i < l; i++) {
271
- // convert float32 -1 to 1 to int16 -32768 to 32767
272
- const int16Value = float32Array[i] * 32768;
273
- this.buffer[this.bufferWriteIndex++] = int16Value;
274
- if(this.bufferWriteIndex >= this.buffer.length) {
275
- this.sendAndClearBuffer();
276
- }
277
- }
278
-
279
- if(this.bufferWriteIndex >= this.buffer.length) {
280
- this.sendAndClearBuffer();
281
- }
282
- }
283
- }
284
- `, audio_processing_default;
285
- var init_audio_processing = __esm(() => {
286
- audio_processing_default = AudioRecordingWorklet;
287
- });
288
-
289
- // ../liveapi/src/worklets/vol-meter.ts
290
- var VolMeterWorket = `
291
- class VolMeter extends AudioWorkletProcessor {
292
- volume
293
- updateIntervalInMS
294
- nextUpdateFrame
295
-
296
- constructor() {
297
- super()
298
- this.volume = 0
299
- this.updateIntervalInMS = 25
300
- this.nextUpdateFrame = this.updateIntervalInMS
301
- this.port.onmessage = event => {
302
- if (event.data.updateIntervalInMS) {
303
- this.updateIntervalInMS = event.data.updateIntervalInMS
304
- }
305
- }
306
- }
307
-
308
- get intervalInFrames() {
309
- return (this.updateIntervalInMS / 1000) * sampleRate
310
- }
311
-
312
- process(inputs) {
313
- const input = inputs[0]
314
-
315
- if (input.length > 0) {
316
- const samples = input[0]
317
- let sum = 0
318
- let rms = 0
319
-
320
- for (let i = 0; i < samples.length; ++i) {
321
- sum += samples[i] * samples[i]
322
- }
323
-
324
- rms = Math.sqrt(sum / samples.length)
325
- this.volume = Math.max(rms, this.volume * 0.7)
326
-
327
- this.nextUpdateFrame -= samples.length
328
- if (this.nextUpdateFrame < 0) {
329
- this.nextUpdateFrame += this.intervalInFrames
330
- this.port.postMessage({volume: this.volume})
331
- }
332
- }
333
-
334
- return true
335
- }
336
- }`, vol_meter_default;
337
- var init_vol_meter = __esm(() => {
338
- vol_meter_default = VolMeterWorket;
339
- });
340
-
341
- // ../liveapi/src/audio-resampler.ts
342
- function downSampleAudioBuffer(buffer, inputSampleRate, outputSampleRate) {
343
- if (outputSampleRate >= inputSampleRate) {
344
- throw new Error("Output sample rate must be less than input sample rate.");
345
- }
346
- const sampleRateRatio = inputSampleRate / outputSampleRate;
347
- const newLength = Math.round(buffer.length / sampleRateRatio);
348
- const result = new Float32Array(newLength);
349
- let offsetResult = 0;
350
- let offsetBuffer = 0;
351
- while (offsetResult < result.length) {
352
- const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
353
- let accum = 0;
354
- let count = 0;
355
- for (let i = offsetBuffer;i < nextOffsetBuffer && i < buffer.length; i++) {
356
- accum += buffer[i];
357
- count++;
358
- }
359
- result[offsetResult] = accum / count;
360
- offsetResult++;
361
- offsetBuffer = nextOffsetBuffer;
362
- }
363
- return result;
364
- }
365
- function downSampleInt16Buffer(buffer, inputSampleRate, outputSampleRate) {
366
- if (outputSampleRate >= inputSampleRate) {
367
- throw new Error("Output sample rate must be less than input sample rate.");
368
- }
369
- const int16Array = new Int16Array(buffer);
370
- const float32Array = new Float32Array(int16Array.length);
371
- for (let i = 0;i < int16Array.length; i++) {
372
- float32Array[i] = int16Array[i] / 32768;
373
- }
374
- const downsampled = downSampleAudioBuffer(float32Array, inputSampleRate, outputSampleRate);
375
- const downsampledInt16 = new Int16Array(downsampled.length);
376
- for (let i = 0;i < downsampled.length; i++) {
377
- downsampledInt16[i] = Math.max(-32768, Math.min(32767, Math.floor(downsampled[i] * 32768)));
378
- }
379
- return downsampledInt16.buffer;
380
- }
381
-
382
- // ../liveapi/src/audioworklet-registry.ts
383
- var registeredWorklets, createWorketFromSrc = (workletName, workletSrc) => {
384
- const script = new Blob([`registerProcessor("${workletName}", ${workletSrc})`], {
385
- type: "application/javascript"
386
- });
387
- return URL.createObjectURL(script);
388
- };
389
- var init_audioworklet_registry = __esm(() => {
390
- registeredWorklets = new Map;
391
- });
392
-
393
- // ../liveapi/src/audio-recorder.ts
394
- import EventEmitter from "eventemitter3";
395
- var AudioRecorder;
396
- var init_audio_recorder = __esm(() => {
397
- init_utils();
398
- init_audio_processing();
399
- init_vol_meter();
400
- init_audioworklet_registry();
401
- AudioRecorder = class AudioRecorder extends EventEmitter.EventEmitter {
402
- sampleRate;
403
- stream;
404
- audioContext;
405
- source;
406
- recording = false;
407
- recordingWorklet;
408
- vuWorklet;
409
- starting = null;
410
- constructor(sampleRate = 16000) {
411
- super();
412
- this.sampleRate = sampleRate;
413
- }
414
- async start() {
415
- if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
416
- throw new Error("Could not request user media");
417
- }
418
- this.starting = new Promise(async (resolve, reject) => {
419
- this.stream = await navigator.mediaDevices.getUserMedia({ audio: true });
420
- this.audioContext = await audioContext({ sampleRate: this.sampleRate });
421
- this.source = this.audioContext.createMediaStreamSource(this.stream);
422
- const workletName = "audio-recorder-worklet";
423
- const src = createWorketFromSrc(workletName, audio_processing_default);
424
- await this.audioContext.audioWorklet.addModule(src);
425
- this.recordingWorklet = new AudioWorkletNode(this.audioContext, workletName);
426
- this.recordingWorklet.port.onmessage = async (ev) => {
427
- const arrayBuffer = ev.data.data.int16arrayBuffer;
428
- if (arrayBuffer) {
429
- let outputBuffer = arrayBuffer;
430
- if (this.sampleRate > 16000) {
431
- outputBuffer = downSampleInt16Buffer(arrayBuffer, this.sampleRate, 16000);
432
- }
433
- this.emit("data", outputBuffer);
434
- }
435
- };
436
- this.source.connect(this.recordingWorklet);
437
- const vuWorkletName = "vu-meter";
438
- await this.audioContext.audioWorklet.addModule(createWorketFromSrc(vuWorkletName, vol_meter_default));
439
- this.vuWorklet = new AudioWorkletNode(this.audioContext, vuWorkletName);
440
- this.vuWorklet.port.onmessage = (ev) => {
441
- this.emit("volume", ev.data.volume);
442
- };
443
- this.source.connect(this.vuWorklet);
444
- this.recording = true;
445
- resolve();
446
- this.starting = null;
447
- });
448
- }
449
- stop() {
450
- const handleStop = () => {
451
- this.source?.disconnect();
452
- this.recordingWorklet?.disconnect();
453
- this.vuWorklet?.disconnect();
454
- this.stream?.getTracks?.()?.forEach((track) => track.stop());
455
- this.stream = undefined;
456
- this.recordingWorklet = undefined;
457
- this.vuWorklet = undefined;
458
- this.recording = false;
459
- };
460
- if (this.starting) {
461
- this.starting.then(handleStop);
462
- return;
463
- }
464
- handleStop();
465
- }
466
- async destroy() {
467
- this.stop();
468
- if (this.audioContext?.state !== "closed") {
469
- await this.audioContext?.close();
470
- }
471
- this.audioContext = undefined;
472
- this.removeAllListeners();
473
- }
474
- };
475
- });
476
-
477
- // ../liveapi/src/audio-streamer.ts
478
- class AudioStreamer {
479
- context;
480
- sampleRate = 24000;
481
- bufferSize = 7680;
482
- audioQueue = [];
483
- isPlaying = false;
484
- isStreamComplete = false;
485
- checkInterval = null;
486
- scheduledTime = 0;
487
- initialBufferTime = 0.1;
488
- gainNode;
489
- source;
490
- endOfQueueAudioSource = null;
491
- onComplete = () => {};
492
- constructor(context) {
493
- this.context = context;
494
- this.gainNode = this.context.createGain();
495
- this.source = this.context.createBufferSource();
496
- this.gainNode.connect(this.context.destination);
497
- this.addPCM16 = this.addPCM16.bind(this);
498
- }
499
- async addWorklet(workletName, workletSrc, handler) {
500
- let workletsRecord = registeredWorklets.get(this.context);
501
- if (workletsRecord && workletsRecord[workletName]) {
502
- workletsRecord[workletName].handlers.push(handler);
503
- return Promise.resolve(this);
504
- }
505
- if (!workletsRecord) {
506
- registeredWorklets.set(this.context, {});
507
- workletsRecord = registeredWorklets.get(this.context);
508
- }
509
- workletsRecord[workletName] = { handlers: [handler] };
510
- const src = createWorketFromSrc(workletName, workletSrc);
511
- await this.context.audioWorklet.addModule(src);
512
- const worklet = new AudioWorkletNode(this.context, workletName);
513
- workletsRecord[workletName].node = worklet;
514
- return this;
515
- }
516
- _processPCM16Chunk(chunk) {
517
- const float32Array = new Float32Array(chunk.length / 2);
518
- const dataView = new DataView(chunk.buffer);
519
- for (let i = 0;i < chunk.length / 2; i++) {
520
- try {
521
- const int16 = dataView.getInt16(i * 2, true);
522
- float32Array[i] = int16 / 32768;
523
- } catch (e) {
524
- console.error(e);
525
- }
526
- }
527
- return float32Array;
528
- }
529
- addPCM16(chunk) {
530
- this.isStreamComplete = false;
531
- let processingBuffer = this._processPCM16Chunk(chunk);
532
- while (processingBuffer.length >= this.bufferSize) {
533
- const buffer = processingBuffer.slice(0, this.bufferSize);
534
- this.audioQueue.push(buffer);
535
- processingBuffer = processingBuffer.slice(this.bufferSize);
536
- }
537
- if (processingBuffer.length > 0) {
538
- this.audioQueue.push(processingBuffer);
539
- }
540
- if (!this.isPlaying) {
541
- this.isPlaying = true;
542
- this.scheduledTime = this.context.currentTime + this.initialBufferTime;
543
- this.scheduleNextBuffer();
544
- }
545
- }
546
- createAudioBuffer(audioData) {
547
- const audioBuffer = this.context.createBuffer(1, audioData.length, this.sampleRate);
548
- audioBuffer.getChannelData(0).set(audioData);
549
- return audioBuffer;
550
- }
551
- scheduleNextBuffer() {
552
- const SCHEDULE_AHEAD_TIME = 0.2;
553
- while (this.audioQueue.length > 0 && this.scheduledTime < this.context.currentTime + SCHEDULE_AHEAD_TIME) {
554
- const audioData = this.audioQueue.shift();
555
- const audioBuffer = this.createAudioBuffer(audioData);
556
- const source = this.context.createBufferSource();
557
- if (this.audioQueue.length === 0) {
558
- if (this.endOfQueueAudioSource) {
559
- this.endOfQueueAudioSource.onended = null;
560
- }
561
- this.endOfQueueAudioSource = source;
562
- source.onended = () => {
563
- if (!this.audioQueue.length && this.endOfQueueAudioSource === source) {
564
- this.endOfQueueAudioSource = null;
565
- this.onComplete();
566
- }
567
- };
568
- }
569
- source.buffer = audioBuffer;
570
- source.connect(this.gainNode);
571
- const worklets = registeredWorklets.get(this.context);
572
- if (worklets) {
573
- Object.entries(worklets).forEach(([workletName, graph]) => {
574
- const { node, handlers } = graph;
575
- if (node) {
576
- source.connect(node);
577
- node.port.onmessage = function(ev) {
578
- handlers.forEach((handler) => {
579
- handler.call(node.port, ev);
580
- });
581
- };
582
- node.connect(this.context.destination);
583
- }
584
- });
585
- }
586
- const startTime = Math.max(this.scheduledTime, this.context.currentTime);
587
- source.start(startTime);
588
- this.scheduledTime = startTime + audioBuffer.duration;
589
- }
590
- if (this.audioQueue.length === 0) {
591
- if (this.isStreamComplete) {
592
- this.isPlaying = false;
593
- if (this.checkInterval) {
594
- clearInterval(this.checkInterval);
595
- this.checkInterval = null;
596
- }
597
- } else {
598
- if (!this.checkInterval) {
599
- this.checkInterval = setInterval(() => {
600
- if (this.audioQueue.length > 0) {
601
- this.scheduleNextBuffer();
602
- }
603
- }, 100);
604
- }
605
- }
606
- } else {
607
- const nextCheckTime = (this.scheduledTime - this.context.currentTime) * 1000;
608
- setTimeout(() => this.scheduleNextBuffer(), Math.max(0, nextCheckTime - 50));
609
- }
610
- }
611
- stop() {
612
- this.isPlaying = false;
613
- this.isStreamComplete = true;
614
- this.audioQueue = [];
615
- this.scheduledTime = this.context.currentTime;
616
- if (this.checkInterval) {
617
- clearInterval(this.checkInterval);
618
- this.checkInterval = null;
619
- }
620
- this.gainNode.gain.linearRampToValueAtTime(0, this.context.currentTime + 0.1);
621
- setTimeout(() => {
622
- this.gainNode.disconnect();
623
- this.gainNode = this.context.createGain();
624
- this.gainNode.connect(this.context.destination);
625
- }, 200);
626
- }
627
- async resume() {
628
- if (this.context.state === "suspended") {
629
- await this.context.resume();
630
- }
631
- this.isStreamComplete = false;
632
- this.scheduledTime = this.context.currentTime + this.initialBufferTime;
633
- this.gainNode.gain.setValueAtTime(1, this.context.currentTime);
634
- }
635
- complete() {
636
- this.isStreamComplete = true;
637
- this.onComplete();
638
- }
639
- async destroy() {
640
- this.stop();
641
- const worklets = registeredWorklets.get(this.context);
642
- if (worklets) {
643
- Object.values(worklets).forEach(({ node }) => {
644
- if (node) {
645
- node.disconnect();
646
- node.port.close();
647
- }
648
- });
649
- registeredWorklets.delete(this.context);
650
- }
651
- this.gainNode.disconnect();
652
- }
653
- }
654
- var init_audio_streamer = __esm(() => {
655
- init_audioworklet_registry();
656
- });
657
-
658
- // ../liveapi/src/cleanup.ts
659
- function registerCleanupHandler(handler) {
660
- cleanupHandlers.push(handler);
661
- }
662
- function unregisterCleanupHandler(handler) {
663
- const index = cleanupHandlers.indexOf(handler);
664
- if (index > -1) {
665
- cleanupHandlers.splice(index, 1);
666
- }
667
- }
668
- async function cleanup() {
669
- console.log("Cleaning up audio resources...");
670
- await Promise.all(cleanupHandlers.map((handler) => Promise.resolve(handler()).catch((err) => console.error("Cleanup handler error:", err))));
671
- await closeAllAudioContexts();
672
- console.log("Audio cleanup complete");
673
- }
674
- var cleanupHandlers;
675
- var init_cleanup = __esm(() => {
676
- init_utils();
677
- cleanupHandlers = [];
678
- if (typeof process !== "undefined") {
679
- process.on("exit", () => {
680
- cleanup().catch(console.error);
681
- });
682
- process.on("SIGINT", async () => {
683
- await cleanup();
684
- process.exit(0);
685
- });
686
- process.on("SIGTERM", async () => {
687
- await cleanup();
688
- process.exit(0);
689
- });
690
- process.on("uncaughtException", async (error) => {
691
- console.error("Uncaught Exception:", error);
692
- await cleanup();
693
- process.exit(1);
694
- });
695
- process.on("unhandledRejection", async (reason, promise) => {
696
- console.error("Unhandled Rejection at:", promise, "reason:", reason);
697
- await cleanup();
698
- process.exit(1);
699
- });
700
- } else if (typeof window !== "undefined") {
701
- window.addEventListener("beforeunload", () => {
702
- cleanup().catch(console.error);
703
- });
704
- window.addEventListener("unload", () => {
705
- cleanup().catch(console.error);
706
- });
707
- }
708
- });
709
-
710
- // ../liveapi/src/live-api-client.ts
711
- import {
712
- GoogleGenAI,
713
- MediaResolution
714
- } from "@google/genai";
715
-
716
- class LiveAPIClient {
717
- client;
718
- session = null;
719
- audioStreamer = null;
720
- audioRecorder = null;
721
- model;
722
- onStateChange;
723
- onMessage;
724
- state = {
725
- connected: false,
726
- muted: false,
727
- inVolume: 0,
728
- outVolume: 0,
729
- logs: [],
730
- config: {
731
- inputAudioTranscription: {},
732
- outputAudioTranscription: {},
733
- mediaResolution: MediaResolution.MEDIA_RESOLUTION_MEDIUM,
734
- contextWindowCompression: {
735
- triggerTokens: "25600",
736
- slidingWindow: { targetTokens: "12800" }
737
- }
738
- },
739
- isAssistantSpeaking: false
740
- };
741
- onUserAudioChunk;
742
- tools = [];
743
- sessionHandle = null;
744
- isExplicitDisconnect = false;
745
- reconnectAttempts = 0;
746
- maxReconnectAttempts;
747
- reconnectTimeout = null;
748
- autoReconnect;
749
- autoMuteOnAssistantSpeaking;
750
- userMutedState = false;
751
- constructor(options) {
752
- const {
753
- model,
754
- onStateChange,
755
- onMessage,
756
- onUserAudioChunk,
757
- apiKey,
758
- enableGoogleSearch,
759
- recordingSampleRate = 16000,
760
- config,
761
- autoReconnect = true,
762
- maxReconnectAttempts = 5,
763
- autoMuteOnAssistantSpeaking = true
764
- } = options;
765
- if (!apiKey) {
766
- throw new Error("API key is required");
767
- }
768
- this.client = new GoogleGenAI({ apiKey });
769
- this.model = model || "models/gemini-2.0-flash-exp";
770
- this.onStateChange = onStateChange;
771
- this.onMessage = onMessage;
772
- this.onUserAudioChunk = onUserAudioChunk;
773
- this.autoReconnect = autoReconnect;
774
- this.maxReconnectAttempts = maxReconnectAttempts;
775
- this.autoMuteOnAssistantSpeaking = autoMuteOnAssistantSpeaking;
776
- this.tools = config?.tools || [];
777
- if (enableGoogleSearch) {
778
- this.tools.push({ googleSearch: {} });
779
- }
780
- if (config) {
781
- this.state.config = { ...this.state.config, ...config };
782
- }
783
- this.audioRecorder = new AudioRecorder(recordingSampleRate);
784
- this.setupAudioRecorder();
785
- const cleanupHandler = () => this.destroy();
786
- registerCleanupHandler(cleanupHandler);
787
- this._cleanupHandler = cleanupHandler;
788
- }
789
- updateState(updates) {
790
- this.state = { ...this.state, ...updates };
791
- this.onStateChange?.(this.state);
792
- }
793
- getState() {
794
- return { ...this.state };
795
- }
796
- async initAudioStreamer() {
797
- if (!this.audioStreamer) {
798
- const audioCtx = await audioContext({ id: "audio-out" });
799
- this.audioStreamer = new AudioStreamer(audioCtx);
800
- await this.audioStreamer.addWorklet("vumeter-out", vol_meter_default, (ev) => {
801
- this.updateState({ outVolume: ev.data.volume });
802
- });
803
- this.audioStreamer.onComplete = () => {
804
- this.onAssistantStopSpeaking();
805
- };
806
- }
807
- }
808
- setupAudioRecorder() {
809
- if (!this.audioRecorder)
810
- return;
811
- this.audioRecorder.on("data", (arrayBuffer) => {
812
- if (this.state.connected && !this.state.muted) {
813
- const binary = String.fromCharCode(...new Uint8Array(arrayBuffer));
814
- const base64 = btoa(binary);
815
- this.sendRealtimeInput([
816
- {
817
- mimeType: "audio/pcm;rate=16000",
818
- data: base64
819
- }
820
- ]);
821
- if (this.onUserAudioChunk) {
822
- this.onUserAudioChunk(arrayBuffer.slice(0));
823
- }
824
- }
825
- });
826
- this.audioRecorder.on("volume", (volume) => {
827
- this.updateState({ inVolume: volume });
828
- });
829
- }
830
- log(message) {
831
- console.log(message);
832
- const newEvents = [...this.state.logs, message];
833
- if (newEvents.length > 200) {
834
- this.updateState({ logs: newEvents.slice(-150) });
835
- } else {
836
- this.updateState({ logs: newEvents });
837
- }
838
- }
839
- async connect() {
840
- if (this.state.connected) {
841
- return false;
842
- }
843
- this.isExplicitDisconnect = false;
844
- this.reconnectAttempts = 0;
845
- if (this.reconnectTimeout) {
846
- clearTimeout(this.reconnectTimeout);
847
- this.reconnectTimeout = null;
848
- }
849
- await this.initAudioStreamer();
850
- const callbacks = {
851
- onopen: this.onOpen.bind(this),
852
- onmessage: this.handleMessage.bind(this),
853
- onerror: this.onError.bind(this),
854
- onclose: this.onClose.bind(this)
855
- };
856
- const connectConfig = { ...this.state.config };
857
- if (this.sessionHandle && connectConfig.sessionResumption) {
858
- connectConfig.sessionResumption = {
859
- ...connectConfig.sessionResumption,
860
- handle: this.sessionHandle
861
- };
862
- this.log("connect: Attempting to resume session with handle");
863
- }
864
- try {
865
- this.session = await this.client.live.connect({
866
- model: this.model,
867
- config: connectConfig,
868
- callbacks
869
- });
870
- } catch (e) {
871
- console.error("Error connecting to GenAI Live:", e);
872
- this.log("error: " + JSON.stringify({ message: "Failed to connect", error: e }));
873
- return false;
874
- }
875
- return true;
876
- }
877
- disconnect() {
878
- if (!this.session) {
879
- return false;
880
- }
881
- this.isExplicitDisconnect = true;
882
- if (this.reconnectTimeout) {
883
- clearTimeout(this.reconnectTimeout);
884
- this.reconnectTimeout = null;
885
- }
886
- this.session?.close();
887
- this.session = null;
888
- this.setConnected(false);
889
- this.audioRecorder?.stop();
890
- this.audioStreamer?.stop();
891
- this.onAssistantStopSpeaking();
892
- this.sessionHandle = null;
893
- this.reconnectAttempts = 0;
894
- this.log("close: " + JSON.stringify({ reason: "User disconnected" }));
895
- return true;
896
- }
897
- setConnected(value) {
898
- this.updateState({ connected: value });
899
- if (value && !this.state.muted) {
900
- this.audioRecorder?.start();
901
- } else {
902
- this.audioRecorder?.stop();
903
- }
904
- }
905
- onOpen() {
906
- this.setConnected(true);
907
- this.log("open");
908
- }
909
- onError(e) {
910
- this.log("error: " + JSON.stringify({ message: e.message, error: e }));
911
- }
912
- async onClose(e) {
913
- this.setConnected(false);
914
- this.log("close: " + JSON.stringify({ reason: e.reason, code: e.code }));
915
- if (this.autoReconnect && !this.isExplicitDisconnect && this.sessionHandle) {
916
- await this.attemptReconnect();
917
- }
918
- }
919
- async attemptReconnect() {
920
- console.log(`attempting to reconnect`);
921
- if (this.reconnectAttempts >= this.maxReconnectAttempts) {
922
- this.log("error: Max reconnection attempts reached, giving up");
923
- this.sessionHandle = null;
924
- this.reconnectAttempts = 0;
925
- return;
926
- }
927
- this.reconnectAttempts++;
928
- const delay = Math.min(1000 * Math.pow(2, this.reconnectAttempts - 1), 1e4);
929
- this.log(`reconnect: Attempting reconnection ${this.reconnectAttempts}/${this.maxReconnectAttempts} in ${delay}ms`);
930
- this.reconnectTimeout = setTimeout(async () => {
931
- this.reconnectTimeout = null;
932
- try {
933
- const success = await this.connect();
934
- if (success) {
935
- this.log("reconnect: Successfully reconnected to session");
936
- this.reconnectAttempts = 0;
937
- } else {
938
- this.log("reconnect: Failed to reconnect, will retry");
939
- }
940
- } catch (error) {
941
- this.log("reconnect: Error during reconnection: " + JSON.stringify(error));
942
- }
943
- }, delay);
944
- }
945
- async handleMessage(message) {
946
- const clonedMessage = JSON.parse(JSON.stringify(message));
947
- this.onMessage?.(clonedMessage);
948
- if (message.setupComplete) {
949
- this.log("setupcomplete");
950
- return;
951
- }
952
- if (message.sessionResumptionUpdate) {
953
- const update = message.sessionResumptionUpdate;
954
- if (update.resumable && update.newHandle) {
955
- this.sessionHandle = update.newHandle;
956
- this.log("sessionResumption: Updated handle for resumption " + update.newHandle);
957
- } else if (!update.resumable) {
958
- this.sessionHandle = null;
959
- this.log("sessionResumption: Session not resumable at this point");
960
- }
961
- return;
962
- }
963
- if (message.goAway) {
964
- this.log("goAway: Server will disconnect, preparing for reconnection");
965
- return;
966
- }
967
- if (message.toolCall) {
968
- this.log("toolcall: " + JSON.stringify(message.toolCall));
969
- if (message.toolCall.functionCalls) {
970
- await this.handleToolCalls(message.toolCall.functionCalls);
971
- }
972
- return;
973
- }
974
- if (message.toolCallCancellation) {
975
- this.log("toolcallcancellation: " + JSON.stringify(message.toolCallCancellation));
976
- return;
977
- }
978
- if (message.serverContent) {
979
- const { serverContent } = message;
980
- if (serverContent.interrupted) {
981
- this.log("interrupted");
982
- this.audioStreamer?.stop();
983
- this.onAssistantStopSpeaking();
984
- return;
985
- }
986
- if (serverContent.turnComplete) {
987
- this.log("turncomplete");
988
- }
989
- if (serverContent.modelTurn) {
990
- let parts = serverContent.modelTurn?.parts || [];
991
- const [audioParts, otherParts] = partition(parts, (p) => p.inlineData && p.inlineData.mimeType?.startsWith("audio/pcm"));
992
- const base64s = audioParts.map((p) => p.inlineData?.data);
993
- base64s.forEach((b64, index) => {
994
- if (b64) {
995
- if (index === 0 && !this.state.isAssistantSpeaking) {
996
- this.onAssistantStartSpeaking();
997
- }
998
- const data = base64ToArrayBuffer(b64);
999
- this.audioStreamer?.addPCM16(new Uint8Array(data));
1000
- }
1001
- });
1002
- if (otherParts.length) {
1003
- this.log("content: " + JSON.stringify({ modelTurn: { parts: otherParts } }));
1004
- otherParts.forEach((part) => {
1005
- if (part.executableCode && part.executableCode.language === "PYTHON" && part.executableCode.code) {
1006
- const preview = part.executableCode.code;
1007
- this.log(`python executableCode:
1008
- ` + preview);
1009
- }
1010
- });
1011
- }
1012
- }
1013
- }
1014
- }
1015
- setMuted(muted) {
1016
- this.userMutedState = muted;
1017
- this.updateState({ muted });
1018
- if (this.state.connected) {
1019
- if (muted) {
1020
- this.audioRecorder?.stop();
1021
- } else {
1022
- this.audioRecorder?.start();
1023
- }
1024
- }
1025
- }
1026
- onAssistantStartSpeaking() {
1027
- this.updateState({ isAssistantSpeaking: true });
1028
- if (this.autoMuteOnAssistantSpeaking && !this.userMutedState && !this.state.muted) {
1029
- this.updateState({ muted: true });
1030
- if (this.state.connected) {
1031
- this.audioRecorder?.stop();
1032
- }
1033
- }
1034
- }
1035
- onAssistantStopSpeaking() {
1036
- this.updateState({ isAssistantSpeaking: false });
1037
- if (this.autoMuteOnAssistantSpeaking && !this.userMutedState && this.state.muted) {
1038
- this.updateState({ muted: false });
1039
- if (this.state.connected) {
1040
- this.audioRecorder?.start();
1041
- }
1042
- }
1043
- }
1044
- sendText(text, turnComplete = true) {
1045
- if (!this.session)
1046
- return;
1047
- const parts = [{ text }];
1048
- this.session.sendClientContent({ turns: parts, turnComplete });
1049
- this.log("client-send: " + JSON.stringify({ turns: parts, turnComplete }));
1050
- }
1051
- sendRealtimeInput(chunks) {
1052
- if (!this.session)
1053
- return;
1054
- let hasAudio = false;
1055
- let hasVideo = false;
1056
- for (const ch of chunks) {
1057
- this.session.sendRealtimeInput({ media: ch });
1058
- if (ch.mimeType.includes("audio")) {
1059
- hasAudio = true;
1060
- }
1061
- if (ch.mimeType.includes("image")) {
1062
- hasVideo = true;
1063
- }
1064
- }
1065
- const mediaType = (() => {
1066
- if (hasAudio && hasVideo)
1067
- return "audio+video";
1068
- if (hasAudio)
1069
- return "audio";
1070
- if (hasVideo)
1071
- return "video";
1072
- return "unknown";
1073
- })();
1074
- }
1075
- async handleToolCalls(functionCalls) {
1076
- if (!this.session) {
1077
- console.log("No session active in handleToolCalls");
1078
- return;
1079
- }
1080
- if (!this.tools.length) {
1081
- console.log("No tools registered in handleToolCalls");
1082
- return;
1083
- }
1084
- try {
1085
- for (const tool2 of this.tools) {
1086
- if (!functionCalls.some((x) => x.name === tool2.name)) {
1087
- continue;
1088
- }
1089
- const parts = await tool2.callTool(functionCalls);
1090
- const functionResponses = parts.filter((part) => part.functionResponse).map((part) => ({
1091
- response: part.functionResponse.response,
1092
- id: part.functionResponse.id,
1093
- name: part.functionResponse.name
1094
- }));
1095
- if (functionResponses.length > 0) {
1096
- this.session.sendToolResponse({ functionResponses });
1097
- this.log("client-toolResponse: " + JSON.stringify({ functionResponses }));
1098
- } else {
1099
- console.log(`no tool call response!`);
1100
- }
1101
- }
1102
- } catch (error) {
1103
- console.error("Error handling tool calls:", error);
1104
- this.log("error: " + JSON.stringify({ message: "Tool call failed", error }));
1105
- }
1106
- }
1107
- sendToolResponse(response) {
1108
- if (!this.session) {
1109
- console.error("Cannot send tool response: session not connected");
1110
- return;
1111
- }
1112
- this.session.sendToolResponse(response);
1113
- this.log("client-toolResponse: " + JSON.stringify(response));
1114
- }
1115
- setConfig(config) {
1116
- if (config.tools) {
1117
- this.tools = config.tools;
1118
- }
1119
- this.updateState({ config });
1120
- }
1121
- getConfig() {
1122
- return { ...this.state.config };
1123
- }
1124
- async destroy() {
1125
- console.log(`calling destroy()`);
1126
- const cleanupHandler = this._cleanupHandler;
1127
- if (cleanupHandler) {
1128
- unregisterCleanupHandler(cleanupHandler);
1129
- delete this._cleanupHandler;
1130
- }
1131
- this.isExplicitDisconnect = true;
1132
- if (this.reconnectTimeout) {
1133
- clearTimeout(this.reconnectTimeout);
1134
- this.reconnectTimeout = null;
1135
- }
1136
- this.disconnect();
1137
- await this.audioRecorder?.destroy();
1138
- this.audioRecorder = null;
1139
- await this.audioStreamer?.destroy();
1140
- this.audioStreamer = null;
1141
- const audioCtx = await audioContext({ id: "audio-out" });
1142
- if (audioCtx?.state !== "closed") {
1143
- await audioCtx.close();
1144
- }
1145
- this.onAssistantStopSpeaking();
1146
- this.tools = [];
1147
- this.sessionHandle = null;
1148
- this.reconnectAttempts = 0;
1149
- }
1150
- }
1151
- function partition(arr, predicate) {
1152
- const truthy = [];
1153
- const falsy = [];
1154
- arr.forEach((item, index) => {
1155
- if (predicate(item, index, arr)) {
1156
- truthy.push(item);
1157
- } else {
1158
- falsy.push(item);
1159
- }
1160
- });
1161
- return [truthy, falsy];
1162
- }
1163
- var init_live_api_client = __esm(() => {
1164
- init_audio_recorder();
1165
- init_audio_streamer();
1166
- init_utils();
1167
- init_vol_meter();
1168
- init_cleanup();
1169
- });
1170
-
1171
- // ../liveapi/src/ai-tool-to-genai.ts
1172
- import { Type } from "@google/genai";
1173
- import { toJSONSchema } from "zod";
1174
- function jsonSchemaToGenAISchema(jsonSchema) {
1175
- const schema = {};
1176
- if (jsonSchema.type) {
1177
- switch (jsonSchema.type) {
1178
- case "string":
1179
- schema.type = Type.STRING;
1180
- break;
1181
- case "number":
1182
- schema.type = Type.NUMBER;
1183
- schema.format = jsonSchema.format || "float";
1184
- break;
1185
- case "integer":
1186
- schema.type = Type.INTEGER;
1187
- schema.format = jsonSchema.format || "int32";
1188
- break;
1189
- case "boolean":
1190
- schema.type = Type.BOOLEAN;
1191
- break;
1192
- case "array":
1193
- schema.type = Type.ARRAY;
1194
- if (jsonSchema.items) {
1195
- schema.items = jsonSchemaToGenAISchema(jsonSchema.items);
1196
- }
1197
- if (jsonSchema.minItems !== undefined) {
1198
- schema.minItems = jsonSchema.minItems;
1199
- }
1200
- if (jsonSchema.maxItems !== undefined) {
1201
- schema.maxItems = jsonSchema.maxItems;
1202
- }
1203
- break;
1204
- case "object":
1205
- schema.type = Type.OBJECT;
1206
- if (jsonSchema.properties) {
1207
- schema.properties = {};
1208
- for (const [key, value] of Object.entries(jsonSchema.properties)) {
1209
- schema.properties[key] = jsonSchemaToGenAISchema(value);
1210
- }
1211
- }
1212
- if (jsonSchema.required) {
1213
- schema.required = jsonSchema.required;
1214
- }
1215
- break;
1216
- default:
1217
- schema.type = jsonSchema.type;
1218
- }
1219
- }
1220
- if (jsonSchema.description) {
1221
- schema.description = jsonSchema.description;
1222
- }
1223
- if (jsonSchema.enum) {
1224
- schema.enum = jsonSchema.enum.map(String);
1225
- }
1226
- if (jsonSchema.default !== undefined) {
1227
- schema.default = jsonSchema.default;
1228
- }
1229
- if (jsonSchema.example !== undefined) {
1230
- schema.example = jsonSchema.example;
1231
- }
1232
- if (jsonSchema.nullable) {
1233
- schema.nullable = true;
1234
- }
1235
- if (jsonSchema.anyOf) {
1236
- schema.anyOf = jsonSchema.anyOf.map((s) => jsonSchemaToGenAISchema(s));
1237
- } else if (jsonSchema.oneOf) {
1238
- schema.anyOf = jsonSchema.oneOf.map((s) => jsonSchemaToGenAISchema(s));
1239
- }
1240
- if (jsonSchema.minimum !== undefined) {
1241
- schema.minimum = jsonSchema.minimum;
1242
- }
1243
- if (jsonSchema.maximum !== undefined) {
1244
- schema.maximum = jsonSchema.maximum;
1245
- }
1246
- if (jsonSchema.minLength !== undefined) {
1247
- schema.minLength = jsonSchema.minLength;
1248
- }
1249
- if (jsonSchema.maxLength !== undefined) {
1250
- schema.maxLength = jsonSchema.maxLength;
1251
- }
1252
- if (jsonSchema.pattern) {
1253
- schema.pattern = jsonSchema.pattern;
1254
- }
1255
- return schema;
1256
- }
1257
- function aiToolToGenAIFunction(tool2) {
1258
- const inputSchema = tool2.inputSchema;
1259
- let toolName = "tool";
1260
- let jsonSchema = {};
1261
- if (inputSchema) {
1262
- jsonSchema = toJSONSchema(inputSchema);
1263
- const description = inputSchema.description;
1264
- if (description) {
1265
- const nameMatch = description.match(/name:\s*(\w+)/);
1266
- if (nameMatch) {
1267
- toolName = nameMatch[1];
1268
- }
1269
- }
1270
- }
1271
- const genAISchema = jsonSchemaToGenAISchema(jsonSchema);
1272
- const functionDeclaration = {
1273
- name: toolName,
1274
- description: tool2.description || jsonSchema.description || "Tool function",
1275
- parameters: genAISchema
1276
- };
1277
- return functionDeclaration;
1278
- }
1279
- function aiToolToCallableTool(tool2, name) {
1280
- const toolName = name || "tool";
1281
- return {
1282
- name,
1283
- async tool() {
1284
- const functionDeclaration = aiToolToGenAIFunction(tool2);
1285
- if (name) {
1286
- functionDeclaration.name = name;
1287
- }
1288
- return {
1289
- functionDeclarations: [functionDeclaration]
1290
- };
1291
- },
1292
- async callTool(functionCalls) {
1293
- const parts = [];
1294
- for (const functionCall of functionCalls) {
1295
- if (functionCall.name !== toolName && name && functionCall.name !== name) {
1296
- continue;
1297
- }
1298
- if (tool2.execute) {
1299
- try {
1300
- const result = await tool2.execute(functionCall.args || {}, {
1301
- toolCallId: functionCall.id || "",
1302
- messages: []
1303
- });
1304
- parts.push({
1305
- functionResponse: {
1306
- id: functionCall.id,
1307
- name: functionCall.name || toolName,
1308
- response: {
1309
- output: result
1310
- }
1311
- }
1312
- });
1313
- } catch (error) {
1314
- parts.push({
1315
- functionResponse: {
1316
- id: functionCall.id,
1317
- name: functionCall.name || toolName,
1318
- response: {
1319
- error: error instanceof Error ? error.message : String(error)
1320
- }
1321
- }
1322
- });
1323
- }
1324
- }
1325
- }
1326
- return parts;
1327
- }
1328
- };
1329
- }
1330
- function extractSchemaFromTool(tool2) {
1331
- const inputSchema = tool2.inputSchema;
1332
- if (!inputSchema) {
1333
- return {};
1334
- }
1335
- return toJSONSchema(inputSchema);
1336
- }
1337
- function callableToolsFromObject(tools) {
1338
- return Object.entries(tools).map(([name, tool2]) => aiToolToCallableTool(tool2, name));
1339
- }
1340
- var init_ai_tool_to_genai = () => {};
1341
-
1342
- // ../liveapi/src/genai-to-ui-message.ts
1343
- import { createIdGenerator } from "ai";
1344
-
1345
- class LiveMessageAssembler {
1346
- currentUserParts = [];
1347
- currentAssistantParts = [];
1348
- allMessages = [];
1349
- idGenerator;
1350
- constructor(options = {}) {
1351
- this.idGenerator = options.idGenerator ?? createIdGenerator({
1352
- prefix: "msg",
1353
- size: 24
1354
- });
1355
- }
1356
- processMessage(message) {
1357
- const updates = this.processServerMessage(message);
1358
- const newMessages = this.addParts(updates);
1359
- if (newMessages.length > 0) {
1360
- this.allMessages.push(...newMessages);
1361
- }
1362
- if (message.serverContent?.interrupted) {
1363
- const flushedMessages = this.flushPending();
1364
- this.allMessages.push(...flushedMessages);
1365
- }
1366
- const allMessages = [...this.allMessages];
1367
- if (this.currentUserParts.length > 0) {
1368
- allMessages.push(this.createMessage("user", this.currentUserParts));
1369
- }
1370
- if (this.currentAssistantParts.length > 0) {
1371
- allMessages.push(this.createMessage("assistant", this.currentAssistantParts));
1372
- }
1373
- return allMessages.map(mergeConsecutiveTextParts);
1374
- }
1375
- clear() {
1376
- this.currentUserParts = [];
1377
- this.currentAssistantParts = [];
1378
- this.allMessages = [];
1379
- }
1380
- getAllMessages() {
1381
- return [...this.allMessages];
1382
- }
1383
- processServerMessage(message) {
1384
- const updates = [];
1385
- if (message.serverContent?.modelTurn) {
1386
- const parts = this.extractPartsFromContent(message.serverContent.modelTurn);
1387
- for (const part of parts) {
1388
- updates.push({
1389
- part,
1390
- role: "assistant",
1391
- isFinal: false
1392
- });
1393
- }
1394
- if (message.serverContent.turnComplete) {
1395
- updates.push({
1396
- part: { type: "text", text: "" },
1397
- role: "assistant",
1398
- isFinal: true
1399
- });
1400
- }
1401
- }
1402
- if (message.serverContent?.inputTranscription?.text) {
1403
- updates.push({
1404
- part: {
1405
- type: "text",
1406
- text: message.serverContent.inputTranscription.text
1407
- },
1408
- role: "user",
1409
- isFinal: message.serverContent.inputTranscription.finished || false
1410
- });
1411
- }
1412
- if (message.serverContent?.outputTranscription?.text) {
1413
- updates.push({
1414
- part: {
1415
- type: "text",
1416
- text: message.serverContent.outputTranscription.text
1417
- },
1418
- role: "assistant",
1419
- isFinal: message.serverContent.outputTranscription.finished || false
1420
- });
1421
- }
1422
- if (message.toolCall?.functionCalls) {
1423
- for (const functionCall of message.toolCall.functionCalls) {
1424
- updates.push({
1425
- part: this.functionCallToToolPart(functionCall),
1426
- role: "assistant",
1427
- isFinal: false
1428
- });
1429
- }
1430
- }
1431
- return updates;
1432
- }
1433
- processClientMessage(message) {
1434
- const updates = [];
1435
- if (message.clientContent?.turns) {
1436
- for (const turn of message.clientContent.turns) {
1437
- const role = this.determineRole(turn.role);
1438
- const parts = this.extractPartsFromContent(turn);
1439
- for (const part of parts) {
1440
- updates.push({
1441
- part,
1442
- role,
1443
- isFinal: false
1444
- });
1445
- }
1446
- }
1447
- if (message.clientContent.turnComplete) {
1448
- const lastTurn = message.clientContent.turns[message.clientContent.turns.length - 1];
1449
- const role = this.determineRole(lastTurn?.role);
1450
- updates.push({
1451
- part: { type: "text", text: "" },
1452
- role,
1453
- isFinal: true
1454
- });
1455
- }
1456
- }
1457
- if (message.realtimeInput) {
1458
- const parts = this.processRealtimeInput(message.realtimeInput);
1459
- const isActivityEnd = !!message.realtimeInput.activityEnd;
1460
- for (const part of parts) {
1461
- updates.push({
1462
- part,
1463
- role: "user",
1464
- isFinal: isActivityEnd
1465
- });
1466
- }
1467
- }
1468
- if (message.toolResponse?.functionResponses) {
1469
- for (const response of message.toolResponse.functionResponses) {
1470
- updates.push({
1471
- part: this.functionResponseToToolPart(response),
1472
- role: "user",
1473
- isFinal: false
1474
- });
1475
- }
1476
- }
1477
- return updates;
1478
- }
1479
- addParts(updates) {
1480
- const completedMessages = [];
1481
- for (const update of updates) {
1482
- if (update.part.type === "text" && update.part.text === "" && update.isFinal) {
1483
- if (update.role === "user" && this.currentUserParts.length > 0) {
1484
- completedMessages.push(this.createMessage("user", this.currentUserParts));
1485
- this.currentUserParts = [];
1486
- } else if (update.role === "assistant" && this.currentAssistantParts.length > 0) {
1487
- completedMessages.push(this.createMessage("assistant", this.currentAssistantParts));
1488
- this.currentAssistantParts = [];
1489
- }
1490
- } else {
1491
- if (update.role === "user") {
1492
- this.currentUserParts.push(update.part);
1493
- } else if (update.role === "assistant") {
1494
- this.currentAssistantParts.push(update.part);
1495
- }
1496
- if (update.isFinal) {
1497
- if (update.role === "user" && this.currentUserParts.length > 0) {
1498
- completedMessages.push(this.createMessage("user", this.currentUserParts));
1499
- this.currentUserParts = [];
1500
- } else if (update.role === "assistant" && this.currentAssistantParts.length > 0) {
1501
- completedMessages.push(this.createMessage("assistant", this.currentAssistantParts));
1502
- this.currentAssistantParts = [];
1503
- }
1504
- }
1505
- }
1506
- }
1507
- return completedMessages;
1508
- }
1509
- getCurrentParts(role) {
1510
- return role === "user" ? [...this.currentUserParts] : [...this.currentAssistantParts];
1511
- }
1512
- flushPending() {
1513
- const messages = [];
1514
- if (this.currentUserParts.length > 0) {
1515
- messages.push(this.createMessage("user", this.currentUserParts));
1516
- this.currentUserParts = [];
1517
- }
1518
- if (this.currentAssistantParts.length > 0) {
1519
- messages.push(this.createMessage("assistant", this.currentAssistantParts));
1520
- this.currentAssistantParts = [];
1521
- }
1522
- return messages;
1523
- }
1524
- flush() {
1525
- const messages = this.flushPending();
1526
- this.allMessages.push(...messages);
1527
- return [...this.allMessages];
1528
- }
1529
- extractPartsFromContent(content) {
1530
- if (!content.parts)
1531
- return [];
1532
- return content.parts.map((part) => this.convertPartToUIPart(part)).filter((p) => p !== null);
1533
- }
1534
- processRealtimeInput(input) {
1535
- const parts = [];
1536
- if (input.text) {
1537
- parts.push({
1538
- type: "text",
1539
- text: input.text,
1540
- state: input.activityEnd ? "done" : "streaming"
1541
- });
1542
- }
1543
- if (input.mediaChunks) {
1544
- for (const chunk of input.mediaChunks) {
1545
- if (chunk && typeof chunk === "object" && chunk.mimeType) {
1546
- if (chunk.mimeType?.startsWith("audio/") || chunk.mimeType?.startsWith("video/")) {
1547
- continue;
1548
- }
1549
- parts.push({
1550
- type: "data-url",
1551
- data: {
1552
- mimeType: chunk.mimeType || "application/octet-stream",
1553
- url: `data:${chunk.mimeType};base64,${chunk.data}`
1554
- }
1555
- });
1556
- }
1557
- }
1558
- }
1559
- return parts;
1560
- }
1561
- convertPartToUIPart(part) {
1562
- if (typeof part === "string") {
1563
- return {
1564
- type: "text",
1565
- text: part
1566
- };
1567
- }
1568
- if (part.text) {
1569
- return {
1570
- type: "text",
1571
- text: part.text,
1572
- providerMetadata: part.thought ? { thought: { value: true } } : undefined
1573
- };
1574
- }
1575
- if (part.inlineData) {
1576
- if (part.inlineData.mimeType?.startsWith("audio/") || part.inlineData.mimeType?.startsWith("video/")) {
1577
- return null;
1578
- }
1579
- const mimeType = part.inlineData.mimeType || "application/octet-stream";
1580
- return {
1581
- type: "data-url",
1582
- data: {
1583
- mimeType,
1584
- url: `data:${mimeType};base64,${part.inlineData.data}`
1585
- }
1586
- };
1587
- }
1588
- if (part.fileData) {
1589
- return {
1590
- type: "file",
1591
- name: part.fileData.fileUri || "file",
1592
- url: part.fileData.fileUri,
1593
- mediaType: part.fileData.mimeType || "application/octet-stream"
1594
- };
1595
- }
1596
- if (part.functionCall) {
1597
- return this.functionCallToToolPart(part.functionCall);
1598
- }
1599
- if (part.functionResponse) {
1600
- return this.functionResponseToToolPart(part.functionResponse);
1601
- }
1602
- if (part.codeExecutionResult) {
1603
- return {
1604
- type: "tool-result",
1605
- toolCallId: this.idGenerator(),
1606
- toolName: "executableCode",
1607
- state: part.codeExecutionResult.outcome === "OUTCOME_OK" ? "output-available" : "output-error",
1608
- input: {},
1609
- output: part.codeExecutionResult.output || ""
1610
- };
1611
- }
1612
- if (part.executableCode) {
1613
- return {
1614
- type: "tool-call",
1615
- toolCallId: this.idGenerator(),
1616
- toolName: "executableCode",
1617
- state: "input-available",
1618
- input: {
1619
- language: part.executableCode.language || "unknown",
1620
- code: part.executableCode.code || ""
1621
- }
1622
- };
1623
- }
1624
- return null;
1625
- }
1626
- functionCallToToolPart(functionCall) {
1627
- return {
1628
- type: "tool-call",
1629
- toolCallId: functionCall.id || this.idGenerator(),
1630
- toolName: functionCall.name,
1631
- state: "input-available",
1632
- input: functionCall.args || {}
1633
- };
1634
- }
1635
- functionResponseToToolPart(response) {
1636
- const responseData = response.response || {};
1637
- const isError = responseData.error !== undefined;
1638
- if (isError) {
1639
- return {
1640
- type: "tool-result",
1641
- toolCallId: response.id || this.idGenerator(),
1642
- toolName: response.name,
1643
- state: "output-error",
1644
- input: {},
1645
- errorText: JSON.stringify(responseData.error)
1646
- };
1647
- }
1648
- return {
1649
- type: "tool-result",
1650
- toolCallId: response.id || this.idGenerator(),
1651
- toolName: response.name,
1652
- state: "output-available",
1653
- input: {},
1654
- output: responseData.output || responseData
1655
- };
1656
- }
1657
- createMessage(role, parts) {
1658
- return {
1659
- id: this.idGenerator(),
1660
- role,
1661
- parts: [...parts]
1662
- };
1663
- }
1664
- determineRole(role) {
1665
- if (!role)
1666
- return "user";
1667
- if (role === "model")
1668
- return "assistant";
1669
- if (role === "system" || role === "user" || role === "assistant")
1670
- return role;
1671
- return "user";
1672
- }
1673
- }
1674
- function mergeConsecutiveTextParts(message) {
1675
- const mergedParts = message.parts.reduce((acc, part) => {
1676
- const lastPart = acc[acc.length - 1];
1677
- if (part.type === "text" && lastPart?.type === "text") {
1678
- const mergedTextPart = {
1679
- type: "text",
1680
- text: lastPart.text + part.text
1681
- };
1682
- const currentTextPart = part;
1683
- if (currentTextPart.state) {
1684
- mergedTextPart.state = currentTextPart.state;
1685
- } else if (lastPart.state) {
1686
- mergedTextPart.state = lastPart.state;
1687
- }
1688
- const lastTextPart = lastPart;
1689
- if (lastTextPart.providerMetadata || currentTextPart.providerMetadata) {
1690
- mergedTextPart.providerMetadata = {
1691
- ...lastTextPart.providerMetadata,
1692
- ...currentTextPart.providerMetadata
1693
- };
1694
- }
1695
- acc[acc.length - 1] = mergedTextPart;
1696
- } else {
1697
- acc.push(part);
1698
- }
1699
- return acc;
1700
- }, []);
1701
- return {
1702
- ...message,
1703
- parts: mergedParts
1704
- };
1705
- }
1706
- function uiMessageToClientMessage(message, turnComplete = true) {
1707
- const toolResponses = message.parts.filter((part) => part.type === "tool-result").map((part) => {
1708
- const toolPart = part;
1709
- return {
1710
- id: toolPart.toolCallId,
1711
- name: "",
1712
- response: toolPart.state === "output-error" ? { error: toolPart.errorText } : toolPart.output || {}
1713
- };
1714
- });
1715
- if (toolResponses.length > 0) {
1716
- return {
1717
- toolResponse: {
1718
- functionResponses: toolResponses
1719
- }
1720
- };
1721
- }
1722
- const parts = message.parts.map((part) => uiPartToGenAIPart(part)).filter((p) => p !== null);
1723
- return {
1724
- clientContent: {
1725
- turns: [
1726
- {
1727
- role: message.role,
1728
- parts
1729
- }
1730
- ],
1731
- turnComplete
1732
- }
1733
- };
1734
- }
1735
- function uiPartToGenAIPart(part) {
1736
- switch (part.type) {
1737
- case "text":
1738
- return {
1739
- text: part.text
1740
- };
1741
- case "file":
1742
- const filePart = part;
1743
- return {
1744
- fileData: {
1745
- fileUri: filePart.url,
1746
- mimeType: filePart.mediaType
1747
- }
1748
- };
1749
- case "tool-call":
1750
- const toolCall = part;
1751
- return {
1752
- functionCall: {
1753
- id: toolCall.toolCallId,
1754
- name: "",
1755
- args: toolCall.input
1756
- }
1757
- };
1758
- default:
1759
- if (part.type.startsWith("data-")) {
1760
- const dataPart = part;
1761
- if (dataPart.data && typeof dataPart.data === "object" && "url" in dataPart.data && dataPart.data.url) {
1762
- const dataUrl = dataPart.data.url;
1763
- const base64Match = dataUrl.match(/^data:([^;]+);base64,(.+)$/);
1764
- if (base64Match) {
1765
- const mimeType = base64Match[1] || "application/octet-stream";
1766
- return {
1767
- inlineData: {
1768
- mimeType,
1769
- data: base64Match[2]
1770
- }
1771
- };
1772
- }
1773
- }
1774
- }
1775
- return null;
1776
- }
1777
- }
1778
- var init_genai_to_ui_message = () => {};
1779
-
1780
- // ../liveapi/src/api.ts
1781
- import { GoogleGenAI as GoogleGenAI2, Modality } from "@google/genai";
1782
- function createAudioChatAPI(options) {
1783
- const { geminiApiKey, tools, onRequest } = options;
1784
- const ai = new GoogleGenAI2({
1785
- apiKey: geminiApiKey,
1786
- apiVersion: "v1alpha"
1787
- });
1788
- return async function handler(request) {
1789
- try {
1790
- if (onRequest) {
1791
- await onRequest({ request });
1792
- }
1793
- const method = request.method.toUpperCase();
1794
- if (method === "GET") {
1795
- const url = new URL(request.url);
1796
- const model = url.searchParams.get("model") || "gemini-2.0-flash-live-001";
1797
- let token;
1798
- try {
1799
- const authToken = await ai.authTokens.create({
1800
- config: {
1801
- uses: 1,
1802
- expireTime: new Date(Date.now() + 30 * 60 * 1000).toISOString(),
1803
- newSessionExpireTime: new Date(Date.now() + 60 * 1000).toISOString(),
1804
- liveConnectConstraints: {
1805
- model,
1806
- config: {
1807
- responseModalities: [Modality.AUDIO, Modality.TEXT],
1808
- sessionResumption: {},
1809
- temperature: 0.7
1810
- }
1811
- }
1812
- }
1813
- });
1814
- token = authToken.name || "";
1815
- } catch (error) {
1816
- console.error("Error generating token:", error);
1817
- return new Response(JSON.stringify({ error: "Failed to generate token" }), {
1818
- status: 500,
1819
- headers: {
1820
- "Content-Type": "application/json"
1821
- }
1822
- });
1823
- }
1824
- const toolDefinitions = Object.entries(tools).map(([name, tool2]) => ({
1825
- name,
1826
- description: tool2.description || `Tool: ${name}`,
1827
- inputSchema: extractSchemaFromTool(tool2)
1828
- }));
1829
- const response = {
1830
- token,
1831
- tools: toolDefinitions
1832
- };
1833
- return new Response(JSON.stringify(response), {
1834
- status: 200,
1835
- headers: {
1836
- "Content-Type": "application/json"
1837
- }
1838
- });
1839
- }
1840
- if (method === "POST") {
1841
- const body = await request.json();
1842
- const toolRequest = body;
1843
- const { tool: toolName, input } = toolRequest;
1844
- if (!toolName) {
1845
- return new Response(JSON.stringify({ error: "Missing tool name" }), {
1846
- status: 400,
1847
- headers: {
1848
- "Content-Type": "application/json"
1849
- }
1850
- });
1851
- }
1852
- const tool2 = tools[toolName];
1853
- if (!tool2) {
1854
- return new Response(JSON.stringify({ error: `Tool not found: ${toolName}` }), {
1855
- status: 404,
1856
- headers: {
1857
- "Content-Type": "application/json"
1858
- }
1859
- });
1860
- }
1861
- if (!tool2.execute) {
1862
- return new Response(JSON.stringify({
1863
- error: `Tool ${toolName} has no execute function`
1864
- }), {
1865
- status: 400,
1866
- headers: {
1867
- "Content-Type": "application/json"
1868
- }
1869
- });
1870
- }
1871
- try {
1872
- const result = await tool2.execute(input || {}, {
1873
- toolCallId: crypto.randomUUID(),
1874
- messages: []
1875
- });
1876
- return new Response(JSON.stringify(result), {
1877
- status: 200,
1878
- headers: {
1879
- "Content-Type": "application/json"
1880
- }
1881
- });
1882
- } catch (error) {
1883
- console.error(`Error executing tool ${toolName}:`, error);
1884
- return new Response(JSON.stringify({
1885
- error: error instanceof Error ? error.message : "Tool execution failed"
1886
- }), {
1887
- status: 500,
1888
- headers: {
1889
- "Content-Type": "application/json"
1890
- }
1891
- });
1892
- }
1893
- }
1894
- return new Response(JSON.stringify({ error: "Method not allowed" }), {
1895
- status: 405,
1896
- headers: {
1897
- "Content-Type": "application/json"
1898
- }
1899
- });
1900
- } catch (error) {
1901
- console.error("API handler error:", error);
1902
- if (error instanceof Error && error.message.toLowerCase().includes("unauthorized")) {
1903
- return new Response(JSON.stringify({ error: "Unauthorized" }), {
1904
- status: 401,
1905
- headers: {
1906
- "Content-Type": "application/json"
1907
- }
1908
- });
1909
- }
1910
- return new Response(JSON.stringify({
1911
- error: error instanceof Error ? error.message : "Internal server error"
1912
- }), {
1913
- status: 500,
1914
- headers: {
1915
- "Content-Type": "application/json"
1916
- }
1917
- });
1918
- }
1919
- };
1920
- }
1921
- var init_api = __esm(() => {
1922
- init_ai_tool_to_genai();
1923
- });
1924
-
1925
- // ../liveapi/src/index.ts
1926
- var exports_src = {};
1927
- __export(exports_src, {
1928
- unregisterCleanupHandler: () => unregisterCleanupHandler,
1929
- uiMessageToClientMessage: () => uiMessageToClientMessage,
1930
- registerCleanupHandler: () => registerCleanupHandler,
1931
- mergeConsecutiveTextParts: () => mergeConsecutiveTextParts,
1932
- extractSchemaFromTool: () => extractSchemaFromTool,
1933
- downSampleInt16Buffer: () => downSampleInt16Buffer,
1934
- downSampleAudioBuffer: () => downSampleAudioBuffer,
1935
- createAudioChatAPI: () => createAudioChatAPI,
1936
- closeAllAudioContexts: () => closeAllAudioContexts,
1937
- callableToolsFromObject: () => callableToolsFromObject,
1938
- aiToolToGenAIFunction: () => aiToolToGenAIFunction,
1939
- aiToolToCallableTool: () => aiToolToCallableTool,
1940
- LiveMessageAssembler: () => LiveMessageAssembler,
1941
- LiveAPIClient: () => LiveAPIClient
1942
- });
1943
- var init_src = __esm(() => {
1944
- init_live_api_client();
1945
- init_ai_tool_to_genai();
1946
- init_genai_to_ui_message();
1947
- init_api();
1948
- init_utils();
1949
- init_cleanup();
1950
- });
1951
-
1952
- // src/cli.tsx
1953
- import { cac } from "cac";
1954
-
1955
- // ../node_modules/.pnpm/string-dedent@3.0.2/node_modules/string-dedent/dist/dedent.mjs
1956
- var cache = new WeakMap;
1957
- var newline = /(\n|\r\n?|\u2028|\u2029)/g;
1958
- var leadingWhitespace = /^\s*/;
1959
- var nonWhitespace = /\S/;
1960
- var slice = Array.prototype.slice;
1961
- var zero = 48;
1962
- var nine = 57;
1963
- var lowerA = 97;
1964
- var lowerF = 102;
1965
- var upperA = 65;
1966
- var upperF = 70;
1967
- function dedent(arg) {
1968
- if (typeof arg === "string") {
1969
- return process2([arg])[0];
1970
- }
1971
- if (typeof arg === "function") {
1972
- return function() {
1973
- const args = slice.call(arguments);
1974
- args[0] = processTemplateStringsArray(args[0]);
1975
- return arg.apply(this, args);
1976
- };
1977
- }
1978
- const strings = processTemplateStringsArray(arg);
1979
- let s = getCooked(strings, 0);
1980
- for (let i = 1;i < strings.length; i++) {
1981
- s += arguments[i] + getCooked(strings, i);
1982
- }
1983
- return s;
1984
- }
1985
- function getCooked(strings, index) {
1986
- const str = strings[index];
1987
- if (str === undefined)
1988
- throw new TypeError(`invalid cooked string at index ${index}`);
1989
- return str;
1990
- }
1991
- function processTemplateStringsArray(strings) {
1992
- const cached = cache.get(strings);
1993
- if (cached)
1994
- return cached;
1995
- const raw = process2(strings.raw);
1996
- const cooked = raw.map(cook);
1997
- Object.defineProperty(cooked, "raw", {
1998
- value: Object.freeze(raw)
1999
- });
2000
- Object.freeze(cooked);
2001
- cache.set(strings, cooked);
2002
- return cooked;
2003
- }
2004
- function process2(strings) {
2005
- const splitQuasis = strings.map((quasi) => quasi.split(newline));
2006
- let common;
2007
- for (let i = 0;i < splitQuasis.length; i++) {
2008
- const lines = splitQuasis[i];
2009
- const firstSplit = i === 0;
2010
- const lastSplit = i + 1 === splitQuasis.length;
2011
- if (firstSplit) {
2012
- if (lines.length === 1 || lines[0].length > 0) {
2013
- throw new Error("invalid content on opening line");
2014
- }
2015
- lines[1] = "";
2016
- }
2017
- if (lastSplit) {
2018
- if (lines.length === 1 || nonWhitespace.test(lines[lines.length - 1])) {
2019
- throw new Error("invalid content on closing line");
2020
- }
2021
- lines[lines.length - 2] = "";
2022
- lines[lines.length - 1] = "";
2023
- }
2024
- for (let j = 2;j < lines.length; j += 2) {
2025
- const text = lines[j];
2026
- const lineContainsTemplateExpression = j + 1 === lines.length && !lastSplit;
2027
- const leading = leadingWhitespace.exec(text)[0];
2028
- if (!lineContainsTemplateExpression && leading.length === text.length) {
2029
- lines[j] = "";
2030
- continue;
2031
- }
2032
- common = commonStart(leading, common);
2033
- }
2034
- }
2035
- const min = common ? common.length : 0;
2036
- return splitQuasis.map((lines) => {
2037
- let quasi = lines[0];
2038
- for (let i = 1;i < lines.length; i += 2) {
2039
- const newline2 = lines[i];
2040
- const text = lines[i + 1];
2041
- quasi += newline2 + text.slice(min);
2042
- }
2043
- return quasi;
2044
- });
2045
- }
2046
- function commonStart(a, b) {
2047
- if (b === undefined || a === b)
2048
- return a;
2049
- let i = 0;
2050
- for (const len = Math.min(a.length, b.length);i < len; i++) {
2051
- if (a[i] !== b[i])
2052
- break;
2053
- }
2054
- return a.slice(0, i);
2055
- }
2056
- function cook(raw) {
2057
- let out = "";
2058
- let start = 0;
2059
- let i = 0;
2060
- while ((i = raw.indexOf("\\", i)) > -1) {
2061
- out += raw.slice(start, i);
2062
- if (++i === raw.length)
2063
- return;
2064
- const next = raw[i++];
2065
- switch (next) {
2066
- case "b":
2067
- out += "\b";
2068
- break;
2069
- case "t":
2070
- out += "\t";
2071
- break;
2072
- case "n":
2073
- out += `
2074
- `;
2075
- break;
2076
- case "v":
2077
- out += "\v";
2078
- break;
2079
- case "f":
2080
- out += "\f";
2081
- break;
2082
- case "r":
2083
- out += "\r";
2084
- break;
2085
- case "\r":
2086
- if (i < raw.length && raw[i] === `
2087
- `)
2088
- ++i;
2089
- case `
2090
- `:
2091
- case "\u2028":
2092
- case "\u2029":
2093
- break;
2094
- case "0":
2095
- if (isDigit(raw, i))
2096
- return;
2097
- out += "\x00";
2098
- break;
2099
- case "x": {
2100
- const n = parseHex(raw, i, i + 2);
2101
- if (n === -1)
2102
- return;
2103
- i += 2;
2104
- out += String.fromCharCode(n);
2105
- break;
2106
- }
2107
- case "u": {
2108
- let n;
2109
- if (i < raw.length && raw[i] === "{") {
2110
- const end = raw.indexOf("}", ++i);
2111
- if (end === -1)
2112
- return;
2113
- n = parseHex(raw, i, end);
2114
- i = end + 1;
2115
- } else {
2116
- n = parseHex(raw, i, i + 4);
2117
- i += 4;
2118
- }
2119
- if (n === -1 || n > 1114111)
2120
- return;
2121
- out += String.fromCodePoint(n);
2122
- break;
2123
- }
2124
- default:
2125
- if (isDigit(next, 0))
2126
- return;
2127
- out += next;
2128
- }
2129
- start = i;
2130
- }
2131
- return out + raw.slice(start);
2132
- }
2133
- function isDigit(str, index) {
2134
- const c = str.charCodeAt(index);
2135
- return c >= zero && c <= nine;
2136
- }
2137
- function parseHex(str, index, end) {
2138
- if (end >= str.length)
2139
- return -1;
2140
- let n = 0;
2141
- for (;index < end; index++) {
2142
- const c = hexToInt(str.charCodeAt(index));
2143
- if (c === -1)
2144
- return -1;
2145
- n = n * 16 + c;
2146
- }
2147
- return n;
2148
- }
2149
- function hexToInt(c) {
2150
- if (c >= zero && c <= nine)
2151
- return c - zero;
2152
- if (c >= lowerA && c <= lowerF)
2153
- return c - lowerA + 10;
2154
- if (c >= upperA && c <= upperF)
2155
- return c - upperA + 10;
2156
- return -1;
2157
- }
2158
-
2159
- // src/cli.tsx
2160
- import { mediaDevices } from "node-web-audio-api";
2161
- import { MediaResolution as MediaResolution2, Modality as Modality2 } from "@google/genai";
2162
- import * as webAudioApi from "node-web-audio-api";
2163
- import fs3 from "node:fs";
2164
- import path2 from "node:path";
2165
- import WaveFile from "wavefile";
2166
- import pc2 from "picocolors";
2167
-
2168
- // src/tools.ts
2169
- import { tool } from "ai";
2170
- import { z as z2 } from "zod";
2171
- import { spawn } from "node:child_process";
2172
- import net from "node:net";
2173
- import {
2174
- createOpencodeClient
2175
- } from "@opencode-ai/sdk";
2176
- import { formatDistanceToNow } from "date-fns";
2177
-
2178
- // src/config.ts
2179
- import fs from "node:fs";
2180
- import os from "node:os";
2181
- import path from "node:path";
2182
- import { z } from "zod";
2183
- var PreferredModelSchema = z.object({
2184
- providerId: z.string(),
2185
- modelId: z.string()
2186
- });
2187
- var KimakiConfigSchema = z.object({
2188
- preferredModel: PreferredModelSchema.optional()
2189
- });
2190
- var configDir = path.join(os.homedir(), ".kimaki");
2191
- var configPath = path.join(configDir, "kimaki.json");
2192
- async function readConfig() {
2193
- try {
2194
- const configData = await fs.promises.readFile(configPath, "utf-8");
2195
- const config = JSON.parse(configData);
2196
- return KimakiConfigSchema.parse(config);
2197
- } catch (error) {
2198
- return {};
2199
- }
2200
- }
2201
- async function writeConfig(config) {
2202
- await fs.promises.mkdir(configDir, { recursive: true });
2203
- await fs.promises.writeFile(configPath, JSON.stringify(config, null, 2));
2204
- }
2205
- async function updateConfig(updates) {
2206
- const config = await readConfig();
2207
- const updatedConfig = { ...config, ...updates };
2208
- await writeConfig(updatedConfig);
2209
- }
2210
-
2211
- // src/markdown.ts
2212
- import { DateTime } from "luxon";
2213
- import * as yaml from "js-yaml";
2214
-
2215
- class ShareMarkdown {
2216
- client;
2217
- constructor(client) {
2218
- this.client = client;
2219
- }
2220
- async generate(options) {
2221
- const { sessionID, includeSystemInfo, lastAssistantOnly } = options;
2222
- const sessionResponse = await this.client.session.get({
2223
- path: { id: sessionID }
2224
- });
2225
- if (!sessionResponse.data) {
2226
- throw new Error(`Session ${sessionID} not found`);
2227
- }
2228
- const session = sessionResponse.data;
2229
- const messagesResponse = await this.client.session.messages({
2230
- path: { id: sessionID }
2231
- });
2232
- if (!messagesResponse.data) {
2233
- throw new Error(`No messages found for session ${sessionID}`);
2234
- }
2235
- const messages = messagesResponse.data;
2236
- const messagesToRender = lastAssistantOnly ? (() => {
2237
- const assistantMessages = messages.filter((m) => m.info.role === "assistant");
2238
- return assistantMessages.length > 0 ? [assistantMessages[assistantMessages.length - 1]] : [];
2239
- })() : messages;
2240
- const lines = [];
2241
- if (!lastAssistantOnly) {
2242
- lines.push(`# ${session.title || "Untitled Session"}`);
2243
- lines.push("");
2244
- if (includeSystemInfo === true) {
2245
- lines.push("## Session Information");
2246
- lines.push("");
2247
- lines.push(`- **Created**: ${DateTime.fromMillis(session.time.created).toLocaleString(DateTime.DATETIME_MED)}`);
2248
- lines.push(`- **Updated**: ${DateTime.fromMillis(session.time.updated).toLocaleString(DateTime.DATETIME_MED)}`);
2249
- if (session.version) {
2250
- lines.push(`- **OpenCode Version**: v${session.version}`);
2251
- }
2252
- lines.push("");
2253
- }
2254
- lines.push("## Conversation");
2255
- lines.push("");
2256
- }
2257
- for (const message of messagesToRender) {
2258
- const messageLines = this.renderMessage(message.info, message.parts);
2259
- lines.push(...messageLines);
2260
- lines.push("");
2261
- }
2262
- return lines.join(`
2263
- `);
2264
- }
2265
- renderMessage(message, parts) {
2266
- const lines = [];
2267
- if (message.role === "user") {
2268
- lines.push("### \uD83D\uDC64 User");
2269
- lines.push("");
2270
- for (const part of parts) {
2271
- if (part.type === "text" && part.text) {
2272
- lines.push(part.text);
2273
- lines.push("");
2274
- } else if (part.type === "file") {
2275
- lines.push(`\uD83D\uDCCE **Attachment**: ${part.filename || "unnamed file"}`);
2276
- if (part.url) {
2277
- lines.push(` - URL: ${part.url}`);
2278
- }
2279
- lines.push("");
2280
- }
2281
- }
2282
- } else if (message.role === "assistant") {
2283
- lines.push(`### \uD83E\uDD16 Assistant (${message.modelID || "unknown model"})`);
2284
- lines.push("");
2285
- const filteredParts = parts.filter((part) => {
2286
- if (part.type === "step-start" && parts.indexOf(part) > 0)
2287
- return false;
2288
- if (part.type === "snapshot")
2289
- return false;
2290
- if (part.type === "patch")
2291
- return false;
2292
- if (part.type === "step-finish")
2293
- return false;
2294
- if (part.type === "text" && part.synthetic === true)
2295
- return false;
2296
- if (part.type === "tool" && part.tool === "todoread")
2297
- return false;
2298
- if (part.type === "text" && !part.text)
2299
- return false;
2300
- if (part.type === "tool" && (part.state.status === "pending" || part.state.status === "running"))
2301
- return false;
2302
- return true;
2303
- });
2304
- for (const part of filteredParts) {
2305
- const partLines = this.renderPart(part, message);
2306
- lines.push(...partLines);
2307
- }
2308
- if (message.time?.completed) {
2309
- const duration = message.time.completed - message.time.created;
2310
- lines.push("");
2311
- lines.push(`*Completed in ${this.formatDuration(duration)}*`);
2312
- }
2313
- }
2314
- return lines;
2315
- }
2316
- renderPart(part, message) {
2317
- const lines = [];
2318
- switch (part.type) {
2319
- case "text":
2320
- if (part.text) {
2321
- lines.push(part.text);
2322
- lines.push("");
2323
- }
2324
- break;
2325
- case "reasoning":
2326
- if (part.text) {
2327
- lines.push("<details>");
2328
- lines.push("<summary>\uD83D\uDCAD Thinking</summary>");
2329
- lines.push("");
2330
- lines.push(part.text);
2331
- lines.push("");
2332
- lines.push("</details>");
2333
- lines.push("");
2334
- }
2335
- break;
2336
- case "tool":
2337
- if (part.state.status === "completed") {
2338
- lines.push(`#### \uD83D\uDEE0️ Tool: ${part.tool}`);
2339
- lines.push("");
2340
- if (part.state.input && Object.keys(part.state.input).length > 0) {
2341
- lines.push("**Input:**");
2342
- lines.push("```yaml");
2343
- lines.push(yaml.dump(part.state.input, { lineWidth: -1 }));
2344
- lines.push("```");
2345
- lines.push("");
2346
- }
2347
- if (part.state.output) {
2348
- lines.push("**Output:**");
2349
- lines.push("```");
2350
- lines.push(part.state.output);
2351
- lines.push("```");
2352
- lines.push("");
2353
- }
2354
- if (part.state.time?.start && part.state.time?.end) {
2355
- const duration = part.state.time.end - part.state.time.start;
2356
- if (duration > 2000) {
2357
- lines.push(`*Duration: ${this.formatDuration(duration)}*`);
2358
- lines.push("");
2359
- }
2360
- }
2361
- } else if (part.state.status === "error") {
2362
- lines.push(`#### ❌ Tool Error: ${part.tool}`);
2363
- lines.push("");
2364
- lines.push("```");
2365
- lines.push(part.state.error || "Unknown error");
2366
- lines.push("```");
2367
- lines.push("");
2368
- }
2369
- break;
2370
- case "step-start":
2371
- lines.push(`**Started using ${message.providerID}/${message.modelID}**`);
2372
- lines.push("");
2373
- break;
2374
- }
2375
- return lines;
2376
- }
2377
- formatDuration(ms) {
2378
- if (ms < 1000)
2379
- return `${ms}ms`;
2380
- if (ms < 60000)
2381
- return `${(ms / 1000).toFixed(1)}s`;
2382
- const minutes = Math.floor(ms / 60000);
2383
- const seconds = Math.floor(ms % 60000 / 1000);
2384
- return `${minutes}m ${seconds}s`;
2385
- }
2386
- }
2387
-
2388
- // src/tools.ts
2389
- import pc from "picocolors";
2390
- async function getOpenPort() {
2391
- return new Promise((resolve, reject) => {
2392
- const server = net.createServer();
2393
- server.listen(0, () => {
2394
- const address = server.address();
2395
- if (address && typeof address === "object") {
2396
- const port = address.port;
2397
- server.close(() => {
2398
- resolve(port);
2399
- });
2400
- } else {
2401
- reject(new Error("Failed to get port"));
2402
- }
2403
- });
2404
- server.on("error", reject);
2405
- });
2406
- }
2407
- async function waitForServer(port, maxAttempts = 30) {
2408
- for (let i = 0;i < maxAttempts; i++) {
2409
- try {
2410
- const endpoints = [
2411
- `http://localhost:${port}/api/health`,
2412
- `http://localhost:${port}/`,
2413
- `http://localhost:${port}/api`
2414
- ];
2415
- for (const endpoint of endpoints) {
2416
- try {
2417
- const response = await fetch(endpoint);
2418
- if (response.status < 500) {
2419
- console.log(pc.green(`OpenCode server ready on port ${port}`));
2420
- return true;
2421
- }
2422
- } catch (e) {}
2423
- }
2424
- } catch (e) {}
2425
- await new Promise((resolve) => setTimeout(resolve, 1000));
2426
- }
2427
- throw new Error(`Server did not start on port ${port} after ${maxAttempts} seconds`);
2428
- }
2429
- async function startOpencodeServer(port) {
2430
- console.log(pc.cyan(`Starting OpenCode server on port ${port}...`));
2431
- const serverProcess = spawn("opencode", ["serve", "--port", port.toString()], {
2432
- stdio: "pipe",
2433
- detached: false,
2434
- env: {
2435
- ...process.env,
2436
- OPENCODE_PORT: port.toString()
2437
- }
2438
- });
2439
- serverProcess.stdout?.on("data", (data) => {
2440
- console.log(pc.gray(`[OpenCode] ${data.toString().trim()}`));
2441
- });
2442
- serverProcess.stderr?.on("data", (data) => {
2443
- console.error(pc.yellow(`[OpenCode Error] ${data.toString().trim()}`));
2444
- });
2445
- serverProcess.on("error", (error) => {
2446
- console.error(pc.red("Failed to start OpenCode server:"), error);
2447
- });
2448
- await waitForServer(port);
2449
- return serverProcess;
2450
- }
2451
- async function selectModelProvider(providedModel) {
2452
- if (providedModel) {
2453
- return {
2454
- providerID: providedModel.providerId,
2455
- modelID: providedModel.modelId
2456
- };
2457
- }
2458
- const config = await readConfig();
2459
- if (config.preferredModel) {
2460
- return {
2461
- providerID: config.preferredModel.providerId,
2462
- modelID: config.preferredModel.modelId
2463
- };
2464
- }
2465
- throw new Error("No model specified and no preferred model set. Please call getModels to see available models and ask user to choose one. Do not spell model names exactly, give rough names like Sonnet or GPT5. Prefer latest versions");
2466
- }
2467
- async function getTools({
2468
- onMessageCompleted
2469
- } = {}) {
2470
- const port = await getOpenPort();
2471
- const serverProcess = await startOpencodeServer(port);
2472
- const client = createOpencodeClient({ baseUrl: `http://localhost:${port}` });
2473
- const markdownRenderer = new ShareMarkdown(client);
2474
- const providersResponse = await client.config.providers({});
2475
- const providers = providersResponse.data?.providers || [];
2476
- const config = await readConfig();
2477
- const preferredModel = config.preferredModel;
2478
- process.on("exit", () => {
2479
- if (serverProcess && !serverProcess.killed) {
2480
- serverProcess.kill("SIGTERM");
2481
- }
2482
- });
2483
- process.on("SIGINT", () => {
2484
- if (serverProcess && !serverProcess.killed) {
2485
- serverProcess.kill("SIGTERM");
2486
- }
2487
- process.exit(0);
2488
- });
2489
- process.on("SIGTERM", () => {
2490
- if (serverProcess && !serverProcess.killed) {
2491
- serverProcess.kill("SIGTERM");
2492
- }
2493
- process.exit(0);
2494
- });
2495
- const getSessionModel = async (sessionId) => {
2496
- const res = await client.session.messages({ path: { id: sessionId } });
2497
- const data = res.data;
2498
- if (!data || data.length === 0)
2499
- return;
2500
- for (let i = data.length - 1;i >= 0; i--) {
2501
- const info = data[i].info;
2502
- if (info.role === "assistant") {
2503
- const ai = info;
2504
- if (!ai.summary && ai.providerID && ai.modelID) {
2505
- return { providerID: ai.providerID, modelID: ai.modelID };
2506
- }
2507
- }
2508
- }
2509
- return;
2510
- };
2511
- const tools = {
2512
- submitMessage: tool({
2513
- description: "Submit a message to an existing chat session. Does not wait for the message to complete",
2514
- inputSchema: z2.object({
2515
- sessionId: z2.string().describe("The session ID to send message to"),
2516
- message: z2.string().describe("The message text to send")
2517
- }),
2518
- execute: async ({ sessionId, message }) => {
2519
- const sessionModel = await getSessionModel(sessionId);
2520
- client.session.prompt({
2521
- path: { id: sessionId },
2522
- body: {
2523
- parts: [{ type: "text", text: message }],
2524
- model: sessionModel
2525
- }
2526
- }).then(async (response) => {
2527
- const markdown = await markdownRenderer.generate({
2528
- sessionID: sessionId,
2529
- lastAssistantOnly: true
2530
- });
2531
- onMessageCompleted?.({
2532
- sessionId,
2533
- messageId: "",
2534
- data: response.data,
2535
- markdown
2536
- });
2537
- }).catch((error) => {
2538
- onMessageCompleted?.({
2539
- sessionId,
2540
- messageId: "",
2541
- error
2542
- });
2543
- });
2544
- return {
2545
- success: true,
2546
- sessionId,
2547
- directive: "Tell user that message has been sent successfully"
2548
- };
2549
- }
2550
- }),
2551
- createNewChat: tool({
2552
- description: "Start a new chat session with an initial message. Does not wait for the message to complete",
2553
- inputSchema: z2.object({
2554
- message: z2.string().describe("The initial message to start the chat with"),
2555
- title: z2.string().optional().describe("Optional title for the session"),
2556
- model: z2.object({
2557
- providerId: z2.string().describe('The provider ID (e.g., "anthropic", "openai")'),
2558
- modelId: z2.string().describe('The model ID (e.g., "claude-opus-4-20250514", "gpt-5")')
2559
- }).optional().describe("Optional model to use for this session")
2560
- }),
2561
- execute: async ({ message, title, model }) => {
2562
- if (!message.trim()) {
2563
- throw new Error(`message must be a non empty string`);
2564
- }
2565
- try {
2566
- const { providerID, modelID } = await selectModelProvider(model);
2567
- if (model) {
2568
- await updateConfig({
2569
- preferredModel: {
2570
- providerId: model.providerId,
2571
- modelId: model.modelId
2572
- }
2573
- });
2574
- }
2575
- const session = await client.session.create({
2576
- body: {
2577
- title: title || message.slice(0, 50)
2578
- }
2579
- });
2580
- if (!session.data) {
2581
- throw new Error("Failed to create session");
2582
- }
2583
- client.session.prompt({
2584
- path: { id: session.data.id },
2585
- body: {
2586
- parts: [{ type: "text", text: message }],
2587
- model: modelID && providerID ? { modelID, providerID } : undefined
2588
- }
2589
- }).then(async (response) => {
2590
- const markdown = await markdownRenderer.generate({
2591
- sessionID: session.data.id,
2592
- lastAssistantOnly: true
2593
- });
2594
- onMessageCompleted?.({
2595
- sessionId: session.data.id,
2596
- messageId: "",
2597
- data: response.data,
2598
- markdown
2599
- });
2600
- }).catch((error) => {
2601
- onMessageCompleted?.({
2602
- sessionId: session.data.id,
2603
- messageId: "",
2604
- error
2605
- });
2606
- });
2607
- return {
2608
- success: true,
2609
- sessionId: session.data.id,
2610
- title: session.data.title
2611
- };
2612
- } catch (error) {
2613
- return {
2614
- success: false,
2615
- error: error instanceof Error ? error.message : "Failed to create chat session"
2616
- };
2617
- }
2618
- }
2619
- }),
2620
- listChats: tool({
2621
- description: "Get a list of available chat sessions sorted by most recent",
2622
- inputSchema: z2.object({}),
2623
- execute: async () => {
2624
- console.log(`listing opencode sessions`);
2625
- const sessions = await client.session.list();
2626
- if (!sessions.data) {
2627
- return { success: false, error: "No sessions found" };
2628
- }
2629
- const sortedSessions = [...sessions.data].sort((a, b) => {
2630
- return b.time.updated - a.time.updated;
2631
- }).slice(0, 20);
2632
- const sessionList = sortedSessions.map(async (session) => {
2633
- const finishedAt = session.time.updated;
2634
- const status = await (async () => {
2635
- if (session.revert)
2636
- return "error";
2637
- const messagesResponse = await client.session.messages({
2638
- path: { id: session.id }
2639
- });
2640
- const messages = messagesResponse.data || [];
2641
- const lastMessage = messages[messages.length - 1];
2642
- if (lastMessage?.info.role === "assistant" && !lastMessage.info.time.completed) {
2643
- return "in_progress";
2644
- }
2645
- return "finished";
2646
- })();
2647
- return {
2648
- id: session.id,
2649
- folder: session.directory,
2650
- status,
2651
- finishedAt: formatDistanceToNow(new Date(finishedAt), {
2652
- addSuffix: true
2653
- }),
2654
- title: session.title,
2655
- prompt: session.title
2656
- };
2657
- });
2658
- const resolvedList = await Promise.all(sessionList);
2659
- return {
2660
- success: true,
2661
- sessions: resolvedList
2662
- };
2663
- }
2664
- }),
2665
- searchFiles: tool({
2666
- description: "Search for files in a folder",
2667
- inputSchema: z2.object({
2668
- folder: z2.string().optional().describe("The folder path to search in, optional. only use if user specifically asks for it"),
2669
- query: z2.string().describe("The search query for files")
2670
- }),
2671
- execute: async ({ folder, query }) => {
2672
- const results = await client.find.files({
2673
- query: {
2674
- query,
2675
- directory: folder
2676
- }
2677
- });
2678
- return {
2679
- success: true,
2680
- files: results.data || []
2681
- };
2682
- }
2683
- }),
2684
- readSessionMessages: tool({
2685
- description: "Read messages from a chat session",
2686
- inputSchema: z2.object({
2687
- sessionId: z2.string().describe("The session ID to read messages from"),
2688
- lastAssistantOnly: z2.boolean().optional().describe("Only read the last assistant message")
2689
- }),
2690
- execute: async ({ sessionId, lastAssistantOnly = false }) => {
2691
- if (lastAssistantOnly) {
2692
- const messages = await client.session.messages({
2693
- path: { id: sessionId }
2694
- });
2695
- if (!messages.data) {
2696
- return { success: false, error: "No messages found" };
2697
- }
2698
- const assistantMessages = messages.data.filter((m) => m.info.role === "assistant");
2699
- if (assistantMessages.length === 0) {
2700
- return {
2701
- success: false,
2702
- error: "No assistant messages found"
2703
- };
2704
- }
2705
- const lastMessage = assistantMessages[assistantMessages.length - 1];
2706
- const status = "completed" in lastMessage.info.time && lastMessage.info.time.completed ? "completed" : "in_progress";
2707
- const markdown = await markdownRenderer.generate({
2708
- sessionID: sessionId,
2709
- lastAssistantOnly: true
2710
- });
2711
- return {
2712
- success: true,
2713
- markdown,
2714
- status
2715
- };
2716
- } else {
2717
- const markdown = await markdownRenderer.generate({
2718
- sessionID: sessionId
2719
- });
2720
- const messages = await client.session.messages({
2721
- path: { id: sessionId }
2722
- });
2723
- const lastMessage = messages.data?.[messages.data.length - 1];
2724
- const status = lastMessage?.info.role === "assistant" && lastMessage?.info.time && "completed" in lastMessage.info.time && !lastMessage.info.time.completed ? "in_progress" : "completed";
2725
- return {
2726
- success: true,
2727
- markdown,
2728
- status
2729
- };
2730
- }
2731
- }
2732
- }),
2733
- abortChat: tool({
2734
- description: "Abort/stop an in-progress chat session",
2735
- inputSchema: z2.object({
2736
- sessionId: z2.string().describe("The session ID to abort")
2737
- }),
2738
- execute: async ({ sessionId }) => {
2739
- try {
2740
- const result = await client.session.abort({
2741
- path: { id: sessionId }
2742
- });
2743
- if (!result.data) {
2744
- return {
2745
- success: false,
2746
- error: "Failed to abort session"
2747
- };
2748
- }
2749
- return {
2750
- success: true,
2751
- sessionId,
2752
- message: "Session aborted successfully"
2753
- };
2754
- } catch (error) {
2755
- return {
2756
- success: false,
2757
- error: error instanceof Error ? error.message : "Unknown error occurred"
2758
- };
2759
- }
2760
- }
2761
- }),
2762
- getModels: tool({
2763
- description: "Get all available AI models from all providers",
2764
- inputSchema: z2.object({}),
2765
- execute: async () => {
2766
- try {
2767
- const providersResponse2 = await client.config.providers({});
2768
- const providers2 = providersResponse2.data?.providers || [];
2769
- const models = [];
2770
- providers2.forEach((provider) => {
2771
- if (provider.models && typeof provider.models === "object") {
2772
- Object.entries(provider.models).forEach(([modelId, model]) => {
2773
- models.push({
2774
- providerId: provider.id,
2775
- modelId
2776
- });
2777
- });
2778
- }
2779
- });
2780
- return {
2781
- success: true,
2782
- models,
2783
- totalCount: models.length
2784
- };
2785
- } catch (error) {
2786
- return {
2787
- success: false,
2788
- error: error instanceof Error ? error.message : "Failed to fetch models",
2789
- models: []
2790
- };
2791
- }
2792
- }
2793
- })
2794
- };
2795
- return {
2796
- tools,
2797
- providers,
2798
- preferredModel
2799
- };
2800
- }
2801
-
2802
- // src/cli.tsx
2803
- import { render, Box, Text, useStdout } from "ink";
2804
- import { useState, useEffect } from "react";
2805
- import { jsxDEV } from "react/jsx-dev-runtime";
2806
- var cli = cac("kimaki");
2807
- cli.help();
2808
- function AsciiVideoPlayer() {
2809
- const [frameIndex, setFrameIndex] = useState(0);
2810
- const [asciiFrame, setAsciiFrame] = useState("Loading frames...");
2811
- const [frames, setFrames] = useState([]);
2812
- const { stdout } = useStdout();
2813
- useEffect(() => {
2814
- const framesDir = path2.join(path2.dirname(new URL(import.meta.url).pathname), "../assets/frames");
2815
- try {
2816
- const files = fs3.readdirSync(framesDir).filter((f) => f.endsWith(".png")).sort();
2817
- setFrames(files.map((f) => path2.join(framesDir, f)));
2818
- } catch (error) {
2819
- setAsciiFrame("Error: Could not find frames directory");
2820
- }
2821
- }, []);
2822
- useEffect(() => {
2823
- if (frames.length === 0)
2824
- return;
2825
- const loadFrame = async () => {
2826
- const { convertImageToAscii: convertImageToAscii2 } = await Promise.resolve().then(() => (init_video_to_ascii(), exports_video_to_ascii));
2827
- const ascii = await convertImageToAscii2({
2828
- imagePath: frames[frameIndex],
2829
- cols: 80,
2830
- rows: 22,
2831
- colored: true,
2832
- keepAspectRatio: false
2833
- });
2834
- setAsciiFrame(ascii);
2835
- };
2836
- loadFrame();
2837
- const timer = setTimeout(() => {
2838
- setFrameIndex((prev) => (prev + 1) % frames.length);
2839
- }, 50);
2840
- return () => clearTimeout(timer);
2841
- }, [frameIndex, frames]);
2842
- return /* @__PURE__ */ jsxDEV(Box, {
2843
- flexDirection: "column",
2844
- children: [
2845
- /* @__PURE__ */ jsxDEV(Text, {
2846
- children: asciiFrame
2847
- }, undefined, false, undefined, this),
2848
- /* @__PURE__ */ jsxDEV(Text, {
2849
- dimColor: true,
2850
- children: [
2851
- "Press Ctrl+C to exit • Frame ",
2852
- frameIndex + 1,
2853
- "/",
2854
- frames.length
2855
- ]
2856
- }, undefined, true, undefined, this)
2857
- ]
2858
- }, undefined, true, undefined, this);
2859
- }
2860
- function KimakiTUI({
2861
- messages,
2862
- isConnected
2863
- }) {
2864
- const formatMessage = (msg) => {
2865
- const data = msg.data;
2866
- if (data.serverContent?.modelTurn?.parts?.[0]?.text) {
2867
- return `\uD83E\uDD16 ${data.serverContent.modelTurn.parts[0].text.slice(0, 80)}...`;
2868
- }
2869
- if (data.serverContent?.turnComplete) {
2870
- return "✅ Turn completed";
2871
- }
2872
- if (data.toolCall) {
2873
- return `\uD83D\uDD27 Tool: ${data.toolCall.name}`;
2874
- }
2875
- if (data.toolResponse) {
2876
- return `\uD83D\uDCE6 Tool response received`;
2877
- }
2878
- if (data.interrupted) {
2879
- return "⚠️ Interrupted";
2880
- }
2881
- if (data.setupComplete) {
2882
- return "\uD83D\uDE80 Setup complete";
2883
- }
2884
- return JSON.stringify(data).slice(0, 80);
2885
- };
2886
- return /* @__PURE__ */ jsxDEV(Box, {
2887
- flexDirection: "column",
2888
- paddingX: 1,
2889
- children: [
2890
- /* @__PURE__ */ jsxDEV(Box, {
2891
- borderStyle: "single",
2892
- paddingX: 1,
2893
- marginBottom: 1,
2894
- children: /* @__PURE__ */ jsxDEV(Text, {
2895
- color: isConnected ? "green" : "yellow",
2896
- children: [
2897
- "Kimaki Voice Assistant -",
2898
- " ",
2899
- isConnected ? "● Connected to Gemini Live API" : "⏳ Connecting..."
2900
- ]
2901
- }, undefined, true, undefined, this)
2902
- }, undefined, false, undefined, this),
2903
- /* @__PURE__ */ jsxDEV(Box, {
2904
- borderStyle: "single",
2905
- paddingX: 1,
2906
- flexDirection: "column",
2907
- children: [
2908
- /* @__PURE__ */ jsxDEV(Text, {
2909
- color: "white",
2910
- bold: true,
2911
- children: "Debug Messages:"
2912
- }, undefined, false, undefined, this),
2913
- /* @__PURE__ */ jsxDEV(Box, {
2914
- flexDirection: "column",
2915
- marginTop: 1,
2916
- children: messages.length === 0 ? /* @__PURE__ */ jsxDEV(Text, {
2917
- dimColor: true,
2918
- children: "No messages yet..."
2919
- }, undefined, false, undefined, this) : messages.slice(-15).map((msg, index) => /* @__PURE__ */ jsxDEV(Box, {
2920
- marginBottom: 0,
2921
- children: [
2922
- /* @__PURE__ */ jsxDEV(Text, {
2923
- color: "cyan",
2924
- children: [
2925
- "[",
2926
- new Date(msg.timestamp).toLocaleTimeString(),
2927
- "]",
2928
- " "
2929
- ]
2930
- }, undefined, true, undefined, this),
2931
- /* @__PURE__ */ jsxDEV(Text, {
2932
- color: "white",
2933
- children: formatMessage(msg)
2934
- }, undefined, false, undefined, this)
2935
- ]
2936
- }, index, true, undefined, this))
2937
- }, undefined, false, undefined, this)
2938
- ]
2939
- }, undefined, true, undefined, this)
2940
- ]
2941
- }, undefined, true, undefined, this);
2942
- }
2943
- cli.command("", "Spawn Kimaki to orchestrate code agents").action(async (options) => {
2944
- try {
2945
- const token = process.env.GEMINI_API_KEY;
2946
- Object.assign(globalThis, webAudioApi);
2947
- navigator.mediaDevices = mediaDevices;
2948
- const { LiveAPIClient: LiveAPIClient2, callableToolsFromObject: callableToolsFromObject2 } = await Promise.resolve().then(() => (init_src(), exports_src));
2949
- let liveApiClient = null;
2950
- let audioChunks = [];
2951
- let isModelSpeaking = false;
2952
- let debugMessages = [];
2953
- let isConnected = false;
2954
- let updateUI = null;
2955
- const { tools, providers, preferredModel } = await getTools({
2956
- onMessageCompleted: (params) => {
2957
- if (!liveApiClient)
2958
- return;
2959
- const text = params.error ? `<systemMessage>
2960
- Chat message failed for session ${params.sessionId}. Error: ${params.error?.message || String(params.error)}
2961
- </systemMessage>` : `<systemMessage>
2962
- Chat message completed for session ${params.sessionId}.
2963
-
2964
- Assistant response:
2965
- ${params.markdown}
2966
- </systemMessage>`;
2967
- liveApiClient.sendText(text);
2968
- }
2969
- });
2970
- const saveUserAudio = async () => {
2971
- console.log("saveUserAudio", audioChunks.length);
2972
- if (audioChunks.length === 0)
2973
- return;
2974
- try {
2975
- const timestamp = Date.now();
2976
- const dir = "useraudio";
2977
- if (!fs3.existsSync(dir)) {
2978
- fs3.mkdirSync(dir, { recursive: true });
2979
- }
2980
- const totalLength = audioChunks.reduce((acc, chunk) => acc + chunk.byteLength, 0);
2981
- const combinedBuffer = new ArrayBuffer(totalLength);
2982
- const combinedView = new Int16Array(combinedBuffer);
2983
- let offset = 0;
2984
- for (const chunk of audioChunks) {
2985
- const chunkView = new Int16Array(chunk);
2986
- combinedView.set(chunkView, offset);
2987
- offset += chunkView.length;
2988
- }
2989
- const wav = new WaveFile.WaveFile;
2990
- wav.fromScratch(1, 16000, "16", Array.from(combinedView));
2991
- const filename = path2.join(dir, `${timestamp}.wav`);
2992
- await fs3.promises.writeFile(filename, Buffer.from(wav.toBuffer()));
2993
- console.log(`Saved user audio to ${filename} (${audioChunks.length} chunks)`);
2994
- audioChunks = [];
2995
- } catch (error) {
2996
- console.error("Failed to save audio file:", error);
2997
- }
2998
- };
2999
- const model = "models/gemini-2.5-flash-live-preview";
3000
- const newClient = new LiveAPIClient2({
3001
- apiKey: token,
3002
- model,
3003
- recordingSampleRate: 44100,
3004
- autoMuteOnAssistantSpeaking: false,
3005
- onUserAudioChunk: (chunk) => {
3006
- if (!isModelSpeaking) {
3007
- audioChunks.push(chunk);
3008
- }
3009
- },
3010
- onMessage: (message) => {
3011
- debugMessages.push({
3012
- timestamp: Date.now(),
3013
- data: message
3014
- });
3015
- if (debugMessages.length > 50) {
3016
- debugMessages = debugMessages.slice(-50);
3017
- }
3018
- if (updateUI) {
3019
- updateUI([...debugMessages], isConnected);
3020
- }
3021
- process.env.DEBUG && console.log(message);
3022
- if (message.serverContent?.turnComplete && audioChunks.length > 0) {
3023
- isModelSpeaking = true;
3024
- process.env.DEBUG && saveUserAudio();
3025
- }
3026
- if (message.serverContent?.turnComplete) {
3027
- isModelSpeaking = false;
3028
- }
3029
- },
3030
- config: {
3031
- tools: callableToolsFromObject2(tools),
3032
- responseModalities: [Modality2.AUDIO],
3033
- speechConfig: {
3034
- voiceConfig: {
3035
- prebuiltVoiceConfig: {
3036
- voiceName: "Charon"
3037
- }
3038
- }
3039
- },
3040
- mediaResolution: MediaResolution2.MEDIA_RESOLUTION_MEDIUM,
3041
- contextWindowCompression: {
3042
- triggerTokens: "25600",
3043
- slidingWindow: { targetTokens: "12800" }
3044
- },
3045
- systemInstruction: {
3046
- parts: [
3047
- {
3048
- text: dedent`
3049
- You are Kimaki, an AI similar to Jarvis: you help your user (an engineer) controlling his coding agent, just like Jarvis controls Ironman armor and machines. Speak fast.
3050
-
3051
- You should talk like Jarvis, British accent, satirical, joking and calm. Be short and concise. Speak fast.
3052
-
3053
- After tool calls give a super short summary of the assistant message, you should say what the assistant message writes.
3054
-
3055
- Before starting a new session ask for confirmation if it is not clear if the user finished describing it. ask "message ready, send?"
3056
-
3057
- NEVER repeat the whole tool call parameters or message.
3058
-
3059
- Your job is to manage many opencode agent chat instances. Opencode is the agent used to write the code, it is similar to Claude Code.
3060
-
3061
- For everything the user asks it is implicit that the user is asking for you to proxy the requests to opencode sessions.
3062
-
3063
- You can
3064
- - start new chats on a given project
3065
- - read the chats to report progress to the user
3066
- - submit messages to the chat
3067
- - list files for a given projects, so you can translate imprecise user prompts to precise messages that mention filename paths using @
3068
-
3069
- Common patterns
3070
- - to get the last session use the listChats tool
3071
- - when user asks you to do something you submit a new session to do it. it's implicit that you proxy requests to the agents chat!
3072
- - when you submit a session assume the session will take a minute or 2 to complete the task
3073
-
3074
- Rules
3075
- - never spell files by mentioning dots, letters, etc. instead give a brief description of the filename
3076
- - NEVER spell hashes or IDs
3077
- - never read session ids or other ids
3078
-
3079
- Your voice is calm and monotone, NEVER excited and goofy. But you speak without jargon or bs and do veiled short jokes.
3080
- You speak like you knew something other don't. You are cool and cold.
3081
- `
3082
- }
3083
- ]
3084
- }
3085
- },
3086
- onStateChange: (state) => {}
3087
- });
3088
- liveApiClient = newClient;
3089
- const connected = await newClient.connect();
3090
- isConnected = true;
3091
- await new Promise((res) => setTimeout(res, 500));
3092
- const App = () => {
3093
- const [messages, setMessages] = useState([]);
3094
- const [connectionStatus, setConnectionStatus] = useState(isConnected);
3095
- useEffect(() => {
3096
- updateUI = (msgs, connected2) => {
3097
- setMessages(msgs);
3098
- setConnectionStatus(connected2);
3099
- };
3100
- setMessages([...debugMessages]);
3101
- setConnectionStatus(isConnected);
3102
- return () => {
3103
- updateUI = null;
3104
- };
3105
- }, []);
3106
- return /* @__PURE__ */ jsxDEV(KimakiTUI, {
3107
- messages,
3108
- isConnected: connectionStatus
3109
- }, undefined, false, undefined, this);
3110
- };
3111
- render(/* @__PURE__ */ jsxDEV(App, {}, undefined, false, undefined, this));
3112
- } catch (error) {
3113
- console.error(pc2.red(`
3114
- Error initializing project:`));
3115
- console.error(pc2.red(error));
3116
- process.exit(1);
3117
- }
3118
- });
3119
- cli.command("ascii", "Play ASCII video").action(async () => {
3120
- render(/* @__PURE__ */ jsxDEV(AsciiVideoPlayer, {}, undefined, false, undefined, this));
3121
- });
3122
-
3123
- // src/bin.ts
3124
- cli.parse();