@wirecat/cli-core 0.480.0 → 0.481.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/dist/cli/messenger/hearing-command.js +2 -2
- package/dist/cli/messenger/store-maintenance-command.d.ts +1 -1
- package/dist/meetings/mcp/index.d.ts +15 -15
- package/dist/speech/decode.d.ts +21 -0
- package/dist/speech/decode.js +164 -0
- package/dist/speech/hearing.d.ts +2 -0
- package/dist/speech/hearing.js +8 -6
- package/dist/speech/transcribe.d.ts +8 -1
- package/dist/speech/transcribe.js +9 -6
- package/dist/store/sqlite/attachment-texts.js +1 -1
- package/dist/store/sqlite/transcripts.js +1 -1
- package/package.json +4 -1
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,21 @@ Notable changes to `@wirecat/cli-core` (`@wirecat/cli-messaging` up to 0.261.0,
|
|
|
5
5
|
minor version may break callers, and says how under "Changed — may break callers". `pnpm docs:check` checks the
|
|
6
6
|
shape of this file.
|
|
7
7
|
|
|
8
|
+
## Unreleased
|
|
9
|
+
|
|
10
|
+
## 0.481.0 — 29.10.2026
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **`messages transcribe` hears an audio file someone sent, not only a voice note** — an iPhone recording reaches
|
|
15
|
+
WhatsApp as an `audio/mp4` file. MP3, FLAC and Ogg Vorbis are read by small bundled decoders (`mpg123-decoder`,
|
|
16
|
+
`@wasm-audio-decoders/flac`, `@wasm-audio-decoders/ogg-vorbis`, MIT). AAC/M4A, other formats and a file a bundled
|
|
17
|
+
decoder fails on go to the machine's own `ffmpeg`, when it has one, limited to audio formats and local files. No
|
|
18
|
+
decoder for AAC is bundled: the small ones are GPL. Audio longer than an hour is refused. A transcript kept for an
|
|
19
|
+
audio file shows in lists and is found by `transcript:`; `--transcribe` on a list still hears only voice notes, so it
|
|
20
|
+
never transcribes music. Where the messenger is asked first (`auto`), the local model takes over only when the
|
|
21
|
+
messenger refuses the account, so a messenger that rejects an audio file for another reason needs `--local`.
|
|
22
|
+
|
|
8
23
|
## 0.480.0 — 29.10.2026
|
|
9
24
|
|
|
10
25
|
### Changed — may break callers
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { hasAudio, hearVoices, openKept, spoken, withTranscript } from "../../speech/hearing.js";
|
|
2
2
|
import { choose } from "../../speech/transcribe.js";
|
|
3
3
|
export const TRANSCRIBE_OPTION = [
|
|
4
4
|
"--transcribe",
|
|
@@ -13,7 +13,7 @@ export const MODEL_OPTION = [
|
|
|
13
13
|
* supplied. It is released before local recognition. `undefined` when there is no voice message, and then no file is opened.
|
|
14
14
|
*/
|
|
15
15
|
export const hearForCommand = async (context, messenger, messages, transcribe, model, connect) => {
|
|
16
|
-
if (!messages.some(
|
|
16
|
+
if (!messages.some(hasAudio))
|
|
17
17
|
return undefined;
|
|
18
18
|
const offline = context.settings.offline;
|
|
19
19
|
if (transcribe && offline)
|
|
@@ -3,9 +3,9 @@ import type { CacheDatabase } from "../../store/driver.js";
|
|
|
3
3
|
import type { Messenger } from "./context.js";
|
|
4
4
|
/** `doctor`'s short summary of the store. Must answer for a broken file, so an error is a field. */
|
|
5
5
|
export declare const storeSummary: (env: NodeJS.ProcessEnv) => Promise<{
|
|
6
|
+
error?: undefined;
|
|
6
7
|
path: string;
|
|
7
8
|
exists: boolean;
|
|
8
|
-
error?: undefined;
|
|
9
9
|
} | {
|
|
10
10
|
path: string;
|
|
11
11
|
exists: boolean;
|
|
@@ -38,7 +38,6 @@ export declare const meetingTools: ({
|
|
|
38
38
|
max_read_bytes?: undefined;
|
|
39
39
|
cursor?: undefined;
|
|
40
40
|
} | {
|
|
41
|
-
limit?: undefined;
|
|
42
41
|
event_id?: undefined;
|
|
43
42
|
series_id?: undefined;
|
|
44
43
|
page?: undefined;
|
|
@@ -54,11 +53,11 @@ export declare const meetingTools: ({
|
|
|
54
53
|
format?: undefined;
|
|
55
54
|
transcript_id?: undefined;
|
|
56
55
|
query?: undefined;
|
|
56
|
+
limit?: undefined;
|
|
57
57
|
scan_limit?: undefined;
|
|
58
58
|
max_read_bytes?: undefined;
|
|
59
59
|
cursor?: undefined;
|
|
60
60
|
} | {
|
|
61
|
-
limit?: undefined;
|
|
62
61
|
event_id?: undefined;
|
|
63
62
|
series_id?: undefined;
|
|
64
63
|
page?: undefined;
|
|
@@ -76,11 +75,11 @@ export declare const meetingTools: ({
|
|
|
76
75
|
format?: undefined;
|
|
77
76
|
transcript_id?: undefined;
|
|
78
77
|
query?: undefined;
|
|
78
|
+
limit?: undefined;
|
|
79
79
|
scan_limit?: undefined;
|
|
80
80
|
max_read_bytes?: undefined;
|
|
81
81
|
cursor?: undefined;
|
|
82
82
|
} | {
|
|
83
|
-
limit?: undefined;
|
|
84
83
|
event_id?: undefined;
|
|
85
84
|
series_id?: undefined;
|
|
86
85
|
page?: undefined;
|
|
@@ -102,6 +101,7 @@ export declare const meetingTools: ({
|
|
|
102
101
|
minimum: number;
|
|
103
102
|
};
|
|
104
103
|
query?: undefined;
|
|
104
|
+
limit?: undefined;
|
|
105
105
|
scan_limit?: undefined;
|
|
106
106
|
max_read_bytes?: undefined;
|
|
107
107
|
cursor?: undefined;
|
|
@@ -188,7 +188,6 @@ export declare const meetingTools: ({
|
|
|
188
188
|
max_read_bytes?: undefined;
|
|
189
189
|
cursor?: undefined;
|
|
190
190
|
} | {
|
|
191
|
-
limit?: undefined;
|
|
192
191
|
event_id?: undefined;
|
|
193
192
|
series_id?: undefined;
|
|
194
193
|
page?: undefined;
|
|
@@ -204,11 +203,11 @@ export declare const meetingTools: ({
|
|
|
204
203
|
format?: undefined;
|
|
205
204
|
transcript_id?: undefined;
|
|
206
205
|
query?: undefined;
|
|
206
|
+
limit?: undefined;
|
|
207
207
|
scan_limit?: undefined;
|
|
208
208
|
max_read_bytes?: undefined;
|
|
209
209
|
cursor?: undefined;
|
|
210
210
|
} | {
|
|
211
|
-
limit?: undefined;
|
|
212
211
|
event_id?: undefined;
|
|
213
212
|
series_id?: undefined;
|
|
214
213
|
page?: undefined;
|
|
@@ -226,11 +225,11 @@ export declare const meetingTools: ({
|
|
|
226
225
|
format?: undefined;
|
|
227
226
|
transcript_id?: undefined;
|
|
228
227
|
query?: undefined;
|
|
228
|
+
limit?: undefined;
|
|
229
229
|
scan_limit?: undefined;
|
|
230
230
|
max_read_bytes?: undefined;
|
|
231
231
|
cursor?: undefined;
|
|
232
232
|
} | {
|
|
233
|
-
limit?: undefined;
|
|
234
233
|
event_id?: undefined;
|
|
235
234
|
series_id?: undefined;
|
|
236
235
|
page?: undefined;
|
|
@@ -252,6 +251,7 @@ export declare const meetingTools: ({
|
|
|
252
251
|
minimum: number;
|
|
253
252
|
};
|
|
254
253
|
query?: undefined;
|
|
254
|
+
limit?: undefined;
|
|
255
255
|
scan_limit?: undefined;
|
|
256
256
|
max_read_bytes?: undefined;
|
|
257
257
|
cursor?: undefined;
|
|
@@ -338,7 +338,6 @@ export declare const meetingTools: ({
|
|
|
338
338
|
max_read_bytes?: undefined;
|
|
339
339
|
cursor?: undefined;
|
|
340
340
|
} | {
|
|
341
|
-
limit?: undefined;
|
|
342
341
|
event_id?: undefined;
|
|
343
342
|
series_id?: undefined;
|
|
344
343
|
page?: undefined;
|
|
@@ -354,11 +353,11 @@ export declare const meetingTools: ({
|
|
|
354
353
|
format?: undefined;
|
|
355
354
|
transcript_id?: undefined;
|
|
356
355
|
query?: undefined;
|
|
356
|
+
limit?: undefined;
|
|
357
357
|
scan_limit?: undefined;
|
|
358
358
|
max_read_bytes?: undefined;
|
|
359
359
|
cursor?: undefined;
|
|
360
360
|
} | {
|
|
361
|
-
limit?: undefined;
|
|
362
361
|
event_id?: undefined;
|
|
363
362
|
series_id?: undefined;
|
|
364
363
|
page?: undefined;
|
|
@@ -376,11 +375,11 @@ export declare const meetingTools: ({
|
|
|
376
375
|
format?: undefined;
|
|
377
376
|
transcript_id?: undefined;
|
|
378
377
|
query?: undefined;
|
|
378
|
+
limit?: undefined;
|
|
379
379
|
scan_limit?: undefined;
|
|
380
380
|
max_read_bytes?: undefined;
|
|
381
381
|
cursor?: undefined;
|
|
382
382
|
} | {
|
|
383
|
-
limit?: undefined;
|
|
384
383
|
event_id?: undefined;
|
|
385
384
|
series_id?: undefined;
|
|
386
385
|
page?: undefined;
|
|
@@ -402,6 +401,7 @@ export declare const meetingTools: ({
|
|
|
402
401
|
minimum: number;
|
|
403
402
|
};
|
|
404
403
|
query?: undefined;
|
|
404
|
+
limit?: undefined;
|
|
405
405
|
scan_limit?: undefined;
|
|
406
406
|
max_read_bytes?: undefined;
|
|
407
407
|
cursor?: undefined;
|
|
@@ -488,7 +488,6 @@ export declare const meetingTools: ({
|
|
|
488
488
|
max_read_bytes?: undefined;
|
|
489
489
|
cursor?: undefined;
|
|
490
490
|
} | {
|
|
491
|
-
limit?: undefined;
|
|
492
491
|
event_id?: undefined;
|
|
493
492
|
series_id?: undefined;
|
|
494
493
|
page?: undefined;
|
|
@@ -504,11 +503,11 @@ export declare const meetingTools: ({
|
|
|
504
503
|
format?: undefined;
|
|
505
504
|
transcript_id?: undefined;
|
|
506
505
|
query?: undefined;
|
|
506
|
+
limit?: undefined;
|
|
507
507
|
scan_limit?: undefined;
|
|
508
508
|
max_read_bytes?: undefined;
|
|
509
509
|
cursor?: undefined;
|
|
510
510
|
} | {
|
|
511
|
-
limit?: undefined;
|
|
512
511
|
event_id?: undefined;
|
|
513
512
|
series_id?: undefined;
|
|
514
513
|
page?: undefined;
|
|
@@ -526,11 +525,11 @@ export declare const meetingTools: ({
|
|
|
526
525
|
format?: undefined;
|
|
527
526
|
transcript_id?: undefined;
|
|
528
527
|
query?: undefined;
|
|
528
|
+
limit?: undefined;
|
|
529
529
|
scan_limit?: undefined;
|
|
530
530
|
max_read_bytes?: undefined;
|
|
531
531
|
cursor?: undefined;
|
|
532
532
|
} | {
|
|
533
|
-
limit?: undefined;
|
|
534
533
|
event_id?: undefined;
|
|
535
534
|
series_id?: undefined;
|
|
536
535
|
page?: undefined;
|
|
@@ -552,6 +551,7 @@ export declare const meetingTools: ({
|
|
|
552
551
|
minimum: number;
|
|
553
552
|
};
|
|
554
553
|
query?: undefined;
|
|
554
|
+
limit?: undefined;
|
|
555
555
|
scan_limit?: undefined;
|
|
556
556
|
max_read_bytes?: undefined;
|
|
557
557
|
cursor?: undefined;
|
|
@@ -638,7 +638,6 @@ export declare const meetingTools: ({
|
|
|
638
638
|
max_read_bytes?: undefined;
|
|
639
639
|
cursor?: undefined;
|
|
640
640
|
} | {
|
|
641
|
-
limit?: undefined;
|
|
642
641
|
event_id?: undefined;
|
|
643
642
|
series_id?: undefined;
|
|
644
643
|
page?: undefined;
|
|
@@ -654,11 +653,11 @@ export declare const meetingTools: ({
|
|
|
654
653
|
format?: undefined;
|
|
655
654
|
transcript_id?: undefined;
|
|
656
655
|
query?: undefined;
|
|
656
|
+
limit?: undefined;
|
|
657
657
|
scan_limit?: undefined;
|
|
658
658
|
max_read_bytes?: undefined;
|
|
659
659
|
cursor?: undefined;
|
|
660
660
|
} | {
|
|
661
|
-
limit?: undefined;
|
|
662
661
|
event_id?: undefined;
|
|
663
662
|
series_id?: undefined;
|
|
664
663
|
page?: undefined;
|
|
@@ -676,11 +675,11 @@ export declare const meetingTools: ({
|
|
|
676
675
|
format?: undefined;
|
|
677
676
|
transcript_id?: undefined;
|
|
678
677
|
query?: undefined;
|
|
678
|
+
limit?: undefined;
|
|
679
679
|
scan_limit?: undefined;
|
|
680
680
|
max_read_bytes?: undefined;
|
|
681
681
|
cursor?: undefined;
|
|
682
682
|
} | {
|
|
683
|
-
limit?: undefined;
|
|
684
683
|
event_id?: undefined;
|
|
685
684
|
series_id?: undefined;
|
|
686
685
|
page?: undefined;
|
|
@@ -702,6 +701,7 @@ export declare const meetingTools: ({
|
|
|
702
701
|
minimum: number;
|
|
703
702
|
};
|
|
704
703
|
query?: undefined;
|
|
704
|
+
limit?: undefined;
|
|
705
705
|
scan_limit?: undefined;
|
|
706
706
|
max_read_bytes?: undefined;
|
|
707
707
|
cursor?: undefined;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface Pcm {
|
|
2
|
+
samples: Float32Array;
|
|
3
|
+
rate: number;
|
|
4
|
+
}
|
|
5
|
+
/** An hour of speech; a longer "voice message" is a podcast or a song, and its samples would not fit in memory. */
|
|
6
|
+
export declare const LONGEST_AUDIO_SECONDS = 3600;
|
|
7
|
+
export type Ffmpeg = (bytes: Uint8Array) => Promise<Pcm>;
|
|
8
|
+
type Format = "opus" | "vorbis" | "flac" | "mp3" | "other";
|
|
9
|
+
/** From the bytes, never the MIME type: WhatsApp calls an iPhone recording `audio/mp4` and a voice note `audio/ogg`. */
|
|
10
|
+
export declare const formatOf: (bytes: Uint8Array) => Format;
|
|
11
|
+
/**
|
|
12
|
+
* The owner's own ffmpeg, for what no bundled decoder reads — AAC above all, which every small decoder does under
|
|
13
|
+
* the GPL. From a file rather than a pipe: an MP4 whose index sits at the end cannot be read from a pipe.
|
|
14
|
+
*/
|
|
15
|
+
export declare const runFfmpeg: Ffmpeg;
|
|
16
|
+
/**
|
|
17
|
+
* Opus, Vorbis, FLAC and MP3 by the bundled decoders; anything else, or what one of them could not read, by ffmpeg
|
|
18
|
+
* when the machine has it. The bundled decoder's error wins: it names the file's own problem.
|
|
19
|
+
*/
|
|
20
|
+
export declare const decodeAudio: (bytes: Uint8Array, ffmpeg?: Ffmpeg) => Promise<Pcm>;
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { createRequire } from "node:module";
|
|
4
|
+
import { tmpdir } from "node:os";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { pathToFileURL } from "node:url";
|
|
7
|
+
import { CliError } from "@wirecat/cli-base";
|
|
8
|
+
import { decodeOgg, SAMPLE_RATE } from "./recognize.js";
|
|
9
|
+
/** An hour of speech; a longer "voice message" is a podcast or a song, and its samples would not fit in memory. */
|
|
10
|
+
export const LONGEST_AUDIO_SECONDS = 3600;
|
|
11
|
+
const FFMPEG_TIMEOUT_MS = 120_000;
|
|
12
|
+
/**
|
|
13
|
+
* Formats a sender's file may claim to be. A playlist (HLS, concat) would make ffmpeg open other files and URLs,
|
|
14
|
+
* so it never gets past this list or the file-only protocol list.
|
|
15
|
+
*/
|
|
16
|
+
const FFMPEG_FORMATS = "mov,mp4,m4a,3gp,3g2,mj2,aac,mp3,wav,amr,caf,aiff,flac,ogg,matroska,webm,asf";
|
|
17
|
+
const text = (bytes, from, length) => String.fromCharCode(...bytes.subarray(from, from + length));
|
|
18
|
+
const contains = (bytes, needle, within) => Buffer.from(bytes.buffer, bytes.byteOffset, Math.min(bytes.length, within)).includes(needle, 0, "latin1");
|
|
19
|
+
/** From the bytes, never the MIME type: WhatsApp calls an iPhone recording `audio/mp4` and a voice note `audio/ogg`. */
|
|
20
|
+
export const formatOf = (bytes) => {
|
|
21
|
+
if (text(bytes, 0, 4) === "OggS") {
|
|
22
|
+
if (contains(bytes, "OpusHead", 128))
|
|
23
|
+
return "opus";
|
|
24
|
+
if (contains(bytes, "\x01vorbis", 128))
|
|
25
|
+
return "vorbis";
|
|
26
|
+
return "other";
|
|
27
|
+
}
|
|
28
|
+
if (text(bytes, 0, 4) === "fLaC")
|
|
29
|
+
return "flac";
|
|
30
|
+
// ADTS AAC shares MPEG's frame sync; its layer bits are 00, which MPEG audio reserves.
|
|
31
|
+
const frame = bytes[0] === 0xff && ((bytes[1] ?? 0) & 0xe0) === 0xe0 && (((bytes[1] ?? 0) >> 1) & 3) !== 0;
|
|
32
|
+
if (text(bytes, 0, 3) === "ID3" || frame)
|
|
33
|
+
return "mp3";
|
|
34
|
+
return "other";
|
|
35
|
+
};
|
|
36
|
+
const firstChannel = (decoded) => {
|
|
37
|
+
const samples = decoded.channelData[0];
|
|
38
|
+
if (!samples || decoded.samplesDecoded === 0)
|
|
39
|
+
throw new CliError("validation_error", "no audio in that file");
|
|
40
|
+
return { samples, rate: decoded.sampleRate };
|
|
41
|
+
};
|
|
42
|
+
/**
|
|
43
|
+
* Found from cli-core's own place and imported by URL, so a CLI's bundler leaves the decoder whole: bundled, it looks
|
|
44
|
+
* for its worker file beside the bundle, where there is none.
|
|
45
|
+
*/
|
|
46
|
+
const load = (name) => import(pathToFileURL(createRequire(import.meta.url).resolve(name)).href);
|
|
47
|
+
const decodeVorbis = async (bytes) => {
|
|
48
|
+
const { OggVorbisDecoder } = await load("@wasm-audio-decoders/ogg-vorbis");
|
|
49
|
+
const decoder = new OggVorbisDecoder();
|
|
50
|
+
await decoder.ready;
|
|
51
|
+
try {
|
|
52
|
+
return firstChannel(await decoder.decodeFile(bytes));
|
|
53
|
+
}
|
|
54
|
+
finally {
|
|
55
|
+
decoder.free();
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
const decodeFlac = async (bytes) => {
|
|
59
|
+
const { FLACDecoder } = await load("@wasm-audio-decoders/flac");
|
|
60
|
+
const decoder = new FLACDecoder();
|
|
61
|
+
await decoder.ready;
|
|
62
|
+
try {
|
|
63
|
+
return firstChannel(await decoder.decodeFile(bytes));
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
decoder.free();
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
const decodeMp3 = async (bytes) => {
|
|
70
|
+
const { MPEGDecoder } = await load("mpg123-decoder");
|
|
71
|
+
const decoder = new MPEGDecoder();
|
|
72
|
+
await decoder.ready;
|
|
73
|
+
try {
|
|
74
|
+
return firstChannel(decoder.decode(bytes));
|
|
75
|
+
}
|
|
76
|
+
finally {
|
|
77
|
+
decoder.free();
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
/**
|
|
81
|
+
* The owner's own ffmpeg, for what no bundled decoder reads — AAC above all, which every small decoder does under
|
|
82
|
+
* the GPL. From a file rather than a pipe: an MP4 whose index sits at the end cannot be read from a pipe.
|
|
83
|
+
*/
|
|
84
|
+
export const runFfmpeg = async (bytes) => {
|
|
85
|
+
const directory = await mkdtemp(join(tmpdir(), "wirecat-audio-"));
|
|
86
|
+
try {
|
|
87
|
+
const input = join(directory, "audio");
|
|
88
|
+
await writeFile(input, bytes, { mode: 0o600 });
|
|
89
|
+
const largest = LONGEST_AUDIO_SECONDS * SAMPLE_RATE * 4;
|
|
90
|
+
const output = await new Promise((resolve, reject) => {
|
|
91
|
+
const child = spawn("ffmpeg", [
|
|
92
|
+
"-nostdin",
|
|
93
|
+
"-hide_banner",
|
|
94
|
+
"-loglevel",
|
|
95
|
+
"error",
|
|
96
|
+
"-protocol_whitelist",
|
|
97
|
+
"file",
|
|
98
|
+
"-format_whitelist",
|
|
99
|
+
FFMPEG_FORMATS,
|
|
100
|
+
"-i",
|
|
101
|
+
input,
|
|
102
|
+
"-vn",
|
|
103
|
+
"-ac",
|
|
104
|
+
"1",
|
|
105
|
+
"-ar",
|
|
106
|
+
String(SAMPLE_RATE),
|
|
107
|
+
"-f",
|
|
108
|
+
"f32le",
|
|
109
|
+
"pipe:1",
|
|
110
|
+
], { stdio: ["ignore", "pipe", "pipe"], timeout: FFMPEG_TIMEOUT_MS });
|
|
111
|
+
const chunks = [];
|
|
112
|
+
let size = 0;
|
|
113
|
+
let errors = "";
|
|
114
|
+
child.stdout.on("data", (chunk) => {
|
|
115
|
+
size += chunk.length;
|
|
116
|
+
if (size > largest)
|
|
117
|
+
child.kill();
|
|
118
|
+
else
|
|
119
|
+
chunks.push(chunk);
|
|
120
|
+
});
|
|
121
|
+
child.stderr.on("data", (chunk) => {
|
|
122
|
+
errors = (errors + chunk.toString()).slice(-500);
|
|
123
|
+
});
|
|
124
|
+
child.on("error", (error) => reject(error.code === "ENOENT"
|
|
125
|
+
? new CliError("validation_error", "only ffmpeg reads this audio format, and ffmpeg is not installed")
|
|
126
|
+
: error));
|
|
127
|
+
child.on("close", (code) => {
|
|
128
|
+
if (size > largest)
|
|
129
|
+
reject(new CliError("validation_error", `the audio is longer than ${LONGEST_AUDIO_SECONDS / 60} minutes`));
|
|
130
|
+
else if (code === 0)
|
|
131
|
+
resolve(Buffer.concat(chunks));
|
|
132
|
+
else
|
|
133
|
+
reject(new CliError("validation_error", `ffmpeg could not read the audio: ${errors.trim() || code}`));
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
const samples = new Float32Array(Math.floor(output.length / 4));
|
|
137
|
+
Buffer.from(samples.buffer).set(output.subarray(0, samples.length * 4));
|
|
138
|
+
return { samples, rate: SAMPLE_RATE };
|
|
139
|
+
}
|
|
140
|
+
finally {
|
|
141
|
+
await rm(directory, { recursive: true, force: true });
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
const BUNDLED = {
|
|
145
|
+
opus: decodeOgg,
|
|
146
|
+
vorbis: decodeVorbis,
|
|
147
|
+
flac: decodeFlac,
|
|
148
|
+
mp3: decodeMp3,
|
|
149
|
+
};
|
|
150
|
+
/**
|
|
151
|
+
* Opus, Vorbis, FLAC and MP3 by the bundled decoders; anything else, or what one of them could not read, by ffmpeg
|
|
152
|
+
* when the machine has it. The bundled decoder's error wins: it names the file's own problem.
|
|
153
|
+
*/
|
|
154
|
+
export const decodeAudio = async (bytes, ffmpeg = runFfmpeg) => {
|
|
155
|
+
const format = formatOf(bytes);
|
|
156
|
+
const pcm = format === "other"
|
|
157
|
+
? await ffmpeg(bytes)
|
|
158
|
+
: await BUNDLED[format](bytes).catch((error) => ffmpeg(bytes).catch(() => {
|
|
159
|
+
throw error;
|
|
160
|
+
}));
|
|
161
|
+
if (pcm.samples.length / pcm.rate > LONGEST_AUDIO_SECONDS)
|
|
162
|
+
throw new CliError("validation_error", `the audio is longer than ${LONGEST_AUDIO_SECONDS / 60} minutes`);
|
|
163
|
+
return pcm;
|
|
164
|
+
};
|
package/dist/speech/hearing.d.ts
CHANGED
|
@@ -25,6 +25,8 @@ export interface Hearing {
|
|
|
25
25
|
export declare const keyOf: ({ chatId, messageId }: Voice) => string;
|
|
26
26
|
/** Telegram says `voice`; MAX says `audio` with Ogg in it. */
|
|
27
27
|
export declare const isVoice: (message: Message) => boolean;
|
|
28
|
+
/** `messages transcribe` hears any audio file; `--transcribe` hears only voice notes, or every song in a chat. */
|
|
29
|
+
export declare const hasAudio: (message: Message) => boolean;
|
|
28
30
|
export interface Kept {
|
|
29
31
|
get(voice: Voice): Promise<{
|
|
30
32
|
text: string;
|
package/dist/speech/hearing.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { CliError } from "@wirecat/cli-base";
|
|
2
2
|
import { recalledAccount } from "../cli/messenger/accounts.js";
|
|
3
3
|
import { openStore } from "../store/store.js";
|
|
4
|
-
import { choose, hearLocally, hearOnline } from "./transcribe.js";
|
|
4
|
+
import { choose, hearLocally, hearOnline, isAudioFile } from "./transcribe.js";
|
|
5
5
|
/** `--model` chooses who hears what `--transcribe` asks for; alone it would quietly do nothing. */
|
|
6
6
|
export const modelWith = (transcribe, model) => {
|
|
7
7
|
if (model !== undefined && transcribe !== true) {
|
|
@@ -14,6 +14,8 @@ export const HEARING_BUDGET_MS = 120_000;
|
|
|
14
14
|
export const keyOf = ({ chatId, messageId }) => `${chatId}/${messageId}`;
|
|
15
15
|
/** Telegram says `voice`; MAX says `audio` with Ogg in it. */
|
|
16
16
|
export const isVoice = (message) => message.attachments.some(({ kind, mime }) => kind === "voice" || mime === "audio/ogg");
|
|
17
|
+
/** `messages transcribe` hears any audio file; `--transcribe` hears only voice notes, or every song in a chat. */
|
|
18
|
+
export const hasAudio = (message) => message.attachments.some(isAudioFile);
|
|
17
19
|
/**
|
|
18
20
|
* Transcripts already heard, in the shared store, for the account this profile last logged in as
|
|
19
21
|
* (owner, NEED-418 B). A profile that has never been online has no account yet: nothing is kept for
|
|
@@ -38,13 +40,13 @@ export const openKept = async (messenger, profile, env) => {
|
|
|
38
40
|
export const hearVoices = async (messenger, messages, kept, transcribe) => {
|
|
39
41
|
const transcripts = new Map();
|
|
40
42
|
const missing = [];
|
|
41
|
-
for (const message of messages.filter(
|
|
43
|
+
for (const message of messages.filter(hasAudio)) {
|
|
42
44
|
const voice = { chatId: message.chatId, messageId: message.id };
|
|
43
45
|
const text = (await kept.get(voice))?.text;
|
|
44
|
-
if (text
|
|
45
|
-
missing.push(voice);
|
|
46
|
-
else
|
|
46
|
+
if (text !== undefined)
|
|
47
47
|
transcripts.set(keyOf(voice), text);
|
|
48
|
+
else if (isVoice(message))
|
|
49
|
+
missing.push(voice);
|
|
48
50
|
}
|
|
49
51
|
if (!transcribe || missing.length === 0)
|
|
50
52
|
return { transcripts, unheard: missing };
|
|
@@ -91,7 +93,7 @@ export const spoken = ({ transcript, ...message }) => transcript === undefined
|
|
|
91
93
|
: { ...message, text: [message.text, `🎤 ${transcript}`].filter(Boolean).join("\n") };
|
|
92
94
|
/** For an MCP tool: the same hearing, over the session's own connection. */
|
|
93
95
|
export const hearForTool = async (messenger, connect, messages, transcribe, { settings, env }, model) => {
|
|
94
|
-
if (!messages.some(
|
|
96
|
+
if (!messages.some(hasAudio))
|
|
95
97
|
return undefined;
|
|
96
98
|
const kept = await openKept(messenger, settings.profile, env);
|
|
97
99
|
try {
|
|
@@ -2,6 +2,7 @@ import { CliError } from "@wirecat/cli-base";
|
|
|
2
2
|
import type { Messenger } from "../cli/messenger/context.js";
|
|
3
3
|
import { type Settings } from "../cli/settings.js";
|
|
4
4
|
import { type MessengerAdapter, type RemoteFile, type Transcript } from "../ports/messenger.js";
|
|
5
|
+
import type { Ffmpeg } from "./decode.js";
|
|
5
6
|
import { type SpeechModel } from "./models.js";
|
|
6
7
|
import type { Recognizer } from "./recognize.js";
|
|
7
8
|
/** `messenger`: its own recognition (Telegram Premium). `local`: a model on this machine. `auto`: the first, then the second. */
|
|
@@ -22,7 +23,13 @@ export interface Choice {
|
|
|
22
23
|
directory: string;
|
|
23
24
|
/** The local recognizer, where a test hands one in. */
|
|
24
25
|
open?: OpenRecognizer;
|
|
26
|
+
ffmpeg?: Ffmpeg;
|
|
25
27
|
}
|
|
28
|
+
/** A voice note, or an audio file someone sent — an iPhone recording reaches WhatsApp as `audio/mp4`. */
|
|
29
|
+
export declare const isAudioFile: ({ kind, mime }: {
|
|
30
|
+
kind: string;
|
|
31
|
+
mime?: string | undefined;
|
|
32
|
+
}) => boolean;
|
|
26
33
|
/** The flag, then the profile's `transcribeWith` and `speechModel`, then `auto` and the CLI's first model. */
|
|
27
34
|
export declare const choose: (messenger: Messenger, settings: Pick<Settings, "configured" | "shared">, flags: {
|
|
28
35
|
local?: boolean;
|
|
@@ -37,4 +44,4 @@ export declare const notDownloaded: (messenger: Messenger, model: SpeechModel) =
|
|
|
37
44
|
*/
|
|
38
45
|
export declare const hearOnline: (messenger: Messenger, connection: MessengerAdapter, chat: string, messageId: string, choice: Choice, read?: (file: RemoteFile) => Promise<void>) => Promise<Heard | Uint8Array>;
|
|
39
46
|
/** The local model, on the bytes `hearOnline` fetched. */
|
|
40
|
-
export declare const hearLocally: (bytes: Uint8Array, messageId: string, { model, directory, open }: Choice) => Promise<Heard>;
|
|
47
|
+
export declare const hearLocally: (bytes: Uint8Array, messageId: string, { model, directory, open, ffmpeg }: Choice) => Promise<Heard>;
|
|
@@ -6,8 +6,10 @@ import { installedBytes, isInstalled, megabytes, modelPath, modelsDirectory, vad
|
|
|
6
6
|
import { orderedModels, speechModel } from "./models.js";
|
|
7
7
|
/** `messenger`: its own recognition (Telegram Premium). `local`: a model on this machine. `auto`: the first, then the second. */
|
|
8
8
|
export const TRANSCRIBE_WITH = ["auto", "messenger", "local"];
|
|
9
|
-
/** A
|
|
9
|
+
/** A recording is read into memory whole; minutes of Opus are a few megabytes. */
|
|
10
10
|
const LARGEST_VOICE = 32 * 1024 * 1024;
|
|
11
|
+
/** A voice note, or an audio file someone sent — an iPhone recording reaches WhatsApp as `audio/mp4`. */
|
|
12
|
+
export const isAudioFile = ({ kind, mime }) => kind === "voice" || kind === "audio" || (mime?.startsWith("audio/") ?? false);
|
|
11
13
|
/** The flag, then the profile's `transcribeWith` and `speechModel`, then `auto` and the CLI's first model. */
|
|
12
14
|
export const choose = (messenger, settings, flags, env = process.env) => {
|
|
13
15
|
const model = flags.model ?? fromFile(settings, "speechModel", undefined).value;
|
|
@@ -45,17 +47,18 @@ export const hearOnline = async (messenger, connection, chat, messageId, choice,
|
|
|
45
47
|
if (!isInstalled(choice.model, choice.directory))
|
|
46
48
|
throw notDownloaded(messenger, choice.model);
|
|
47
49
|
const { files } = await capability(connection, "download", "download attachments")(chat, messageId);
|
|
48
|
-
const voice = files.find(
|
|
50
|
+
const voice = files.find(isAudioFile);
|
|
49
51
|
if (!voice)
|
|
50
|
-
throw new CliError("validation_error", "that message
|
|
52
|
+
throw new CliError("validation_error", "that message has no voice message or audio file");
|
|
51
53
|
const bytes = await readAll(voice);
|
|
52
54
|
await read?.(voice);
|
|
53
55
|
return bytes;
|
|
54
56
|
};
|
|
55
57
|
/** The local model, on the bytes `hearOnline` fetched. */
|
|
56
|
-
export const hearLocally = async (bytes, messageId, { model, directory, open }) => {
|
|
57
|
-
const {
|
|
58
|
-
const {
|
|
58
|
+
export const hearLocally = async (bytes, messageId, { model, directory, open, ffmpeg }) => {
|
|
59
|
+
const { openRecognizer, toModelRate } = await import("./recognize.js");
|
|
60
|
+
const { decodeAudio } = await import("./decode.js");
|
|
61
|
+
const { samples, rate } = await decodeAudio(bytes, ffmpeg);
|
|
59
62
|
const recognizer = (open ?? openRecognizer)(model, modelPath(directory, model), vadPath(directory));
|
|
60
63
|
try {
|
|
61
64
|
return {
|
|
@@ -78,7 +78,7 @@ export const resetAttachmentWords = (database) => {
|
|
|
78
78
|
extraction_error = NULL, extracted_at = t.heard_at, updated_at = t.updated_at
|
|
79
79
|
FROM message_transcripts t
|
|
80
80
|
WHERE attachments.attachable_type = 'message' AND attachments.attachable_id = t.message_id
|
|
81
|
-
AND (attachments.kind
|
|
81
|
+
AND (attachments.kind IN ('voice', 'audio') OR attachments.mime LIKE 'audio/%')
|
|
82
82
|
AND coalesce(attachments.text, '') = '' AND coalesce(attachments.extraction, '') <> 'agent'`)
|
|
83
83
|
.run();
|
|
84
84
|
const update = database.prepare("UPDATE attachments SET normalized_text = ? WHERE id = ?");
|
|
@@ -47,7 +47,7 @@ export const intoVoiceAttachment = (context, messagePk) => {
|
|
|
47
47
|
const row = context.database
|
|
48
48
|
.prepare(`SELECT a.id, t.text, t.source FROM message_transcripts t
|
|
49
49
|
JOIN attachments a ON a.attachable_type = 'message' AND a.attachable_id = t.message_id
|
|
50
|
-
WHERE t.message_id = ? AND (a.kind
|
|
50
|
+
WHERE t.message_id = ? AND (a.kind IN ('voice', 'audio') OR a.mime LIKE 'audio/%') ORDER BY a.position LIMIT 1`)
|
|
51
51
|
.get(messagePk);
|
|
52
52
|
if (!row)
|
|
53
53
|
return;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wirecat/cli-core",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.481.0",
|
|
4
4
|
"description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Viacheslav Ptsarev",
|
|
@@ -200,6 +200,8 @@
|
|
|
200
200
|
"@modelcontextprotocol/server": "^2.4.0",
|
|
201
201
|
"@resvg/resvg-js": "2.6.2",
|
|
202
202
|
"@valibot/to-json-schema": "1.8.0",
|
|
203
|
+
"@wasm-audio-decoders/flac": "0.2.11",
|
|
204
|
+
"@wasm-audio-decoders/ogg-vorbis": "0.1.20",
|
|
203
205
|
"@wirecat/cli-core-onnx": "1.2.0",
|
|
204
206
|
"@wirecat/cli-core-sqlite": "1.1.0",
|
|
205
207
|
"@xmldom/xmldom": "0.9.12",
|
|
@@ -220,6 +222,7 @@
|
|
|
220
222
|
"mdast-util-from-markdown": "2.1.0",
|
|
221
223
|
"mdast-util-gfm": "3.1.0",
|
|
222
224
|
"micromark-extension-gfm": "3.0.0",
|
|
225
|
+
"mpg123-decoder": "1.0.3",
|
|
223
226
|
"ogg-opus-decoder": "1.7.5",
|
|
224
227
|
"postal-mime": "4.1.4",
|
|
225
228
|
"psl": "1.15.0",
|