@alexkroman1/aai-cli 6.10.0 → 6.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/scaffold/CLAUDE.md +58 -0
- package/dist/scaffold/package.json +3 -3
- package/dist/scaffold/server.mjs +12 -3
- package/dist/scaffold/vite.config.ts +1 -1
- package/dist/templates/call-audit/agent.test.ts +965 -0
- package/dist/templates/call-audit/agent.ts +158 -0
- package/dist/templates/call-audit/client.tsx +235 -0
- package/dist/templates/call-audit/workflows/audit.ts +305 -0
- package/dist/templates/call-audit/workflows/ingest.ts +259 -0
- package/dist/templates/call-audit/workflows/media.ts +647 -0
- package/dist/templates/call-audit/workflows/summarize.ts +206 -0
- package/dist/templates/call-audit/workflows/sync-api.ts +44 -0
- package/dist/templates/call-audit/workflows/temp-media.ts +138 -0
- package/dist/templates/recap-workflow/agent.test.ts +11 -3
- package/dist/templates/recap-workflow/workflows/recap.ts +19 -8
- package/dist/templates/spoken-summary/agent.test.ts +343 -0
- package/dist/templates/spoken-summary/agent.ts +142 -0
- package/dist/templates/spoken-summary/client.tsx +225 -0
- package/dist/templates/spoken-summary/workflows/summarize.ts +242 -0
- package/dist/templates/spoken-summary/workflows/transcribe.ts +145 -0
- package/dist/templates/transcription-workflow/agent.test.ts +241 -18
- package/dist/templates/transcription-workflow/agent.ts +20 -6
- package/dist/templates/transcription-workflow/workflows/batch.ts +75 -173
- package/dist/templates/transcription-workflow/workflows/normalize.ts +343 -0
- package/dist/templates/transcription-workflow/workflows/stream.ts +6 -4
- package/dist/templates/transcription-workflow/workflows/sync-api.ts +26 -94
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +23 -14
- package/dist/templates/transcription-workflow/workflows/wav.ts +31 -0
- package/package.json +3 -3
|
@@ -8,13 +8,12 @@
|
|
|
8
8
|
* the two ways to arrange that. The async API has no such cap. You submit a job, it
|
|
9
9
|
* answers with an id in milliseconds, and the transcript is ready minutes later.
|
|
10
10
|
*
|
|
11
|
-
* So this flow is
|
|
11
|
+
* So this flow is three steps and no arithmetic:
|
|
12
12
|
*
|
|
13
13
|
* ```text
|
|
14
14
|
* uploadToProvider one step → the file, streamed, and the URL it answered
|
|
15
15
|
* createJob one step → the transcript id
|
|
16
|
-
* pollTranscript one step + a durable sleep, until
|
|
17
|
-
* readTranscript one step → the text
|
|
16
|
+
* pollTranscript one step + a durable sleep, until the text comes back
|
|
18
17
|
* ```
|
|
19
18
|
*
|
|
20
19
|
* **It is here to be compared against the other two, and it usually wins.** No
|
|
@@ -24,6 +23,21 @@
|
|
|
24
23
|
* inside: the latency is the provider's queue rather than your fan-out, and there is
|
|
25
24
|
* nothing to report between "submitted" and "done" except the job's own status.
|
|
26
25
|
*
|
|
26
|
+
* ## The endpoint is the SDK's; the STEPS are ours
|
|
27
|
+
*
|
|
28
|
+
* `stepTranscribeUpload` / `stepTranscribeSubmit` / `stepTranscribePoll` on
|
|
29
|
+
* `@alexkroman1/aai/utils` own the URL, the raw-key auth, the windowed streaming
|
|
30
|
+
* upload, the PLURAL `speech_models` field and the failure classification. This file
|
|
31
|
+
* used to spell all of that out, and so did `spoken-summary` — the same ~200 lines
|
|
32
|
+
* twice, reworded, identical in behaviour, and drifting apart at the edges.
|
|
33
|
+
*
|
|
34
|
+
* What stays here is what a dependency cannot decide: how many steps to cut the job
|
|
35
|
+
* into, and therefore what is journaled and what a retry repeats. That is also
|
|
36
|
+
* structural rather than stylistic — the Workflow DevKit's builder transforms
|
|
37
|
+
* exactly the files under this `workflows/` directory, so a `"use step"` shipped
|
|
38
|
+
* inside the SDK would be transformed by nothing and would run inline with no
|
|
39
|
+
* journal and no retry, silently.
|
|
40
|
+
*
|
|
27
41
|
* ## The one thing that makes this a WORKFLOW rather than a request
|
|
28
42
|
*
|
|
29
43
|
* The wait. A job takes minutes, and nothing about an HTTP request survives minutes:
|
|
@@ -32,66 +46,35 @@
|
|
|
32
46
|
* shape and its module doc carries the argument; this is the same pattern with the
|
|
33
47
|
* poll bounded by attempts rather than by a deadline.
|
|
34
48
|
*
|
|
35
|
-
* ##
|
|
49
|
+
* ## Three steps, not four, and the fourth was a wasted round trip
|
|
36
50
|
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
* That is also why the step that does it is the step the DevKit retries: a streaming
|
|
42
|
-
* body is consumed once, so a retry has to re-read the upload from the start, which
|
|
43
|
-
* it does. One window of READ-AHEAD keeps the store and the socket busy at the same
|
|
44
|
-
* time; `windows` carries the argument.
|
|
51
|
+
* This used to poll `GET /v2/transcript/:id` for a status and then fetch the
|
|
52
|
+
* identical URL a second time to read the text the poll already had in its hand.
|
|
53
|
+
* {@link pollTranscript} answers with the transcript, so a finished job costs one
|
|
54
|
+
* request rather than two and the value journaled by the last poll IS the result.
|
|
45
55
|
*/
|
|
46
56
|
|
|
47
|
-
import {
|
|
48
|
-
import {
|
|
57
|
+
import { throwStepError } from "@alexkroman1/aai/step-errors";
|
|
58
|
+
import {
|
|
59
|
+
report,
|
|
60
|
+
stepTranscribePoll,
|
|
61
|
+
stepTranscribeSubmit,
|
|
62
|
+
stepTranscribeUpload,
|
|
63
|
+
TRANSCRIBE_API,
|
|
64
|
+
uploadInfo,
|
|
65
|
+
} from "@alexkroman1/aai/utils";
|
|
49
66
|
import { sleep } from "workflow";
|
|
50
|
-
import { apiKeyOrFatal } from "./sync-api.ts";
|
|
51
67
|
import { countWords, startClock, type Transcript } from "./transcribe.ts";
|
|
52
68
|
|
|
53
|
-
/** The async API's base. */
|
|
54
|
-
const API = "https://api.assemblyai.com";
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* The models this desk asks for, best first.
|
|
58
|
-
*
|
|
59
|
-
* `speech_models`, PLURAL and an array. The singular `speech_model` is deprecated on
|
|
60
|
-
* the async API and answers **400** for any current model name — which is how this
|
|
61
|
-
* was found: the first live run of this flow failed on it, and the API said so in
|
|
62
|
-
* exactly those words. Note the streaming API still uses the singular field, so the
|
|
63
|
-
* two are not interchangeable and neither is "the" spelling.
|
|
64
|
-
*
|
|
65
|
-
* Omitting it entirely is also legal and routes to the default; naming it is what
|
|
66
|
-
* pins the model so a default change does not silently move this template's output.
|
|
67
|
-
*/
|
|
68
|
-
const MODELS = ["universal-3-5-pro"];
|
|
69
|
-
|
|
70
|
-
/** How much of our stored upload one outbound window carries. */
|
|
71
|
-
const UPLOAD_WINDOW_BYTES = 4 * 1024 * 1024;
|
|
72
|
-
|
|
73
|
-
/** How long a single request may take. The upload is not one of these — see below. */
|
|
74
|
-
const REQUEST_TIMEOUT_MS = 60_000;
|
|
75
|
-
|
|
76
|
-
/**
|
|
77
|
-
* How long the upload leg may take.
|
|
78
|
-
*
|
|
79
|
-
* Its own budget because it is the one request whose duration is a function of the
|
|
80
|
-
* FILE rather than of the service: a gigabyte at 8 MB/s is over two minutes, and a
|
|
81
|
-
* deadline sized for a JSON round trip would cancel exactly the uploads this flow
|
|
82
|
-
* exists to handle.
|
|
83
|
-
*/
|
|
84
|
-
const UPLOAD_TIMEOUT_MS = 30 * 60_000;
|
|
85
|
-
|
|
86
69
|
/** How long between polls of a submitted job. */
|
|
87
70
|
const POLL_INTERVAL = "10s";
|
|
88
71
|
|
|
89
72
|
/**
|
|
90
73
|
* Polls before the run gives up on a job.
|
|
91
74
|
*
|
|
92
|
-
* At {@link POLL_INTERVAL} this is an hour,
|
|
93
|
-
*
|
|
94
|
-
*
|
|
75
|
+
* At {@link POLL_INTERVAL} this is an hour, well past what the async API takes for
|
|
76
|
+
* any recording it accepts. Bounded rather than endless because a job that never
|
|
77
|
+
* leaves `queued` is a run that would otherwise be replayed forever.
|
|
95
78
|
*/
|
|
96
79
|
const MAX_POLLS = 360;
|
|
97
80
|
|
|
@@ -109,15 +92,15 @@ export async function transcribeBatchFlow(input: { recording: string }): Promise
|
|
|
109
92
|
const job = await createJob(audioUrl);
|
|
110
93
|
|
|
111
94
|
for (let poll = 0; poll < MAX_POLLS; poll += 1) {
|
|
112
|
-
const
|
|
113
|
-
if (
|
|
95
|
+
const progress = await pollTranscript(input.recording, job.id, startedAt);
|
|
96
|
+
if (progress.done) return progress.transcript;
|
|
114
97
|
await sleep(POLL_INTERVAL);
|
|
115
98
|
}
|
|
116
99
|
// A plain throw: this is the BODY, where the fatal/retryable distinction has
|
|
117
100
|
// nothing to apply to — see `stream.ts`'s `abandon` for the same reasoning.
|
|
118
101
|
throw new Error(
|
|
119
102
|
`Transcript ${job.id} was still unfinished after ${MAX_POLLS} polls. It is not lost — ` +
|
|
120
|
-
`read it directly with GET ${
|
|
103
|
+
`read it directly with GET ${TRANSCRIBE_API}/v2/transcript/${job.id}.`,
|
|
121
104
|
);
|
|
122
105
|
}
|
|
123
106
|
|
|
@@ -134,28 +117,17 @@ export async function transcribeBatchFlow(input: { recording: string }): Promise
|
|
|
134
117
|
* So the URL is journaled after all. The risk that made that look wrong is real but
|
|
135
118
|
* far smaller: if it expires before the next step runs, the run fails and a fresh one
|
|
136
119
|
* re-uploads — which is what would have happened anyway, once, instead of five times.
|
|
120
|
+
*
|
|
121
|
+
* `.catch(throwStepError)` is what turns the SDK's `TranscribeError` into the
|
|
122
|
+
* DevKit's verdict: a missing key and a 400 stop, a 429 waits as long as the service
|
|
123
|
+
* asked. Every step here ends the same way for the same reason.
|
|
137
124
|
*/
|
|
138
125
|
export async function uploadToProvider(uploadId: string): Promise<{ audioUrl: string }> {
|
|
139
126
|
"use step";
|
|
140
127
|
|
|
141
|
-
const apiKey = apiKeyOrFatal();
|
|
142
128
|
const stored = await uploadInfo(uploadId);
|
|
143
129
|
await report(`Uploading ${stored.name || uploadId} (${mb(stored.size)}) to the async API.`);
|
|
144
|
-
|
|
145
|
-
const uploaded = await stepFetch(`${API}/v2/upload`, {
|
|
146
|
-
method: "POST",
|
|
147
|
-
headers: { Authorization: apiKey, "Content-Type": "application/octet-stream" },
|
|
148
|
-
// An async iterable, not bytes: this file may be gigabytes, and nothing here holds
|
|
149
|
-
// more than one window of it. See the module doc.
|
|
150
|
-
body: windows(uploadId, stored.size),
|
|
151
|
-
signal: AbortSignal.timeout(UPLOAD_TIMEOUT_MS),
|
|
152
|
-
});
|
|
153
|
-
if (!uploaded.ok) throw await failure(uploaded, "Upload");
|
|
154
|
-
const { upload_url: audioUrl } = (await uploaded.json()) as { upload_url?: string };
|
|
155
|
-
if (!audioUrl) {
|
|
156
|
-
return throwFatalStepError(new Error("The async API accepted the upload but named no URL."));
|
|
157
|
-
}
|
|
158
|
-
return { audioUrl };
|
|
130
|
+
return await stepTranscribeUpload(uploadId).catch(throwStepError);
|
|
159
131
|
}
|
|
160
132
|
|
|
161
133
|
/** Retries beyond the default 3: an upload is the one call here worth another attempt. */
|
|
@@ -165,125 +137,55 @@ uploadToProvider.maxRetries = 5;
|
|
|
165
137
|
export async function createJob(audioUrl: string): Promise<{ id: string }> {
|
|
166
138
|
"use step";
|
|
167
139
|
|
|
168
|
-
const
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
body: JSON.stringify({ audio_url: audioUrl, speech_models: MODELS }),
|
|
172
|
-
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
|
173
|
-
});
|
|
174
|
-
if (!created.ok) throw await failure(created, "Submit");
|
|
175
|
-
const { id } = (await created.json()) as { id?: string };
|
|
176
|
-
if (!id) return throwFatalStepError(new Error("The async API created no transcript id."));
|
|
177
|
-
|
|
178
|
-
await report(`Submitted transcript ${id}.`);
|
|
179
|
-
return { id };
|
|
140
|
+
const job = await stepTranscribeSubmit(audioUrl).catch(throwStepError);
|
|
141
|
+
await report(`Submitted — job ${job.id}.`);
|
|
142
|
+
return job;
|
|
180
143
|
}
|
|
181
144
|
|
|
182
145
|
/**
|
|
183
|
-
* Ask once whether the job has finished.
|
|
146
|
+
* Ask once whether the job has finished, and read it when it has.
|
|
184
147
|
*
|
|
185
148
|
* `done` rather than the raw status, because the BODY branches on it and a body must
|
|
186
|
-
* not be
|
|
187
|
-
* otherwise
|
|
188
|
-
*
|
|
149
|
+
* not be where a provider's vocabulary is interpreted — a new status string would
|
|
150
|
+
* otherwise read as "not done yet" forever. A failed job is a terminal failure
|
|
151
|
+
* inside the SDK call, not a `done: true` the caller has to re-check.
|
|
189
152
|
*/
|
|
190
|
-
export async function pollTranscript(
|
|
191
|
-
"use step";
|
|
192
|
-
|
|
193
|
-
const res = await stepFetch(`${API}/v2/transcript/${id}`, {
|
|
194
|
-
headers: { Authorization: apiKeyOrFatal() },
|
|
195
|
-
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
|
196
|
-
});
|
|
197
|
-
if (!res.ok) throw await failure(res, `Transcript ${id}`);
|
|
198
|
-
const body = (await res.json()) as { status?: string; error?: string };
|
|
199
|
-
const status = body.status ?? "unknown";
|
|
200
|
-
if (status === "error") {
|
|
201
|
-
// The provider has decided; no number of polls changes it.
|
|
202
|
-
return throwFatalStepError(
|
|
203
|
-
new Error(
|
|
204
|
-
`The async API could not transcribe that recording: ${body.error ?? "no reason given"}`,
|
|
205
|
-
),
|
|
206
|
-
);
|
|
207
|
-
}
|
|
208
|
-
await report(`Transcript ${id} is ${status}.`);
|
|
209
|
-
return { done: status === "completed", status };
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
/** Read the finished transcript, and report it the way both sync flows do. */
|
|
213
|
-
export async function readTranscript(
|
|
153
|
+
export async function pollTranscript(
|
|
214
154
|
uploadId: string,
|
|
215
155
|
id: string,
|
|
216
156
|
startedAt: number,
|
|
217
|
-
): Promise<Transcript> {
|
|
157
|
+
): Promise<{ done: false } | { done: true; transcript: Transcript }> {
|
|
218
158
|
"use step";
|
|
219
159
|
|
|
220
|
-
const
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
const body = (await res.json()) as { text?: string; audio_duration?: number };
|
|
226
|
-
const transcript = (body.text ?? "").trim();
|
|
227
|
-
const stored = await uploadInfo(uploadId);
|
|
160
|
+
const progress = await stepTranscribePoll(id).catch(throwStepError);
|
|
161
|
+
if (!progress.done) {
|
|
162
|
+
await report(`Transcript ${id} is ${progress.status}.`);
|
|
163
|
+
return { done: false };
|
|
164
|
+
}
|
|
228
165
|
|
|
166
|
+
const stored = await uploadInfo(uploadId);
|
|
167
|
+
const transcript = progress.transcript.text;
|
|
229
168
|
return {
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
169
|
+
done: true,
|
|
170
|
+
transcript: {
|
|
171
|
+
source: stored.name || uploadId,
|
|
172
|
+
// ONE, and it is not a fudge: the async API transcribed the recording in one
|
|
173
|
+
// piece, which is the difference this flow is here to show. A reader comparing
|
|
174
|
+
// the three sees 7 segments, 7 segments, and 1.
|
|
175
|
+
segments: 1,
|
|
176
|
+
// The provider's own measurement — the only one of the three flows that does
|
|
177
|
+
// not have to derive this from byte offsets.
|
|
178
|
+
durationMs: progress.transcript.durationMs,
|
|
179
|
+
// Wall clock, the same way both sync flows measure it — see `startClock`. For
|
|
180
|
+
// this flow it is mostly the provider's queue, which is exactly the thing a
|
|
181
|
+
// reader comparing the three wants to see.
|
|
182
|
+
elapsedMs: Date.now() - startedAt,
|
|
183
|
+
words: countWords(transcript),
|
|
184
|
+
transcript,
|
|
185
|
+
},
|
|
244
186
|
};
|
|
245
187
|
}
|
|
246
188
|
|
|
247
|
-
/**
|
|
248
|
-
* The stored upload as a sequence of windows, with the next one already in flight.
|
|
249
|
-
*
|
|
250
|
-
* A generator rather than one `readUpload`, because the whole point is that the file
|
|
251
|
-
* is never held: each window is read, sent, and dropped. `readUpload` clamps to what
|
|
252
|
-
* is stored, so the loop ends on the real end of the file even if `size` moved.
|
|
253
|
-
*
|
|
254
|
-
* **One window of READ-AHEAD**, which is the whole concurrency available here: the
|
|
255
|
-
* consumer is a socket and the producer is the app's own store, and read-then-send
|
|
256
|
-
* makes them strictly alternate — the store idles while bytes go out, and the socket
|
|
257
|
-
* idles while the next window is fetched. Starting the next read before yielding the
|
|
258
|
-
* current window overlaps them, so a gigabyte upload pays the larger of the two
|
|
259
|
-
* rather than their sum. Exactly one, not a queue: a deeper buffer holds more of a
|
|
260
|
-
* file this generator exists to avoid holding, and there is nothing to gain past
|
|
261
|
-
* keeping both ends busy.
|
|
262
|
-
*/
|
|
263
|
-
async function* windows(uploadId: string, size: number): AsyncGenerator<Uint8Array> {
|
|
264
|
-
const read = (at: number): Promise<Uint8Array> =>
|
|
265
|
-
readUpload(uploadId, { start: at, end: at + UPLOAD_WINDOW_BYTES }).then((slice) => slice.bytes);
|
|
266
|
-
let at = 0;
|
|
267
|
-
let next = at < size ? read(at) : undefined;
|
|
268
|
-
while (next !== undefined) {
|
|
269
|
-
const bytes = await next;
|
|
270
|
-
if (bytes.length === 0) return;
|
|
271
|
-
at += UPLOAD_WINDOW_BYTES;
|
|
272
|
-
// Issued BEFORE the yield, so the store is fetching while the socket sends.
|
|
273
|
-
next = at < size ? read(at) : undefined;
|
|
274
|
-
yield bytes;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
/** A failed call, classified for the DevKit — see `sync-api.ts` for the three-way rule. */
|
|
279
|
-
async function failure(res: Response, what: string): Promise<Error> {
|
|
280
|
-
const body = (await res.json().catch(() => ({}))) as { error?: string };
|
|
281
|
-
return toStepError(
|
|
282
|
-
res,
|
|
283
|
-
`${what} failed: HTTP ${res.status}${body.error ? ` — ${body.error}` : ""}`,
|
|
284
|
-
);
|
|
285
|
-
}
|
|
286
|
-
|
|
287
189
|
/** A size a person can read, because the number that matters is the scale. */
|
|
288
190
|
function mb(bytes: number): string {
|
|
289
191
|
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
// Copyright 2026 the AAI authors. MIT license.
|
|
2
|
+
/**
|
|
3
|
+
* The step that makes the rest of the desk possible on a real file: whatever was
|
|
4
|
+
* uploaded, converted to the one format the arithmetic works on.
|
|
5
|
+
*
|
|
6
|
+
* `wav.ts` explains why this desk cuts linear-PCM WAV and nothing else — a byte
|
|
7
|
+
* offset is only a timestamp when every sample is the same size — and for a long
|
|
8
|
+
* time the remedy for anything else was a SENTENCE telling the caller to run
|
|
9
|
+
* `ffmpeg -i in.m4a -c:a pcm_s16le out.wav` on their own machine first. Every
|
|
10
|
+
* recording anyone actually has is an `.m4a` off a phone or an `.mp3` out of a
|
|
11
|
+
* conferencing tool, so that sentence was the desk's real front door, and it
|
|
12
|
+
* opened onto the user's shell. The platform installs ffmpeg in every guest
|
|
13
|
+
* image; this file is the desk using it.
|
|
14
|
+
*
|
|
15
|
+
* ```text
|
|
16
|
+
* normalizeRecording one step → an upload id the rest of the flow can cut
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* ## `parseWav` is asked as a QUESTION
|
|
20
|
+
*
|
|
21
|
+
* The obvious implementation probes the file with `ffprobe` and passes it
|
|
22
|
+
* through when the codec looks like PCM. It is wrong in a way that only shows up
|
|
23
|
+
* on a Windows recorder's output: a `WAVE_FORMAT_EXTENSIBLE` file reports
|
|
24
|
+
* `pcm_s16le` to ffprobe and is refused by {@link parseWav}, whose encoding
|
|
25
|
+
* check reads the format tag ffprobe does not surface. The desk would convert
|
|
26
|
+
* nothing and then fail to cut it.
|
|
27
|
+
*
|
|
28
|
+
* So the test is {@link parseWav} ITSELF, run against the same
|
|
29
|
+
* {@link HEADER_PROBE_BYTES} window `splitRecording` will use. A throw is the
|
|
30
|
+
* signal to convert. That makes the pass-through decision and the cut decision
|
|
31
|
+
* the same decision by construction — there is no second opinion to disagree —
|
|
32
|
+
* and it means the desk fixes anything the parser rejects for any reason,
|
|
33
|
+
* including a 192 kHz 32-bit stereo WAV that trips
|
|
34
|
+
* {@link MAX_BYTES_PER_SECOND}, which downsampling genuinely repairs.
|
|
35
|
+
*
|
|
36
|
+
* The fast path costs one 64 KB read and no subprocess at all: a WAV that was
|
|
37
|
+
* already cuttable is returned by the id it came in under, so nothing is copied
|
|
38
|
+
* and nothing is re-encoded.
|
|
39
|
+
*
|
|
40
|
+
* ## File → file, not bytes → bytes
|
|
41
|
+
*
|
|
42
|
+
* `transcodeToWav(bytes)` is one line and is the wrong call here, twice over:
|
|
43
|
+
*
|
|
44
|
+
* - **The output would be buffered.** Piped stdout is capped
|
|
45
|
+
* (`DEFAULT_MAX_FFMPEG_OUTPUT_BYTES`, 64 MiB), which is about an hour of
|
|
46
|
+
* 16 kHz mono — and this desk exists for the two-hour recording.
|
|
47
|
+
* - **The input could not be READ.** A pipe cannot seek, and an `.m4a` written
|
|
48
|
+
* by a phone usually carries its `moov` index at the END of the file, so
|
|
49
|
+
* ffmpeg fails on the flagship input with `moov atom not found`. That is the
|
|
50
|
+
* one caveat `@alexkroman1/aai/ffmpeg`'s own doc names, and this is the case
|
|
51
|
+
* it names it for.
|
|
52
|
+
*
|
|
53
|
+
* So the recording is materialized to a temp file in windows, converted file to
|
|
54
|
+
* file, and streamed back into the upload store. Nothing here holds a whole
|
|
55
|
+
* recording in memory at any point, which is the property that makes the step
|
|
56
|
+
* work on the input it was written for.
|
|
57
|
+
*
|
|
58
|
+
* ## A temp file cannot cross a step boundary
|
|
59
|
+
*
|
|
60
|
+
* Everything above happens in ONE step, and that is structural rather than
|
|
61
|
+
* tidy. A step is journaled by its RETURN VALUE and may be dispatched into a
|
|
62
|
+
* different process than its neighbours, so a path in a return value is a path
|
|
63
|
+
* that is replayed after the file behind it is gone. What crosses the boundary
|
|
64
|
+
* is an upload ID; the temp directory is created and removed inside the step
|
|
65
|
+
* that uses it.
|
|
66
|
+
*/
|
|
67
|
+
|
|
68
|
+
import { mkdtemp, open, rm } from "node:fs/promises";
|
|
69
|
+
import { tmpdir } from "node:os";
|
|
70
|
+
import { basename, extname, join } from "node:path";
|
|
71
|
+
import { isFfmpegError, probeMedia, runFfmpeg, wavEncodeArgs } from "@alexkroman1/aai/ffmpeg";
|
|
72
|
+
import { throwFatalStepError, throwStepError } from "@alexkroman1/aai/step-errors";
|
|
73
|
+
import { readUpload, report, uploadInfo, writeUpload } from "@alexkroman1/aai/utils";
|
|
74
|
+
import { clock } from "./stitch.ts";
|
|
75
|
+
import { HEADER_PROBE_BYTES, parseWav, UnsupportedRecordingError } from "./wav.ts";
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The rate everything is converted TO.
|
|
79
|
+
*
|
|
80
|
+
* 16 kHz because that is what speech models are trained at — a higher rate
|
|
81
|
+
* carries no information the decoder uses and costs proportional bytes in a
|
|
82
|
+
* fan-out whose width is bounded by bytes in flight (`BYTES_IN_FLIGHT` in
|
|
83
|
+
* `transcribe.ts`). A converted two-hour recording is 230 MB of 16 kHz mono
|
|
84
|
+
* against 1.4 GB of 48 kHz stereo, which is the difference between a fan-out
|
|
85
|
+
* that saturates on width and one that saturates on the queue.
|
|
86
|
+
*/
|
|
87
|
+
export const NORMALIZED_SAMPLE_RATE = 16_000;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Channels everything is converted TO.
|
|
91
|
+
*
|
|
92
|
+
* Mono, and it is a real loss rather than a free win: a stereo call recording
|
|
93
|
+
* with one party per channel is exactly the file where the channels are the most
|
|
94
|
+
* interesting thing about it, and downmixing throws that away. This desk
|
|
95
|
+
* transcribes rather than diarizes, so it takes the 2x saving; a desk that wants
|
|
96
|
+
* the speakers apart splits the channels first and transcribes each one.
|
|
97
|
+
*/
|
|
98
|
+
export const NORMALIZED_CHANNELS = 1;
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Bytes moved per `readUpload` while materializing, and per write while storing.
|
|
102
|
+
*
|
|
103
|
+
* 8 MiB is large enough that a two-hour recording is a few hundred round trips
|
|
104
|
+
* rather than tens of thousands, and small enough that the step's resident set
|
|
105
|
+
* is a constant that does not depend on the recording. The number this must NOT
|
|
106
|
+
* be is "the whole file", which is the shape every first draft of this step has.
|
|
107
|
+
*/
|
|
108
|
+
const WINDOW_BYTES = 8 * 1024 * 1024;
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* How long a conversion may run before it is killed.
|
|
112
|
+
*
|
|
113
|
+
* Well past what the work takes — ffmpeg decodes and resamples faster than
|
|
114
|
+
* realtime by two orders of magnitude, so a two-hour recording is under a
|
|
115
|
+
* minute — and the reason for a bound at all is a file that makes a decoder
|
|
116
|
+
* pathological rather than one that is merely long. A `timeout` is retryable
|
|
117
|
+
* and an `exit` is not; see {@link classifyFfmpeg}.
|
|
118
|
+
*/
|
|
119
|
+
const CONVERT_TIMEOUT_MS = 15 * 60_000;
|
|
120
|
+
|
|
121
|
+
/** What the flow is handed: the id to cut, and whether it had to be made. */
|
|
122
|
+
export type NormalizedRecording = {
|
|
123
|
+
/**
|
|
124
|
+
* The upload id every later step reads.
|
|
125
|
+
*
|
|
126
|
+
* The SAME id that came in when the file was already cuttable, and a new one
|
|
127
|
+
* when it was converted — which is why the flow threads this rather than its
|
|
128
|
+
* own input from here on.
|
|
129
|
+
*/
|
|
130
|
+
recording: string;
|
|
131
|
+
/** Whether ffmpeg ran. Reported, so a reader can tell a fast path from a slow one. */
|
|
132
|
+
converted: boolean;
|
|
133
|
+
};
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Make sure the recording is something the desk can cut, converting if not.
|
|
137
|
+
*
|
|
138
|
+
* A step, for the ordinary two reasons — it does I/O, and its RESULT is what
|
|
139
|
+
* every later step addresses — plus one specific to what it produces: the
|
|
140
|
+
* conversion writes a file, and journaling the id means a resumed run reads the
|
|
141
|
+
* file that already exists instead of paying for a second one.
|
|
142
|
+
*/
|
|
143
|
+
export async function normalizeRecording(uploadId: string): Promise<NormalizedRecording> {
|
|
144
|
+
"use step";
|
|
145
|
+
|
|
146
|
+
const stored = await uploadInfo(uploadId);
|
|
147
|
+
const head = await readUpload(uploadId, { end: HEADER_PROBE_BYTES });
|
|
148
|
+
|
|
149
|
+
if (cuttable(head.bytes, stored.size)) {
|
|
150
|
+
// No subprocess, no copy, no second upload. The overwhelmingly common case
|
|
151
|
+
// for a desk whose form says WAV, and the reason the check is a 64 KB read.
|
|
152
|
+
await report(`${stored.name || uploadId} is already linear-PCM WAV — cutting it as it is.`);
|
|
153
|
+
return { recording: uploadId, converted: false };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// Named before any work starts, because everything below is minutes of it on a
|
|
157
|
+
// long recording and a run that says nothing until the conversion finishes looks
|
|
158
|
+
// stuck. It is also the line that distinguishes "this file needs converting" from
|
|
159
|
+
// the fast path above.
|
|
160
|
+
await report(
|
|
161
|
+
`Converting ${stored.name || uploadId} (${mb(stored.size)}) — not a WAV we can cut.`,
|
|
162
|
+
);
|
|
163
|
+
|
|
164
|
+
const dir = await mkdtemp(join(tmpdir(), "aai-normalize-"));
|
|
165
|
+
try {
|
|
166
|
+
const source = join(dir, "source");
|
|
167
|
+
const converted = join(dir, "converted.wav");
|
|
168
|
+
|
|
169
|
+
await materialize(uploadId, stored.size, source);
|
|
170
|
+
|
|
171
|
+
// What it WAS, for the progress line. Worth one ffprobe: "converted 41
|
|
172
|
+
// minutes of aac" is a line that explains the run's shape, where
|
|
173
|
+
// "converted the recording" leaves a reader wondering what the desk decided.
|
|
174
|
+
// On a temp file rather than a pipe, so a trailing index is readable.
|
|
175
|
+
const info = await probeMedia(source, { timeoutMs: CONVERT_TIMEOUT_MS }).catch(classifyFfmpeg);
|
|
176
|
+
await report(
|
|
177
|
+
`It is ${describeSource(info.audio?.codec, info.durationSec)} — re-encoding to ` +
|
|
178
|
+
`${NORMALIZED_SAMPLE_RATE / 1000} kHz mono WAV.`,
|
|
179
|
+
);
|
|
180
|
+
|
|
181
|
+
await runFfmpeg(
|
|
182
|
+
[
|
|
183
|
+
// The argv is the caller's, verbatim — `runFfmpeg` adds nothing. So the
|
|
184
|
+
// standing flags are here: quiet, non-interactive, overwrite. `-nostdin`
|
|
185
|
+
// matters most in a guest, where there is no terminal and an ffmpeg that
|
|
186
|
+
// decides to read stdin is a process that never exits.
|
|
187
|
+
"-hide_banner",
|
|
188
|
+
"-loglevel",
|
|
189
|
+
"error",
|
|
190
|
+
"-nostdin",
|
|
191
|
+
"-y",
|
|
192
|
+
"-i",
|
|
193
|
+
source,
|
|
194
|
+
...wavEncodeArgs({
|
|
195
|
+
sampleRate: NORMALIZED_SAMPLE_RATE,
|
|
196
|
+
channels: NORMALIZED_CHANNELS,
|
|
197
|
+
}),
|
|
198
|
+
converted,
|
|
199
|
+
],
|
|
200
|
+
{ timeoutMs: CONVERT_TIMEOUT_MS },
|
|
201
|
+
).catch(classifyFfmpeg);
|
|
202
|
+
|
|
203
|
+
const written = await writeUpload(chunks(converted), {
|
|
204
|
+
// Named after the ORIGINAL, so a download reads as the recording it came
|
|
205
|
+
// from. The extension has to change with the bytes: a file served as
|
|
206
|
+
// `audio/wav` under a `.m4a` name is one no player will open.
|
|
207
|
+
name: `${basename(stored.name || uploadId, extname(stored.name || uploadId))}.wav`,
|
|
208
|
+
type: "audio/wav",
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
await report(`Converted to ${mb(written.size)} of WAV (from ${mb(stored.size)}).`);
|
|
212
|
+
return { recording: written.id, converted: true };
|
|
213
|
+
} finally {
|
|
214
|
+
// Always, including on the failure paths above: a guest's disk is small and
|
|
215
|
+
// a step that leaves a copy of every recording it touched fills it. `force`
|
|
216
|
+
// so a conversion that never created its output does not fail HERE and
|
|
217
|
+
// replace the real error with this one.
|
|
218
|
+
await rm(dir, { recursive: true, force: true });
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Retries beyond the default 3.
|
|
224
|
+
*
|
|
225
|
+
* Not because a conversion is flaky — a corrupt file fails identically forever,
|
|
226
|
+
* and {@link classifyFfmpeg} is what stops the DevKit retrying that. It is the
|
|
227
|
+
* two I/O halves that are worth another attempt: this step reads a whole
|
|
228
|
+
* recording out of the store and writes a whole one back, and either can lose a
|
|
229
|
+
* connection on a file this size.
|
|
230
|
+
*/
|
|
231
|
+
normalizeRecording.maxRetries = 5;
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Whether `splitRecording` will be able to read this header.
|
|
235
|
+
*
|
|
236
|
+
* The question, not a guess at it — see the module doc. Only
|
|
237
|
+
* {@link UnsupportedRecordingError} is answered `false`: anything else thrown by
|
|
238
|
+
* the parser is a bug in the parser, and swallowing it here would turn that into
|
|
239
|
+
* a mysterious re-encode of a file that was fine.
|
|
240
|
+
*/
|
|
241
|
+
export function cuttable(head: Uint8Array, totalBytes: number): boolean {
|
|
242
|
+
try {
|
|
243
|
+
parseWav(head, totalBytes);
|
|
244
|
+
return true;
|
|
245
|
+
} catch (err: unknown) {
|
|
246
|
+
if (err instanceof UnsupportedRecordingError) return false;
|
|
247
|
+
throw err;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/**
|
|
252
|
+
* Turn an ffmpeg failure into the DevKit's verdict.
|
|
253
|
+
*
|
|
254
|
+
* The whole reason `FfmpegError.kind` exists, used the way it was meant to be:
|
|
255
|
+
* an `exit` is ffmpeg having read the file and refused it, so every retry
|
|
256
|
+
* re-reads the same bytes and reaches the same conclusion while burning the
|
|
257
|
+
* budget a real transient needs. A `timeout` or an `aborted` is worth another
|
|
258
|
+
* attempt, and a `missing-binary` is `aai dev` on a laptop with no ffmpeg —
|
|
259
|
+
* fatal, and already carrying the install instructions in its message.
|
|
260
|
+
*
|
|
261
|
+
* **The retryable arm goes through `throwStepError` even though it classifies
|
|
262
|
+
* nothing**, and that is deliberate rather than a leftover. `toStepError` reaches
|
|
263
|
+
* a verdict from a `Response` or from an SDK error that already carries one; an
|
|
264
|
+
* `FfmpegError` is neither, so it is rethrown UNCHANGED — which the DevKit treats
|
|
265
|
+
* as retryable by default, the outcome this arm wants. Writing
|
|
266
|
+
* `new RetryableError(...)` here instead would replace ffmpeg's own message and
|
|
267
|
+
* its `argv` with a sentence, and the argv is the thing you paste into a shell.
|
|
268
|
+
* So the call reads as the decision it is: everything this function does not
|
|
269
|
+
* declare terminal keeps its retries.
|
|
270
|
+
*
|
|
271
|
+
* Exported for its spec. It is the one decision in this file a unit test can
|
|
272
|
+
* reach — everything around it spawns a subprocess — and it is also the one worth
|
|
273
|
+
* reaching: getting it backwards means either five re-reads of a corrupt file or
|
|
274
|
+
* no second attempt at a conversion that was merely slow.
|
|
275
|
+
*/
|
|
276
|
+
export function classifyFfmpeg(err: unknown): never {
|
|
277
|
+
if (isFfmpegError(err) && (err.kind === "timeout" || err.kind === "aborted")) {
|
|
278
|
+
return throwStepError(err);
|
|
279
|
+
}
|
|
280
|
+
return throwFatalStepError(err);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Write an upload to a local path, a window at a time.
|
|
285
|
+
*
|
|
286
|
+
* The `readUpload` window is the same primitive `transcribeSegment` cuts with;
|
|
287
|
+
* what differs is only that this one walks the whole file in order. A `for` loop
|
|
288
|
+
* rather than a fan-out deliberately — the bytes land in one file at one offset
|
|
289
|
+
* each, so concurrency buys nothing here and costs the memory the windows are
|
|
290
|
+
* there to bound.
|
|
291
|
+
*/
|
|
292
|
+
async function materialize(uploadId: string, size: number, path: string): Promise<void> {
|
|
293
|
+
const handle = await open(path, "w");
|
|
294
|
+
try {
|
|
295
|
+
for (let at = 0; at < size; at += WINDOW_BYTES) {
|
|
296
|
+
const slice = await readUpload(uploadId, {
|
|
297
|
+
start: at,
|
|
298
|
+
end: Math.min(at + WINDOW_BYTES, size),
|
|
299
|
+
});
|
|
300
|
+
await handle.write(slice.bytes);
|
|
301
|
+
}
|
|
302
|
+
} finally {
|
|
303
|
+
await handle.close();
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
/**
|
|
308
|
+
* A local file as the stream `writeUpload` takes.
|
|
309
|
+
*
|
|
310
|
+
* A generator rather than `readFile`, for the reason the windows exist: the
|
|
311
|
+
* converted WAV is the largest thing this step touches, and handing the store an
|
|
312
|
+
* `AsyncIterable` is what keeps it off the heap.
|
|
313
|
+
*
|
|
314
|
+
* The `.slice()` is load-bearing. One buffer is reused across reads, so yielding
|
|
315
|
+
* a view of it hands the consumer memory the next read overwrites — a bug whose
|
|
316
|
+
* symptom is a stored file made of the LAST chunk repeated, and which does not
|
|
317
|
+
* reproduce whenever the consumer happens to copy before the next iteration.
|
|
318
|
+
*/
|
|
319
|
+
async function* chunks(path: string): AsyncIterable<Uint8Array> {
|
|
320
|
+
const handle = await open(path, "r");
|
|
321
|
+
try {
|
|
322
|
+
const buffer = new Uint8Array(WINDOW_BYTES);
|
|
323
|
+
for (;;) {
|
|
324
|
+
const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
|
|
325
|
+
if (bytesRead === 0) return;
|
|
326
|
+
yield buffer.subarray(0, bytesRead).slice();
|
|
327
|
+
}
|
|
328
|
+
} finally {
|
|
329
|
+
await handle.close();
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** `41:20 of aac`, or as much of that as ffprobe would say. */
|
|
334
|
+
function describeSource(codec: string | undefined, durationSec: number | undefined): string {
|
|
335
|
+
const length = durationSec === undefined ? undefined : clock(Math.round(durationSec * 1000));
|
|
336
|
+
if (length !== undefined && codec !== undefined) return `${length} of ${codec}`;
|
|
337
|
+
return length ?? codec ?? "the recording";
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
/** A size a person can read, because the number that matters is the scale. */
|
|
341
|
+
function mb(bytes: number): string {
|
|
342
|
+
return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
343
|
+
}
|