@alexkroman1/aai-cli 6.10.0 → 6.11.0

package/dist/templates/transcription-workflow/agent.test.ts

@@ -17,14 +17,18 @@
  * the decoder happily transcribes into confident nonsense.
  */
 
+import { readdir } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { FfmpegError } from "@alexkroman1/aai/ffmpeg";
 import { stubReporter, stubStepFetch, stubUploads } from "@alexkroman1/aai/testing";
 import { omitUndefined, readUpload } from "@alexkroman1/aai/utils";
 import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
 import { FatalError, RetryableError } from "workflow";
 import { z } from "zod";
 import agentDef, { transcribe, transcribeBatch, transcribeStream } from "./agent.ts";
-import { createJob, pollTranscript, readTranscript, uploadToProvider } from "./workflows/batch.ts";
-import { planStreamed, probeUpload } from "./workflows/stream.ts";
+import { createJob, pollTranscript, uploadToProvider } from "./workflows/batch.ts";
+import { classifyFfmpeg, cuttable, normalizeRecording } from "./workflows/normalize.ts";
+import { expectedSegments, planStreamed, probeUpload } from "./workflows/stream.ts";
 import {
   clock,
   mergeTranscript,
@@ -819,45 +823,59 @@ describe("the async flow", () => {
     stubBatch(() => ({ body: { status: "error", error: "audio too quiet" } }));
     // The provider has decided; no number of polls changes it, so this must not come
     // back as "not done yet".
-    await expect(pollTranscript("tr_1")).rejects.toBeInstanceOf(FatalError);
+    await expect(pollTranscript(UPLOAD_ID, "tr_1", STARTED_AT)).rejects.toBeInstanceOf(FatalError);
   });
 
   test("pollTranscript answers `done` on completed and not before", async () => {
     publishRecording(new Uint8Array(10));
     stubBatch(() => ({ body: { status: "processing" } }));
-    await expect(pollTranscript("tr_1")).resolves.toEqual({ done: false, status: "processing" });
+    await expect(pollTranscript(UPLOAD_ID, "tr_1", STARTED_AT)).resolves.toEqual({ done: false });
   });
 
   test("an unknown status is NOT done, so a new one cannot end a run early", async () => {
     publishRecording(new Uint8Array(10));
     stubBatch(() => ({ body: {} }));
-    await expect(pollTranscript("tr_1")).resolves.toMatchObject({ done: false });
+    await expect(pollTranscript(UPLOAD_ID, "tr_1", STARTED_AT)).resolves.toMatchObject({
+      done: false,
+    });
   });
 
-  test("readTranscript reports the provider's own duration and ONE segment", async () => {
+  test("a completed poll carries the transcript — ONE request, not two", async () => {
     publishRecording(new Uint8Array(10), "standup.wav");
-    stubBatch(() => ({ body: { text: "  hello there  ", audio_duration: 12.5 } }));
-    await expect(readTranscript(UPLOAD_ID, "tr_1", STARTED_AT)).resolves.toMatchObject({
-      source: "standup.wav",
-      // Not a fudge: the async API transcribed the recording in one piece, which is
-      // the difference this flow is here to show.
-      segments: 1,
-      durationMs: 12_500,
-      words: 2,
-      transcript: "hello there",
+    // It used to poll for a status and then fetch the identical URL again for the
+    // text the poll already had in its hand.
+    const calls = stubBatch(() => ({
+      body: { status: "completed", text: "  hello there  ", audio_duration: 12.5 },
+    }));
+    await expect(pollTranscript(UPLOAD_ID, "tr_1", STARTED_AT)).resolves.toMatchObject({
+      done: true,
+      transcript: {
+        source: "standup.wav",
+        // Not a fudge: the async API transcribed the recording in one piece, which is
+        // the difference this flow is here to show.
+        segments: 1,
+        durationMs: 12_500,
+        words: 2,
+        transcript: "hello there",
+      },
     });
+    expect(calls).toHaveLength(1);
   });
 
   test("a rate limit is RETRYABLE, so a busy minute does not fail the run", async () => {
     publishRecording(new Uint8Array(10));
     stubBatch(() => ({ status: 429, body: { error: "slow down" } }));
-    await expect(pollTranscript("tr_1")).rejects.toBeInstanceOf(RetryableError);
+    await expect(pollTranscript(UPLOAD_ID, "tr_1", STARTED_AT)).rejects.toBeInstanceOf(
+      RetryableError,
+    );
   });
 
   test("all three flows report the same SHAPE, which is what lets one page render any", async () => {
     publishRecording(new Uint8Array(10), "standup.wav");
-    stubBatch(() => ({ body: { text: "hi", audio_duration: 1 } }));
-    const batched = await readTranscript(UPLOAD_ID, "tr_1", STARTED_AT);
+    stubBatch(() => ({ body: { status: "completed", text: "hi", audio_duration: 1 } }));
+    const progress = await pollTranscript(UPLOAD_ID, "tr_1", STARTED_AT);
+    if (!progress.done) return expect.fail("the stub reports a completed job");
+    const batched = progress.transcript;
     // One key set, so the page's summary line renders every flow's output. A field on
     // one flow and not the others is a panel that shows it for some runs and not
     // others, with nothing saying why.
@@ -873,3 +891,208 @@ describe("the async flow", () => {
     expect(batched.elapsedMs).toBeGreaterThan(0);
   });
 });
+
+describe("normalizing the recording", () => {
+  /**
+   * The CONVERSION is not driven here, and the reason is the tier rather than the
+   * code: it spawns ffmpeg and writes a temp file, neither of which a unit test
+   * may do. What is reachable is everything that DECIDES — whether a file needs
+   * converting at all, and how a conversion's failure is classified — and those
+   * are the two places a mistake is silent. A file wrongly passed through fails
+   * later in `splitRecording` with a message about a header; a `timeout`
+   * classified as fatal is a run that gives up on work that would have finished.
+   */
+  test("a canonical WAV is cuttable, so the desk converts nothing", () => {
+    expect(cuttable(wavFile(MONO_16K, 32_000), 44 + 32_000)).toBe(true);
+  });
+
+  test("an extra chunk before the samples is still cuttable", () => {
+    // The `LIST`-chunk case, which is the reason the probe window is 64 KB rather
+    // than 44 bytes: a file the walk CAN read must not be re-encoded.
+    const head = wavFile(MONO_16K, 32_000, { extraChunk: "recorder" });
+    expect(cuttable(head, head.length + 32_000)).toBe(true);
+  });
+
+  test("an m4a is not, which is what puts ffmpeg in the path", () => {
+    // An MPEG-4 `ftyp` box — what a phone recording really starts with.
+    const m4a = new Uint8Array([
+      0, 0, 0, 0x20, 0x66, 0x74, 0x79, 0x70, 0x4d, 0x34, 0x41, 0x20, 0, 0, 0, 0,
+    ]);
+    expect(cuttable(m4a, 4_000_000)).toBe(false);
+  });
+
+  test("a WAV whose encoding is not linear PCM is not", () => {
+    // The case an `ffprobe`-based check gets WRONG, which is why the check is
+    // `parseWav` itself: this file reports a PCM codec to ffprobe and is refused
+    // by the parser, so a probe would pass it through and the cut would fail.
+    const extensible = wavFile(MONO_16K, 32_000);
+    new DataView(extensible.buffer).setUint16(20, 0xff_fe, true);
+    expect(cuttable(extensible, 44 + 32_000)).toBe(false);
+  });
+
+  test("a WAV too dense to cut is not, and downsampling is what repairs it", () => {
+    // Past `MAX_BYTES_PER_SECOND`, so `parseWav` refuses it — and a conversion to
+    // 16 kHz mono makes it cuttable, which is a fix the desk gets for free from
+    // asking the parser rather than asking about the codec.
+    const dense = wavFile({ sampleRate: 4_000_000, channels: 8, bitsPerSample: 32 }, 32_000);
+    expect(cuttable(dense, 44 + 32_000)).toBe(false);
+  });
+
+  test("an already-cuttable recording keeps the id it came in under", async () => {
+    // The property that matters: no second upload, so the fan-out reads the file
+    // the caller stored. A step that copied it would double the storage every run
+    // pays for and would still report success.
+    publishRecording(wavFile(MONO_16K, 32_000), "standup.wav");
+    const reporter = stubReporter();
+    try {
+      await expect(normalizeRecording(UPLOAD_ID)).resolves.toEqual({
+        recording: UPLOAD_ID,
+        converted: false,
+      });
+      expect(reporter.lines.join(" ")).toContain("already linear-PCM WAV");
+    } finally {
+      reporter.restore();
+    }
+  });
+
+  test("a conversion that ffmpeg REFUSED is fatal, so it is not attempted five times", () => {
+    // `exit` means ffmpeg read the file and would read it the same way again.
+    expect(() =>
+      classifyFfmpeg(
+        new FfmpegError({
+          kind: "exit",
+          message: "Invalid data found when processing input",
+          binary: "ffmpeg",
+          argv: [],
+          exitCode: 1,
+        }),
+      ),
+    ).toThrow(FatalError);
+  });
+
+  test("a conversion that ran out of time keeps its retries, and its argv", () => {
+    // Rethrown UNCHANGED rather than wrapped, which is what `toStepError` does
+    // with an error carrying no verdict — and the DevKit's default for anything
+    // that is not a `FatalError` is to retry. Asserting the class survives is
+    // asserting the diagnosis does: `argv` is the command you paste into a shell,
+    // and a `new RetryableError(message)` here would throw it away.
+    const timedOut = new FfmpegError({
+      kind: "timeout",
+      message: "timed out",
+      binary: "ffmpeg",
+      argv: ["-i", "source"],
+    });
+    expect(() => classifyFfmpeg(timedOut)).toThrow(timedOut);
+    expect(() => classifyFfmpeg(timedOut)).not.toThrow(FatalError);
+  });
+
+  test("no ffmpeg at all is fatal — a retry cannot install one", () => {
+    // The `aai dev` case. Fatal deliberately: the message already carries the
+    // install instructions, and four more attempts only delay a person reading it.
+    expect(() =>
+      classifyFfmpeg(
+        new FfmpegError({
+          kind: "missing-binary",
+          message: "ffmpeg is not installed",
+          binary: "ffmpeg",
+          argv: [],
+        }),
+      ),
+    ).toThrow(FatalError);
+  });
+
+  test("something that is not an ffmpeg failure at all is fatal", () => {
+    // The store rejecting a write, say. Fatal rather than retryable because this
+    // step's own `maxRetries` exists for the I/O halves that report themselves as
+    // transient; an unrecognized error has said nothing about being worth another
+    // attempt, and guessing yes is how a run burns its budget before failing.
+    expect(() => classifyFfmpeg(new Error("no space left on device"))).toThrow(FatalError);
+  });
+});
+
+describe("expectedSegments", () => {
+  /** A plan over 16 kHz mono, cut into three 90-second segments. */
+  const PLAN = {
+    format: { ...MONO_16K, dataStart: 44, dataEnd: 44 + 270 * 32_000 },
+    segments: [0, 1, 2].map((index) => ({
+      index,
+      start: 44 + index * 90 * 32_000,
+      end: 44 + (index + 1) * 90 * 32_000,
+      startMs: index * 90_000,
+      endMs: (index + 1) * 90_000,
+    })),
+  };
+
+  test("counts every segment once the whole recording has arrived", () => {
+    expect(expectedSegments(PLAN, PLAN.format.dataEnd)).toBe(3);
+  });
+
+  test("ignores segments that start past the end of a SHORT upload", () => {
+    // The failure this guards is a run that never ENDS rather than one that fails:
+    // the plan came from the header's declared length, so a recording that came up
+    // short has segments beginning past the last byte, and waiting for them is
+    // waiting for audio nobody is going to send.
+    expect(expectedSegments(PLAN, 44 + 100 * 32_000)).toBe(2);
+    expect(expectedSegments(PLAN, 44 + 1)).toBe(1);
+  });
+
+  test("an upload with nothing in it expects no segments at all", () => {
+    expect(expectedSegments(PLAN, 0)).toBe(0);
+  });
+});
+
+describe("the conversion, up to the spawn", () => {
+  /**
+   * Reaches the point where ffmpeg would run and stops there, DETERMINISTICALLY.
+   *
+   * `AAI_FFPROBE_PATH` names the binary the SDK resolves, so pointing it at a path
+   * that does not exist produces `kind: "missing-binary"` on every machine — one
+   * where ffmpeg is installed, one where it is not, and CI's Linux leg alike. A
+   * test that instead relied on ffmpeg being ABSENT would pass on a laptop and
+   * behave differently in CI.
+   *
+   * What it covers is the whole step up to the subprocess: reading the header,
+   * deciding the file needs converting, and materializing it to a temp file. Plus
+   * the behaviour a developer actually meets — `aai dev` with no ffmpeg is the one
+   * place dev/prod parity is partial, and it must fail FATALLY with an installable
+   * remedy rather than burn five attempts on a binary that will not appear.
+   */
+  test("materializes the recording, then fails fatally with no ffprobe", async () => {
+    vi.stubEnv("AAI_FFPROBE_PATH", "/nonexistent/aai-test/ffprobe");
+    vi.stubEnv("AAI_FFMPEG_PATH", "/nonexistent/aai-test/ffmpeg");
+    // An m4a `ftyp` box, so `cuttable` says no and the conversion path is taken.
+    publishRecording(
+      new Uint8Array([0, 0, 0, 0x20, 0x66, 0x74, 0x79, 0x70, 0x4d, 0x34, 0x41, 0x20]),
+      "standup.m4a",
+    );
+    const reporter = stubReporter();
+    try {
+      // Fatal, not retryable: four more attempts find the same missing binary, and
+      // the message already carries the install instructions.
+      await expect(normalizeRecording(UPLOAD_ID)).rejects.toThrow(/ffprobe/);
+      await expect(normalizeRecording(UPLOAD_ID)).rejects.toBeInstanceOf(FatalError);
+      // It got as far as deciding the file needs converting — the failure is the
+      // binary, not the input.
+      expect(reporter.lines.join(" ")).toContain("standup.m4a");
+      expect(normalizeRecording.maxRetries).toBe(5);
+    } finally {
+      reporter.restore();
+    }
+  });
+
+  test("leaves no temp directory behind when the conversion fails", async () => {
+    // The `finally`, on the path that matters: a guest's disk is small, and a step
+    // that leaked a directory per failed run would fill it.
+    vi.stubEnv("AAI_FFPROBE_PATH", "/nonexistent/aai-test/ffprobe");
+    const leaked = (names: string[]) => names.filter((n) => n.startsWith("aai-normalize-"));
+    const before = leaked(await readdir(tmpdir()));
+    publishRecording(new Uint8Array([0, 0, 0, 0x20, 0x66, 0x74, 0x79, 0x70]), "standup.m4a");
+    const reporter = stubReporter();
+    try {
+      await expect(normalizeRecording(UPLOAD_ID)).rejects.toBeInstanceOf(FatalError);
+      expect(leaked(await readdir(tmpdir()))).toEqual(before);
+    } finally {
+      reporter.restore();
+    }
+  });
+});

package/dist/templates/transcription-workflow/agent.ts

@@ -21,10 +21,14 @@
  *   `DATABASE_URL` under `aai dev`). REQUIRED here, unlike most workflow apps:
  *   a run survives without it, but an UPLOAD's record is a row, so the form
  *   below refuses by name until storage is on.
- * - **A linear-PCM WAV.** The cutting is arithmetic over byte offsets, which is
- *   only possible on uncompressed audio; `workflows/wav.ts` says so in more
- *   detail, and an unsupported file fails the run by name with the `ffmpeg`
- *   line that fixes it.
+ * - **ffmpeg, under `aai dev` only.** A deployed guest's image installs it; on a
+ *   laptop it is whatever is on `PATH` (or `AAI_FFMPEG_PATH`). The `transcribe`
+ *   flow needs it for anything that is not already a linear-PCM WAV, because the
+ *   cutting is arithmetic over byte offsets and that is only possible on
+ *   uncompressed audio — `workflows/normalize.ts` is the conversion and
+ *   `workflows/wav.ts` is why it has to happen. A WAV needs no ffmpeg at all,
+ *   and `transcribeStream` never uses it: a recording that is still uploading is
+ *   not something a decoder can be pointed at.
  *
  * ## The recording is UPLOADED, and the run carries its id
  *
@@ -49,7 +53,8 @@
  * | client sends | `POST /uploads` | `PUT /uploads/<id>` | `POST /uploads` |
  * | shape | plan, fan out, merge | poll, fan out, merge | submit, poll, read |
  * | client hook | `useWorkflowSubmit` | `useWorkflowStream` | `useWorkflowSubmit` |
- * | accepts | linear-PCM WAV | linear-PCM WAV | **any audio** |
+ * | accepts | **any audio** | linear-PCM WAV | **any audio** |
+ * | converts first | when it must | never | not needed |
  * | segments | 7 for 10 minutes | 7 | **1** |
  *
  * The first two are the same fan-out arranged two ways, and the difference between
@@ -102,7 +107,12 @@ export const transcribe = workflow({
   input: z.object({
     // A plain string, because an upload id is what the run really receives. What
     // makes it a file picker rather than a text box is the `uploads` line below.
-    recording: z.string().describe("A linear-PCM WAV recording (16-bit or 8-bit, any rate)"),
+    //
+    // It says "any recording" now, and that is the whole of what
+    // `workflows/normalize.ts` bought: this used to name linear-PCM WAV and mean
+    // it, so the desk's real front door was a sentence telling the caller to run
+    // ffmpeg themselves. The cut still needs WAV — the run makes one.
+    recording: z.string().describe("Any recording — WAV, MP3, M4A, or a video's audio track"),
   }),
   // The one line that makes the form take a file: `<WorkflowFields>` renders a
   // picker for this property, `useWorkflowSubmit` stores the chosen file, and
@@ -128,6 +138,10 @@ export const transcribe = workflow({
 export const transcribeStream = workflow({
   description: "Transcribe a recording while it is still uploading",
   input: z.object({
+    // Still WAV, and this is the one flow where that is not a limitation to be
+    // fixed: it cuts the recording while the bytes are arriving, and a partial
+    // file is not something ffmpeg can transcode. `transcribe` converts because
+    // it has the whole file before it plans anything.
     recording: z.string().describe("A linear-PCM WAV recording (16-bit or 8-bit, any rate)"),
   }),
   uploads: ["recording"],