@alexkroman1/aai-cli 6.10.1 → 7.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{_agent-DpH2pBJd.mjs → _agent-BTwZJu4P.mjs} +2 -2
- package/dist/{_bundler-C2BDwU7r.mjs → _bundler-B4RqNF5Z.mjs} +2 -2
- package/dist/_dev-env.d.ts +22 -2
- package/dist/{_dev-server-LLLWnFBs.mjs → _dev-server-CiKFH9qw.mjs} +38 -47
- package/dist/_dev-server.d.ts +0 -17
- package/dist/{_init-C57bWAQ8.mjs → _init-CE4HKdgV.mjs} +3 -2
- package/dist/{_preflight-BtfaYtbE.mjs → _preflight-Co1G9Jww.mjs} +1 -1
- package/dist/_preflight.d.ts +1 -1
- package/dist/{_slug-api-BxrsJXVI.mjs → _slug-api-nRFaBEPJ.mjs} +2 -2
- package/dist/{_studio-U3jhKlA3.mjs → _studio-DR5yzZ1T.mjs} +2 -1
- package/dist/{_templates-Dxb_P2Wz.mjs → _templates-FI3xPEMj.mjs} +1 -1
- package/dist/_workflow-scan.d.ts +108 -0
- package/dist/{build-Dfza2pRI.mjs → build-6jfaakgc.mjs} +5 -2
- package/dist/cli.mjs +30 -25
- package/dist/{client-bundler-BEIqgOtd.mjs → client-bundler-CWnG42cU.mjs} +2 -1
- package/dist/client-bundler.mjs +1 -1
- package/dist/{delete-vbpjGzqs.mjs → delete-Dl_LSfvI.mjs} +3 -3
- package/dist/{deploy-B8lubiRT.mjs → deploy-DvYeD8ia.mjs} +5 -5
- package/dist/{dev-Uq5ujP8a.mjs → dev-B59JIfWO.mjs} +1 -1
- package/dist/{eject-31gjtaHF.mjs → eject-C9WJyyr2.mjs} +1 -1
- package/dist/{init-l6xfU4xX.mjs → init-CpJVjEg0.mjs} +3 -3
- package/dist/{login-C71-qz8F.mjs → login-BBuM1sxH.mjs} +3 -4
- package/dist/{logs-BWVGSjLR.mjs → logs-9q-psa4q.mjs} +2 -2
- package/dist/project-config.mjs +1 -1
- package/dist/scaffold/CLAUDE.md +93 -14
- package/dist/scaffold/package.json +5 -3
- package/dist/scaffold/server.mjs +13 -4
- package/dist/scaffold/vite.config.ts +1 -1
- package/dist/{secret-BuMuFR4B.mjs → secret-CVvSLIDV.mjs} +2 -2
- package/dist/{storage-DzRZ-eCw.mjs → storage-BvUrnvM3.mjs} +2 -2
- package/dist/{studio-BCNUpDgP.mjs → studio--MUV0cid.mjs} +4 -4
- package/dist/templates/call-audit/agent.test.ts +818 -0
- package/dist/templates/call-audit/agent.ts +171 -0
- package/dist/templates/call-audit/client.tsx +198 -0
- package/dist/templates/call-audit/workflows/audit.ts +298 -0
- package/dist/templates/call-audit/workflows/ingest.ts +251 -0
- package/dist/templates/call-audit/workflows/media.ts +637 -0
- package/dist/templates/call-audit/workflows/summarize.ts +201 -0
- package/dist/templates/call-audit/workflows/sync-api.ts +44 -0
- package/dist/templates/dispatch-center/agent.test.ts +149 -25
- package/dist/templates/dispatch-center/client.tsx +239 -129
- package/dist/templates/dispatch-center/shared.ts +99 -1
- package/dist/templates/dispatch-center/system-prompt.md +3 -1
- package/dist/templates/dispatch-center/tools/incident_add_note.ts +16 -13
- package/dist/templates/dispatch-center/tools/incident_create.ts +19 -1
- package/dist/templates/dispatch-center/tools/incident_escalate.ts +68 -59
- package/dist/templates/dispatch-center/tools/incident_triage.ts +57 -43
- package/dist/templates/dispatch-center/tools/incident_update_status.ts +49 -40
- package/dist/templates/dispatch-center/tools/ops_run_scenario.ts +8 -2
- package/dist/templates/dispatch-center/tools/resources_dispatch.ts +69 -52
- package/dist/templates/dispatch-center/tools/resources_update_status.ts +45 -41
- package/dist/templates/embedded-assets/agent.test.ts +17 -14
- package/dist/templates/health-assistant/agent.test.ts +22 -4
- package/dist/templates/infocom-adventure/agent.test.ts +4 -6
- package/dist/templates/infocom-adventure/client.tsx +246 -164
- package/dist/templates/link-digest/agent.test.ts +24 -19
- package/dist/templates/link-digest/client.tsx +47 -61
- package/dist/templates/link-digest/workflows/digest.ts +19 -31
- package/dist/templates/night-owl/agent.test.ts +70 -19
- package/dist/templates/night-owl/agent.ts +5 -0
- package/dist/templates/night-owl/client.tsx +56 -56
- package/dist/templates/night-owl/shared.ts +24 -0
- package/dist/templates/night-owl/tools/recommend.ts +18 -6
- package/dist/templates/pizza-ordering/agent.test.ts +27 -27
- package/dist/templates/pizza-ordering/client.tsx +9 -26
- package/dist/templates/plan-and-execute/agent.test.ts +99 -58
- package/dist/templates/plan-and-execute/agent.ts +1 -1
- package/dist/templates/plan-and-execute/client.tsx +12 -15
- package/dist/templates/plan-and-execute/shared.ts +71 -2
- package/dist/templates/plan-and-execute/tools/plan_status.ts +18 -6
- package/dist/templates/plan-and-execute/tools/revise_plan.ts +18 -10
- package/dist/templates/plan-and-execute/tools/start_plan.ts +13 -2
- package/dist/templates/plan-and-execute/tools/work_next_step.ts +70 -22
- package/dist/templates/podcast-digest/agent.test.ts +746 -0
- package/dist/templates/podcast-digest/agent.ts +139 -0
- package/dist/templates/podcast-digest/client.tsx +154 -0
- package/dist/templates/podcast-digest/workflows/digest.ts +411 -0
- package/dist/templates/podcast-digest/workflows/feeds.ts +507 -0
- package/dist/templates/podcast-digest/workflows/slack.ts +209 -0
- package/dist/templates/recap-workflow/agent.test.ts +107 -96
- package/dist/templates/recap-workflow/tools/recap_progress.ts +9 -12
- package/dist/templates/recap-workflow/tools/recap_status.ts +8 -9
- package/dist/templates/recap-workflow/tools/request_recap.ts +2 -1
- package/dist/templates/recap-workflow/workflows/recap.ts +48 -41
- package/dist/templates/redline/agent.test.ts +17 -15
- package/dist/templates/redline/client.tsx +12 -12
- package/dist/templates/redline/workflows/redline.ts +19 -31
- package/dist/templates/research-workflow/agent.test.ts +60 -59
- package/dist/templates/research-workflow/tools/research_progress.ts +9 -12
- package/dist/templates/research-workflow/tools/research_status.ts +9 -11
- package/dist/templates/research-workflow/workflows/research.ts +44 -61
- package/dist/templates/retail/agent.test.ts +26 -23
- package/dist/templates/retail/client.tsx +226 -117
- package/dist/templates/retail/registry.test.ts +38 -6
- package/dist/templates/retail/store.test.ts +82 -15
- package/dist/templates/retail/store.ts +174 -47
- package/dist/templates/retail/system-prompt.md +11 -2
- package/dist/templates/retail/tools/cancel_pending_order.ts +2 -2
- package/dist/templates/retail/tools/exchange_delivered_order_items.ts +2 -2
- package/dist/templates/retail/tools/find_user_id_by_email.ts +8 -5
- package/dist/templates/retail/tools/find_user_id_by_name_zip.ts +4 -5
- package/dist/templates/retail/tools/get_item_details.ts +3 -4
- package/dist/templates/retail/tools/get_order_details.ts +3 -4
- package/dist/templates/retail/tools/get_product_details.ts +3 -4
- package/dist/templates/retail/tools/get_user_details.ts +2 -2
- package/dist/templates/retail/tools/list_all_product_types.ts +5 -5
- package/dist/templates/retail/tools/modify_pending_order_address.ts +2 -2
- package/dist/templates/retail/tools/modify_pending_order_items.ts +2 -3
- package/dist/templates/retail/tools/modify_pending_order_payment.ts +2 -2
- package/dist/templates/retail/tools/modify_user_address.ts +2 -2
- package/dist/templates/retail/tools/return_delivered_order_items.ts +2 -2
- package/dist/templates/retail/tools/transfer_to_human_agents.ts +8 -4
- package/dist/templates/solo-rpg/agent.test.ts +227 -48
- package/dist/templates/solo-rpg/client.tsx +38 -37
- package/dist/templates/solo-rpg/shared.ts +145 -19
- package/dist/templates/solo-rpg/system-prompt.md +3 -2
- package/dist/templates/solo-rpg/tools/action_roll.ts +82 -63
- package/dist/templates/solo-rpg/tools/burn_momentum.ts +85 -50
- package/dist/templates/solo-rpg/tools/check_state.ts +24 -7
- package/dist/templates/solo-rpg/tools/load_game.ts +13 -1
- package/dist/templates/solo-rpg/tools/save_game.ts +16 -7
- package/dist/templates/solo-rpg/tools/setup_character.ts +22 -2
- package/dist/templates/solo-rpg/tools/update_state.ts +113 -100
- package/dist/templates/spoken-summary/agent.test.ts +314 -0
- package/dist/templates/spoken-summary/agent.ts +155 -0
- package/dist/templates/spoken-summary/client.tsx +185 -0
- package/dist/templates/spoken-summary/workflows/summarize.ts +237 -0
- package/dist/templates/spoken-summary/workflows/transcribe.ts +138 -0
- package/dist/templates/support-line/agent.test.ts +11 -16
- package/dist/templates/support-line/agent.ts +1 -1
- package/dist/templates/support-line/client.tsx +9 -9
- package/dist/templates/support-line/nodes.ts +100 -0
- package/dist/templates/support-line/procedure.ts +407 -0
- package/dist/templates/support-line/tools/answer_question.ts +17 -4
- package/dist/templates/transcription-workflow/agent.test.ts +246 -123
- package/dist/templates/transcription-workflow/agent.ts +21 -7
- package/dist/templates/transcription-workflow/client.tsx +17 -42
- package/dist/templates/transcription-workflow/workflows/batch.ts +79 -180
- package/dist/templates/transcription-workflow/workflows/normalize.ts +247 -0
- package/dist/templates/transcription-workflow/workflows/stitch.ts +0 -11
- package/dist/templates/transcription-workflow/workflows/stream.ts +10 -9
- package/dist/templates/transcription-workflow/workflows/sync-api.ts +26 -94
- package/dist/templates/transcription-workflow/workflows/transcribe.ts +55 -40
- package/dist/templates/transcription-workflow/workflows/wav.ts +31 -31
- package/dist/templates/travel-concierge/agent.test.ts +64 -33
- package/dist/templates/travel-concierge/client.tsx +11 -23
- package/dist/templates/travel-concierge/routing.ts +34 -15
- package/dist/templates/travel-concierge/shared.ts +70 -3
- package/dist/templates/travel-concierge/tools/book_car_rental.ts +2 -2
- package/dist/templates/travel-concierge/tools/book_excursion.ts +2 -2
- package/dist/templates/travel-concierge/tools/book_hotel.ts +2 -2
- package/dist/templates/travel-concierge/tools/cancel_ticket.ts +2 -2
- package/dist/templates/travel-concierge/tools/update_ticket.ts +2 -2
- package/dist/{worker-bundler-CGD4r8Kc.mjs → worker-bundler-COxnqstQ.mjs} +221 -3
- package/dist/worker-bundler.mjs +1 -1
- package/dist/{workflow-CFpxOFfQ.mjs → workflow-D2AQf2Pl.mjs} +28 -1
- package/dist/workflow-bundler.d.ts +6 -1
- package/dist/workflow.d.ts +1 -1
- package/package.json +5 -4
- package/dist/templates/support-line/graph.ts +0 -224
- package/dist/{_config-D_s09e7g.mjs → _config-CmJOFsAP.mjs} +1 -1
- /package/dist/templates/plan-and-execute/{graph.ts → procedure.ts} +0 -0
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
// Copyright 2026 the AAI authors. MIT license.
|
|
2
|
+
/**
|
|
3
|
+
* A WORKFLOW APP that goes audio in, audio out: upload a recording and it comes
|
|
4
|
+
* back with a summary you can read AND one you can listen to.
|
|
5
|
+
*
|
|
6
|
+
* `link-digest` is the template to read first: it owns the shape —
|
|
7
|
+
* `workflowApp()`, no session, no tools, a form that starts a run and a page
|
|
8
|
+
* that watches it — and none of that is restated here. `transcription-workflow`
|
|
9
|
+
* owns the other half of the background: uploads, and what it costs to cut a
|
|
10
|
+
* long recording up. What THIS one adds is the return trip.
|
|
11
|
+
*
|
|
12
|
+
* ```text
|
|
13
|
+
* a WAV → transcript → summary → a WAV of the summary
|
|
14
|
+
* async STT LLM Gateway streaming TTS
|
|
15
|
+
* ```
|
|
16
|
+
*
|
|
17
|
+
* ## The last arrow is the one that needed the SDK to grow
|
|
18
|
+
*
|
|
19
|
+
* The first three are ordinary step work. The fourth was impossible until two
|
|
20
|
+
* things existed, and they are what this template is the reference use of:
|
|
21
|
+
*
|
|
22
|
+
* - **`stepSpeak`** (`@alexkroman1/aai/step`) synthesizes from inside a step.
|
|
23
|
+
* The session TTS surface cannot be used here at all: a `TtsSession` is an
|
|
24
|
+
* event stream wired into a live pipeline's playback, with a turn tracker and
|
|
25
|
+
* barge-in behind it, and a step has no turn to be part of and has to return
|
|
26
|
+
* a VALUE.
|
|
27
|
+
* - **`writeUpload`** (same subpath) puts that value where a browser can reach
|
|
28
|
+
* it. A run's OUTPUT is read back as JSON, so audio cannot travel in one —
|
|
29
|
+
* the same rule that keeps a recording's bytes out of a run's INPUT, arriving
|
|
30
|
+
* at the other end of the run.
|
|
31
|
+
*
|
|
32
|
+
* And **`api.download(id)`** is the browser half: the run's output names an
|
|
33
|
+
* upload id, and the page turns it into a `Blob` it can play and offer as a
|
|
34
|
+
* file. `workflows/summarize.ts` carries the rest, including why the model is
|
|
35
|
+
* asked for a spoken script as well as a bullet list.
|
|
36
|
+
*
|
|
37
|
+
* ## What it needs
|
|
38
|
+
*
|
|
39
|
+
* - **`ASSEMBLYAI_API_KEY` in the agent env** — `.env` under `aai dev`,
|
|
40
|
+
* `aai secret put ASSEMBLYAI_API_KEY` once deployed. One key covers all
|
|
41
|
+
* three services this uses: transcription, the LLM Gateway, and the voice.
|
|
42
|
+
* `requiredEnv` below is what makes a deploy check for it rather than letting
|
|
43
|
+
* the first run find out.
|
|
44
|
+
* - **Storage** (`aai storage enable`, Settings → Database in the studio, or
|
|
45
|
+
* `DATABASE_URL` under `aai dev`). REQUIRED here, and more so than for most
|
|
46
|
+
* workflow apps: an upload's record is a row, and this app uses uploads at
|
|
47
|
+
* BOTH ends — the recording coming in and the summary going out.
|
|
48
|
+
*
|
|
49
|
+
* ## The recording is UPLOADED, and the run carries its id
|
|
50
|
+
*
|
|
51
|
+
* A workflow's input is journaled and replayed on every resume, so a
|
|
52
|
+
* recording's BYTES cannot live in it. So the file goes to
|
|
53
|
+
* `POST /workflows/uploads` (the browser does this for you: `uploads` below is
|
|
54
|
+
* what makes `<WorkflowFields>` render a file picker, and `useWorkflowSubmit`
|
|
55
|
+
* stores the file before starting the run), the input carries the returned id,
|
|
56
|
+
* and the step that needs the bytes streams them out with `readUpload`.
|
|
57
|
+
*
|
|
58
|
+
* ## It is scriptable, which is the other half of having an API
|
|
59
|
+
*
|
|
60
|
+
* The page is one caller. Three requests do the whole thing from a shell —
|
|
61
|
+
* upload, start a run, then fetch the summary's audio by the id the run
|
|
62
|
+
* reported:
|
|
63
|
+
*
|
|
64
|
+
* ```sh
|
|
65
|
+
* ID=$(curl -s -X POST "https://<your-agent>/workflows/uploads?name=standup.wav" \
|
|
66
|
+
* -H 'content-type: audio/wav' --data-binary @standup.wav | jq -r .id)
|
|
67
|
+
*
|
|
68
|
+
* OUT=$(curl -s -X POST https://<your-agent>/workflows/runs \
|
|
69
|
+
* -H 'content-type: application/json' \
|
|
70
|
+
* -d "{\"workflow\":\"spokenSummary\",\"wait\":30000,\"input\":{\"recording\":\"$ID\"}}")
|
|
71
|
+
*
|
|
72
|
+
* curl -s "https://<your-agent>/workflows/uploads/$(echo "$OUT" | jq -r .run.output.audio)" \
|
|
73
|
+
* -o summary.wav
|
|
74
|
+
* ```
|
|
75
|
+
*/
|
|
76
|
+
|
|
77
|
+
import { workflow, workflowApp } from "@alexkroman1/aai";
|
|
78
|
+
import { ASSEMBLYAI_TTS_DEFAULT_VOICE, ASSEMBLYAI_TTS_VOICES } from "@alexkroman1/aai/tts";
|
|
79
|
+
import type { WorkflowDef } from "@alexkroman1/aai/workflow-api";
|
|
80
|
+
import { z } from "zod";
|
|
81
|
+
import { type SpokenSummary, spokenSummaryFlow } from "./workflows/summarize.ts";
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The voices the form offers.
|
|
85
|
+
*
|
|
86
|
+
* READ from the SDK's catalog rather than listed, because a wrong voice id is a
|
|
87
|
+
* SILENT failure — it is a free-form string the service rejects in band after
|
|
88
|
+
* the socket is open, so the synthesis simply produces nothing. Narrowed to the
|
|
89
|
+
* English ones because the summary is written in the transcript's language and
|
|
90
|
+
* the prompt does not translate; every voice in the catalog speaks exactly one.
|
|
91
|
+
*/
|
|
92
|
+
const VOICES = Object.entries(ASSEMBLYAI_TTS_VOICES)
|
|
93
|
+
.filter(([, spec]) => spec.language === "en")
|
|
94
|
+
.map(([id]) => id);
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* The same list as a TUPLE, which is what `z.enum` takes.
|
|
98
|
+
*
|
|
99
|
+
* Destructured rather than cast: a `.map` produces an array, and
|
|
100
|
+
* `as [string, ...string[]]` would be a template teaching a cast. The default
|
|
101
|
+
* covers the empty case honestly — a catalog with no English voice falls back
|
|
102
|
+
* to the SDK's own default rather than rendering a picker with no options.
|
|
103
|
+
*/
|
|
104
|
+
const [FIRST_VOICE = ASSEMBLYAI_TTS_DEFAULT_VOICE, ...OTHER_VOICES] = VOICES;
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The run input, as its own const.
|
|
108
|
+
*
|
|
109
|
+
* Named rather than inline because {@link spokenSummary} carries an explicit
|
|
110
|
+
* type, and that annotation is what lets `workflows/summarize.ts` name
|
|
111
|
+
* `WorkflowInputOf<typeof spokenSummary>` for its body's parameter: the body's
|
|
112
|
+
* own signature would otherwise be part of what infers this declaration's type,
|
|
113
|
+
* and TypeScript refuses the cycle (`TS7022`).
|
|
114
|
+
*/
|
|
115
|
+
const spokenSummaryInput = z.object({
|
|
116
|
+
// A plain string, because an upload id is what the run really receives.
|
|
117
|
+
// What makes it a file picker rather than a text box is the `uploads` line
|
|
118
|
+
// below.
|
|
119
|
+
recording: z.string().describe("A recording to summarize — WAV, MP3 or M4A"),
|
|
120
|
+
// An enum, so the form renders a SELECT rather than a text box — which is
|
|
121
|
+
// the whole reason the list is derived above rather than left free-form.
|
|
122
|
+
// Optional, so the SDK's own default voice applies when nobody chooses.
|
|
123
|
+
voice: z
|
|
124
|
+
.enum([FIRST_VOICE, ...OTHER_VOICES])
|
|
125
|
+
.optional()
|
|
126
|
+
.describe("Voice to read the summary in"),
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The declaration: schema, description, and the directive body.
|
|
131
|
+
*
|
|
132
|
+
* Exported so `WorkflowOutputOf<typeof spokenSummary>` names the output type in
|
|
133
|
+
* one place — including from `client.tsx`, where `import type` is erased and so
|
|
134
|
+
* bundles nothing server-side — and so `workflows/summarize.ts` can name
|
|
135
|
+
* `WorkflowInputOf<typeof spokenSummary>` for the body's parameter.
|
|
136
|
+
*/
|
|
137
|
+
export const spokenSummary: WorkflowDef<typeof spokenSummaryInput, SpokenSummary> = workflow({
|
|
138
|
+
description: "Transcribe a recording, summarize it, and read the summary back as audio",
|
|
139
|
+
input: spokenSummaryInput,
|
|
140
|
+
// The one line that makes the form take a file: `<WorkflowFields>` renders a
|
|
141
|
+
// picker for this property, `useWorkflowSubmit` stores the chosen file, and
|
|
142
|
+
// the step that transcribes it reads it back with `readUpload`.
|
|
143
|
+
uploads: ["recording"],
|
|
144
|
+
run: spokenSummaryFlow,
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
export default workflowApp({
|
|
148
|
+
name: "Spoken Summary",
|
|
149
|
+
workflows: { spokenSummary },
|
|
150
|
+
// Checked at deploy time, so a missing key is a warning naming it rather than
|
|
151
|
+
// a run that fails on its second step. A workflow app declares no providers,
|
|
152
|
+
// so this is the only thing that can name the credential its steps read — and
|
|
153
|
+
// this one key covers transcription, the model and the voice alike.
|
|
154
|
+
requiredEnv: ["ASSEMBLYAI_API_KEY"],
|
|
155
|
+
});
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// Copyright 2026 the AAI authors. MIT license.
|
|
2
|
+
/**
|
|
3
|
+
* The page: a form, a progress log, a summary, and a player.
|
|
4
|
+
*
|
|
5
|
+
* `link-digest` shows these primitives raw and `transcription-workflow` shows
|
|
6
|
+
* the form layer in full; neither is restated here. What this page adds is the
|
|
7
|
+
* one thing a workflow app could not do before — **playing a file the RUN
|
|
8
|
+
* produced.**
|
|
9
|
+
*
|
|
10
|
+
* ## An upload id is not a URL, and `api.download` is why
|
|
11
|
+
*
|
|
12
|
+
* The run's output carries `audio`, which is an upload id in the agent's own
|
|
13
|
+
* store. The obvious next line is `<audio src={`/workflows/uploads/${id}`}>`,
|
|
14
|
+
* and it is wrong in a way that only shows up after a deploy: the byte route
|
|
15
|
+
* takes the same `Authorization` header every other route does, and neither
|
|
16
|
+
* `<audio src>` nor `<a href>` can send one. So a page built on a URL works
|
|
17
|
+
* against `aai dev`, where there is no token, and 401s the moment the agent has
|
|
18
|
+
* one.
|
|
19
|
+
*
|
|
20
|
+
* `api.download(id)` reads it with the header and answers a `Blob`;
|
|
21
|
+
* `URL.createObjectURL` turns that into something both elements take. The
|
|
22
|
+
* object URL is REVOKED when the run changes, which is not tidiness — an object
|
|
23
|
+
* URL pins its blob for the life of the document, so a page that summarized
|
|
24
|
+
* five recordings would be holding five files it can no longer reach.
|
|
25
|
+
*
|
|
26
|
+
* ## The form is DECLARED, not written
|
|
27
|
+
*
|
|
28
|
+
* There is no field markup here at all. `<WorkflowFields>` renders a control
|
|
29
|
+
* per property of the workflow's own input schema, read from `GET /workflows` —
|
|
30
|
+
* so the file picker exists because `agent.ts` declares `recording` in
|
|
31
|
+
* `uploads`, and the voice SELECT exists because it declares `voice` as an
|
|
32
|
+
* enum. Adding a field there adds a control here with no edit.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import "@alexkroman1/aai-ui/styles.css";
|
|
36
|
+
// ERASED at build time, so naming the agent's own type costs the browser bundle
|
|
37
|
+
// nothing — and it is what stops this file restating a shape
|
|
38
|
+
// `workflows/summarize.ts` already declares.
|
|
39
|
+
import { formatDuration } from "@alexkroman1/aai/utils";
|
|
40
|
+
import type { WorkflowOutputOf } from "@alexkroman1/aai/workflow-api";
|
|
41
|
+
import {
|
|
42
|
+
createWorkflowApi,
|
|
43
|
+
Form,
|
|
44
|
+
page,
|
|
45
|
+
SubmitButton,
|
|
46
|
+
UploadProgressBar,
|
|
47
|
+
useDownloadUrl,
|
|
48
|
+
useWorkflowSubmit,
|
|
49
|
+
WorkflowFields,
|
|
50
|
+
WorkflowProgress,
|
|
51
|
+
} from "@alexkroman1/aai-ui";
|
|
52
|
+
import type { spokenSummary } from "./agent.ts";
|
|
53
|
+
|
|
54
|
+
/** What a completed run reports, derived from the workflow rather than restated. */
|
|
55
|
+
type Summary = WorkflowOutputOf<typeof spokenSummary>;
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The workflow's name, as a page starts a run by one.
|
|
59
|
+
*
|
|
60
|
+
* A rename in `agent.ts` is a runtime 400 rather than a compile error, which is
|
|
61
|
+
* why `agent.test.ts` pins this string.
|
|
62
|
+
*/
|
|
63
|
+
const WORKFLOW = "spokenSummary";
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Hoisted out of the component deliberately.
|
|
67
|
+
*
|
|
68
|
+
* The hooks hold the client in a ref precisely so a fresh object per render
|
|
69
|
+
* cannot restart their watch, but building one in render is still a new `fetch`
|
|
70
|
+
* closure every time and reads as though it were free.
|
|
71
|
+
*/
|
|
72
|
+
const api = createWorkflowApi();
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The spoken text as a one-cue WebVTT track, inline.
|
|
76
|
+
*
|
|
77
|
+
* A data URL rather than another stored file: the words are already on the page
|
|
78
|
+
* and the whole track is a few hundred bytes, so a second upload — and a second
|
|
79
|
+
* `download` round trip to read it — would buy nothing.
|
|
80
|
+
*/
|
|
81
|
+
function captionsUrl(text: string, durationMs: number): string {
|
|
82
|
+
// `hh:mm:ss.mmm`, which is the only timestamp shape WebVTT accepts.
|
|
83
|
+
const end = new Date(durationMs).toISOString().slice(11, 23);
|
|
84
|
+
const vtt = `WEBVTT\n\n00:00:00.000 --> ${end}\n${text}\n`;
|
|
85
|
+
return `data:text/vtt;charset=utf-8,${encodeURIComponent(vtt)}`;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export function App() {
|
|
89
|
+
// The generic is what makes `run.status === "completed"` narrow to a TYPED
|
|
90
|
+
// `run.output` instead of `unknown`.
|
|
91
|
+
const { submit, run, pending, upload, pauseUpload, resumeUpload, error } =
|
|
92
|
+
useWorkflowSubmit<Summary>(WORKFLOW, { api });
|
|
93
|
+
const output = run?.status === "completed" ? run.output : undefined;
|
|
94
|
+
// `useDownloadUrl` is the SDK's: the byte route takes the agent's bearer, so the
|
|
95
|
+
// bytes have to be FETCHED and handed to the element as an object URL — and the
|
|
96
|
+
// object URL has to be revoked, which is the half a page written by hand forgets.
|
|
97
|
+
const audio = useDownloadUrl(output?.audio, { api });
|
|
98
|
+
|
|
99
|
+
return (
|
|
100
|
+
<main className="mx-auto flex max-w-2xl flex-col gap-6 p-8">
|
|
101
|
+
<header className="flex flex-col gap-1">
|
|
102
|
+
<h1 className="text-2xl font-medium">Spoken Summary</h1>
|
|
103
|
+
<p className="text-sm opacity-70">
|
|
104
|
+
Upload a recording. It comes back summarized — in writing, and read aloud.
|
|
105
|
+
</p>
|
|
106
|
+
</header>
|
|
107
|
+
|
|
108
|
+
<Form onSubmit={submit} error={error} className="flex flex-col gap-4">
|
|
109
|
+
{/* Every control, from the workflow's own input schema. See the module doc. */}
|
|
110
|
+
<WorkflowFields workflow={WORKFLOW} />
|
|
111
|
+
<SubmitButton pending={pending} pendingLabel="Working…">
|
|
112
|
+
Summarize
|
|
113
|
+
</SubmitButton>
|
|
114
|
+
</Form>
|
|
115
|
+
|
|
116
|
+
{/* The upload is its own wait, and the one nothing else can describe: the
|
|
117
|
+
run does not EXIST until the bytes are in, so there is no run id and
|
|
118
|
+
nothing for `<WorkflowProgress>` to read. */}
|
|
119
|
+
<UploadProgressBar upload={upload} onPause={pauseUpload} onResume={resumeUpload} />
|
|
120
|
+
|
|
121
|
+
{/* What the run itself says, from `report()` in the workflow's steps. */}
|
|
122
|
+
<WorkflowProgress runId={run?.runId} api={api} />
|
|
123
|
+
|
|
124
|
+
{run?.status === "failed" && <p className="text-red-600">That one failed: {run.error}</p>}
|
|
125
|
+
|
|
126
|
+
{output !== undefined && (
|
|
127
|
+
<article className="flex flex-col gap-5">
|
|
128
|
+
<div className="flex flex-col gap-1">
|
|
129
|
+
<h2 className="text-xl">{output.headline}</h2>
|
|
130
|
+
<p className="text-sm opacity-70">
|
|
131
|
+
{output.source} · {formatDuration(output.durationMs)} · {output.words} words
|
|
132
|
+
</p>
|
|
133
|
+
</div>
|
|
134
|
+
|
|
135
|
+
<ul className="flex list-disc flex-col gap-1 pl-5">
|
|
136
|
+
{output.points.map((point) => (
|
|
137
|
+
<li key={point}>{point}</li>
|
|
138
|
+
))}
|
|
139
|
+
</ul>
|
|
140
|
+
|
|
141
|
+
<section className="flex flex-col gap-2">
|
|
142
|
+
<h3 className="text-sm font-medium opacity-70">
|
|
143
|
+
Read aloud · {formatDuration(output.audioDurationMs)}
|
|
144
|
+
</h3>
|
|
145
|
+
{audio.pending && <p className="text-sm opacity-70">Fetching the audio…</p>}
|
|
146
|
+
{audio.error !== undefined && (
|
|
147
|
+
<p className="text-red-600">Could not load the audio: {audio.error}</p>
|
|
148
|
+
)}
|
|
149
|
+
{audio.url !== undefined && (
|
|
150
|
+
<>
|
|
151
|
+
<audio controls src={audio.url} className="w-full">
|
|
152
|
+
{/* A real caption track, not a suppression: the summary was
|
|
153
|
+
written before it was spoken, so the words are already
|
|
154
|
+
here and one cue spanning the clip is an honest
|
|
155
|
+
transcript of it. */}
|
|
156
|
+
<track
|
|
157
|
+
kind="captions"
|
|
158
|
+
srcLang="en"
|
|
159
|
+
label="Summary"
|
|
160
|
+
default
|
|
161
|
+
src={captionsUrl(output.spoken, output.audioDurationMs)}
|
|
162
|
+
/>
|
|
163
|
+
</audio>
|
|
164
|
+
{/* `download` works on an object URL because the bytes are
|
|
165
|
+
already in the tab; it is the href that could not carry the
|
|
166
|
+
agent's bearer, not the attribute. */}
|
|
167
|
+
<a href={audio.url} download="summary.wav" className="text-sm underline">
|
|
168
|
+
Download summary.wav
|
|
169
|
+
</a>
|
|
170
|
+
</>
|
|
171
|
+
)}
|
|
172
|
+
<p className="text-sm opacity-70">{output.spoken}</p>
|
|
173
|
+
</section>
|
|
174
|
+
|
|
175
|
+
<details className="text-sm">
|
|
176
|
+
<summary className="cursor-pointer opacity-70">Transcript</summary>
|
|
177
|
+
<p className="mt-2 whitespace-pre-wrap">{output.transcript}</p>
|
|
178
|
+
</details>
|
|
179
|
+
</article>
|
|
180
|
+
)}
|
|
181
|
+
</main>
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
page({ name: "Spoken Summary", component: App });
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
// Copyright 2026 the AAI authors. MIT license.
|
|
2
|
+
/**
|
|
3
|
+
* The workflow body, and the two legs after the transcript: a model reads it,
|
|
4
|
+
* and a voice reads the model back.
|
|
5
|
+
*
|
|
6
|
+
* ```text
|
|
7
|
+
* transcribe (workflows/transcribe.ts) → the words
|
|
8
|
+
* summarize one step, LLM Gateway → a headline, points, and a script
|
|
9
|
+
* speak one step, streaming TTS → a WAV, stored, and its id
|
|
10
|
+
* ```
|
|
11
|
+
*
|
|
12
|
+
* ## Audio in, audio out — and the second half is the part that needed the SDK
|
|
13
|
+
*
|
|
14
|
+
* Reading a file and transcribing it is what `transcription-workflow` already
|
|
15
|
+
* shows. What this template is for is the return trip, which until recently a
|
|
16
|
+
* workflow could not make at all:
|
|
17
|
+
*
|
|
18
|
+
* - **`stepSpeak`** synthesizes from inside a step. The session TTS surface
|
|
19
|
+
* cannot: a `TtsSession` is an event stream wired into a live pipeline's
|
|
20
|
+
* playback, and a step has no turn to be part of and has to return a VALUE.
|
|
21
|
+
* - **`writeUpload`** puts that value somewhere. A run's OUTPUT is read back as
|
|
22
|
+
* JSON, so audio cannot travel in one — the same rule that keeps a
|
|
23
|
+
* recording's bytes out of a run's INPUT, arriving at the other end of the
|
|
24
|
+
* run. The bytes go to the store, the output carries the id, and the page
|
|
25
|
+
* turns it back into something to play with `api.download(id)`.
|
|
26
|
+
*
|
|
27
|
+
* Both are on `@alexkroman1/aai/step`, imported from THERE rather than the
|
|
28
|
+
* root: a `workflows/*.ts` module is bundled separately by the WDK builder, so
|
|
29
|
+
* the root barrel's module graph would ride into the step bundle.
|
|
30
|
+
*
|
|
31
|
+
* ## The model is asked for TWO things, and the difference is the point
|
|
32
|
+
*
|
|
33
|
+
* `points` is for reading and `spoken` is for hearing, and a template that
|
|
34
|
+
* synthesized the bullet list would produce something nobody wants to listen
|
|
35
|
+
* to — a voice reading "one. two. three." with no connective tissue. So the
|
|
36
|
+
* schema asks for a script as well, in sentences, and that is what
|
|
37
|
+
* {@link speak} is handed. It is the same decision `recap-workflow` makes for
|
|
38
|
+
* the sentence it reads down a phone, and it is one prompts get wrong when the
|
|
39
|
+
* shape does not force it.
|
|
40
|
+
*
|
|
41
|
+
* ## Why each leg is its own step
|
|
42
|
+
*
|
|
43
|
+
* They fail differently and cost differently. The transcription is minutes of a
|
|
44
|
+
* provider's queue; the model call is seconds and rate-limited; the synthesis
|
|
45
|
+
* is a socket. Splitting them means a rate-limited model call replays the
|
|
46
|
+
* transcript from the journal instead of transcribing the recording again, and
|
|
47
|
+
* a synthesis that failed does not re-run the model — which is the ordinary
|
|
48
|
+
* reason to split steps, made sharp here because the first leg is the
|
|
49
|
+
* expensive one.
|
|
50
|
+
*/
|
|
51
|
+
|
|
52
|
+
import { report, stepSpeak, TRANSCRIBE_API, writeUpload } from "@alexkroman1/aai/step";
|
|
53
|
+
import { stepGenerateJsonClassified } from "@alexkroman1/aai/step-errors";
|
|
54
|
+
import { countWords, omitUndefined } from "@alexkroman1/aai/utils";
|
|
55
|
+
// ERASED at build time, so the body can name the schema's own output type without
|
|
56
|
+
// a runtime cycle back through `agent.ts` — the same mechanism `client.tsx` uses
|
|
57
|
+
// for `WorkflowOutputOf`.
|
|
58
|
+
import type { WorkflowInputOf } from "@alexkroman1/aai/workflow-api";
|
|
59
|
+
import { sleep } from "workflow";
|
|
60
|
+
import { z } from "zod";
|
|
61
|
+
import type { spokenSummary } from "../agent.ts";
|
|
62
|
+
import {
|
|
63
|
+
createJob,
|
|
64
|
+
MAX_POLLS,
|
|
65
|
+
POLL_INTERVAL,
|
|
66
|
+
pollTranscript,
|
|
67
|
+
type Transcript,
|
|
68
|
+
uploadToProvider,
|
|
69
|
+
} from "./transcribe.ts";
|
|
70
|
+
|
|
71
|
+
/** Points the summary is reduced to. Enough to be a summary, few enough to scan. */
|
|
72
|
+
const POINTS = 4;
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Characters of transcript handed to the model.
|
|
76
|
+
*
|
|
77
|
+
* The pass-an-id-not-a-payload rule meeting a case where the payload IS the
|
|
78
|
+
* work: the text has to cross the queue between two steps, so it is bounded
|
|
79
|
+
* rather than trusted. 40k characters is roughly four hours of speech — past
|
|
80
|
+
* where another paragraph changes a four-point summary.
|
|
81
|
+
*/
|
|
82
|
+
const MAX_TRANSCRIPT_CHARS = 40_000;
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* The shape the model must answer in, as something that CHECKS.
|
|
86
|
+
*
|
|
87
|
+
* `stepGenerateJson` validates against this and throws plainly when the reply
|
|
88
|
+
* misses, which is what a retry is for: a model that answered with prose may
|
|
89
|
+
* well obey on the next attempt.
|
|
90
|
+
*/
|
|
91
|
+
const SummaryReply = z.object({
|
|
92
|
+
headline: z.string().trim().min(1),
|
|
93
|
+
points: z.array(z.string().trim().min(1)).min(1),
|
|
94
|
+
// NOT `.default("")` — the whole second half of this workflow has nothing to
|
|
95
|
+
// say without it, and a default would turn a missing field into a silent
|
|
96
|
+
// half-second of audio rather than a retry.
|
|
97
|
+
spoken: z.string().trim().min(1),
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
/** What a finished run reports. Small and JSON-shaped, like every step result. */
|
|
101
|
+
export type SpokenSummary = {
|
|
102
|
+
/** The uploaded file's own name. */
|
|
103
|
+
source: string;
|
|
104
|
+
/** The recording's length, as the provider measured it. */
|
|
105
|
+
durationMs: number;
|
|
106
|
+
/** Words in the transcript. */
|
|
107
|
+
words: number;
|
|
108
|
+
/** One line naming what the recording was about. */
|
|
109
|
+
headline: string;
|
|
110
|
+
/** The summary, for reading. */
|
|
111
|
+
points: string[];
|
|
112
|
+
/** The summary, for hearing — what {@link speak} was handed. */
|
|
113
|
+
spoken: string;
|
|
114
|
+
/** The whole transcript, so the page can show its work. */
|
|
115
|
+
transcript: string;
|
|
116
|
+
/**
|
|
117
|
+
* The upload id of the spoken summary — a WAV, in this app's own store.
|
|
118
|
+
*
|
|
119
|
+
* An ID rather than the bytes, and that is the rule rather than a
|
|
120
|
+
* preference: a run's output is read back as JSON. `api.download(id)` is the
|
|
121
|
+
* browser half.
|
|
122
|
+
*/
|
|
123
|
+
audio: string;
|
|
124
|
+
/** How long the spoken summary lasts. */
|
|
125
|
+
audioDurationMs: number;
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
/** Transcribe a recording, summarize it, and read the summary back. */
|
|
129
|
+
export async function spokenSummaryFlow(
|
|
130
|
+
input: WorkflowInputOf<typeof spokenSummary>,
|
|
131
|
+
): Promise<SpokenSummary> {
|
|
132
|
+
"use workflow";
|
|
133
|
+
|
|
134
|
+
const transcript = await transcribe(input.recording);
|
|
135
|
+
const summary = await summarize(transcript.text);
|
|
136
|
+
const spoken = await speak(summary.spoken, input.voice);
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
source: transcript.source,
|
|
140
|
+
durationMs: transcript.durationMs,
|
|
141
|
+
words: countWords(transcript.text),
|
|
142
|
+
headline: summary.headline,
|
|
143
|
+
points: summary.points,
|
|
144
|
+
spoken: summary.spoken,
|
|
145
|
+
transcript: transcript.text,
|
|
146
|
+
audio: spoken.audio,
|
|
147
|
+
audioDurationMs: spoken.durationMs,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* The whole first leg, factored out of the body.
|
|
153
|
+
*
|
|
154
|
+
* A plain async function rather than a step, and NOT because it is small: it
|
|
155
|
+
* calls steps and it `sleep`s durably between polls, neither of which a step
|
|
156
|
+
* may do. So it runs as part of the BODY and is replayed with it — which is
|
|
157
|
+
* legal here for the ordinary reason, that everything it does is either a step
|
|
158
|
+
* call or a `sleep`, so a replay re-derives exactly the same sequence.
|
|
159
|
+
*/
|
|
160
|
+
async function transcribe(recording: string): Promise<Transcript> {
|
|
161
|
+
const { audioUrl } = await uploadToProvider(recording);
|
|
162
|
+
const job = await createJob(audioUrl);
|
|
163
|
+
|
|
164
|
+
for (let poll = 0; poll < MAX_POLLS; poll += 1) {
|
|
165
|
+
const progress = await pollTranscript(recording, job.id);
|
|
166
|
+
if (progress.done) return progress.transcript;
|
|
167
|
+
await sleep(POLL_INTERVAL);
|
|
168
|
+
}
|
|
169
|
+
// A plain throw: this is the BODY, where the fatal/retryable distinction has
|
|
170
|
+
// nothing to apply to. The transcript is not lost, so the message says where
|
|
171
|
+
// it is rather than only that the wait ran out.
|
|
172
|
+
throw new Error(
|
|
173
|
+
`Transcript ${job.id} was still unfinished after ${MAX_POLLS} polls. It is not lost — ` +
|
|
174
|
+
`read it directly with GET ${TRANSCRIBE_API}/v2/transcript/${job.id}.`,
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Reduce the transcript to a headline, {@link POINTS} points, and a script. */
|
|
179
|
+
export async function summarize(
|
|
180
|
+
text: string,
|
|
181
|
+
): Promise<{ headline: string; points: string[]; spoken: string }> {
|
|
182
|
+
"use step";
|
|
183
|
+
|
|
184
|
+
await report("Summarizing the transcript.");
|
|
185
|
+
const reply = await stepGenerateJsonClassified(
|
|
186
|
+
"Summarize this transcript of a recording.\n\n" +
|
|
187
|
+
"Answer with JSON only, in this shape:\n" +
|
|
188
|
+
`{"headline": "...", "points": ["..."], "spoken": "..."}\n\n` +
|
|
189
|
+
"- headline: one line naming what the recording was about.\n" +
|
|
190
|
+
`- points: at most ${POINTS} short points, each a complete thought. Concrete ` +
|
|
191
|
+
`specifics — decisions, numbers, names, what happens next — never "the ` +
|
|
192
|
+
`speaker discussed several topics".\n` +
|
|
193
|
+
"- spoken: the same summary written to be READ ALOUD. Full sentences that " +
|
|
194
|
+
"flow, under 120 words, no bullet markers, no headings, no markdown. " +
|
|
195
|
+
"Someone will hear this without seeing the points.\n\n" +
|
|
196
|
+
`Transcript:\n${text.slice(0, MAX_TRANSCRIPT_CHARS)}`,
|
|
197
|
+
{
|
|
198
|
+
system: "You summarize recordings. You answer with JSON and nothing else.",
|
|
199
|
+
schema: SummaryReply,
|
|
200
|
+
},
|
|
201
|
+
// The `Classified` caller is `stepGenerateJson` plus `throwStepError`, which
|
|
202
|
+
// reads the gateway's own status: a 429 is worth another attempt and a 400
|
|
203
|
+
// is not, and that is what tells the DevKit which.
|
|
204
|
+
);
|
|
205
|
+
|
|
206
|
+
return { headline: reply.headline, points: reply.points.slice(0, POINTS), spoken: reply.spoken };
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Read the summary aloud, store the WAV, and answer with its id.
|
|
211
|
+
*
|
|
212
|
+
* **Both halves belong in ONE step**, and the reason is what a journal records:
|
|
213
|
+
* a step is replayed by its RETURN VALUE, so an id is replayed and bytes are
|
|
214
|
+
* not. Split in two, the audio would have to cross the queue between them —
|
|
215
|
+
* megabytes of it, on every resume. Together, a resumed run replays the id and
|
|
216
|
+
* re-reads a file that is already there.
|
|
217
|
+
*/
|
|
218
|
+
export async function speak(
|
|
219
|
+
script: string,
|
|
220
|
+
voice?: string,
|
|
221
|
+
): Promise<{ audio: string; durationMs: number }> {
|
|
222
|
+
"use step";
|
|
223
|
+
|
|
224
|
+
const spoken = await stepSpeak(script, omitUndefined({ voice }));
|
|
225
|
+
const stored = await writeUpload(spoken.audio, {
|
|
226
|
+
// Named, because this is what a person sees on the download link rather
|
|
227
|
+
// than an opaque id — and typed, because the byte route serves the type it
|
|
228
|
+
// was given and a browser will not play a file it was handed as bytes.
|
|
229
|
+
name: "summary.wav",
|
|
230
|
+
type: "audio/wav",
|
|
231
|
+
});
|
|
232
|
+
|
|
233
|
+
await report(
|
|
234
|
+
`Recorded a ${Math.round(spoken.durationMs / 1000)}s summary in ${spoken.voice}'s voice.`,
|
|
235
|
+
);
|
|
236
|
+
return { audio: stored.id, durationMs: spoken.durationMs };
|
|
237
|
+
}
|