@warlock.js/ai 4.5.0 → 4.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/CHANGELOG.md +14 -0
  2. package/cjs/index.cjs +20 -1
  3. package/cjs/{src-DFibP2FQ.cjs → src-Bmajk4Qg.cjs} +1 -1
  4. package/cjs/{src-C02yzsLs.cjs → src-OZyDYHxm.cjs} +2789 -691
  5. package/cjs/src-OZyDYHxm.cjs.map +1 -0
  6. package/esm/agent/agent-config.type.d.mts +29 -0
  7. package/esm/agent/agent-config.type.d.mts.map +1 -1
  8. package/esm/agent/agent.d.mts.map +1 -1
  9. package/esm/agent/agent.mjs +126 -7
  10. package/esm/agent/agent.mjs.map +1 -1
  11. package/esm/agent/signature.mjs +57 -0
  12. package/esm/agent/signature.mjs.map +1 -0
  13. package/esm/agent/snapshot.mjs +101 -0
  14. package/esm/agent/snapshot.mjs.map +1 -0
  15. package/esm/ai-openai/src/image.mjs +5 -0
  16. package/esm/ai-openai/src/index.mjs +3 -0
  17. package/esm/ai-openai/src/sdk.mjs +3 -0
  18. package/esm/ai-openai/src/speech.mjs +5 -0
  19. package/esm/ai-openai/src/transcription.mjs +6 -0
  20. package/esm/ai-openai/src/utils/index.mjs +1 -0
  21. package/esm/ai-openai/src/utils/to-openai-messages.mjs +3 -0
  22. package/esm/ai.d.mts +45 -0
  23. package/esm/ai.d.mts.map +1 -1
  24. package/esm/ai.mjs +37 -1
  25. package/esm/ai.mjs.map +1 -1
  26. package/esm/contracts/agent/agent-options.type.d.mts +22 -2
  27. package/esm/contracts/agent/agent-options.type.d.mts.map +1 -1
  28. package/esm/contracts/agent/agent-snapshot.type.d.mts +90 -0
  29. package/esm/contracts/agent/agent-snapshot.type.d.mts.map +1 -0
  30. package/esm/contracts/agent/agent.contract.d.mts +29 -1
  31. package/esm/contracts/agent/agent.contract.d.mts.map +1 -1
  32. package/esm/contracts/agent/index.d.mts +2 -1
  33. package/esm/contracts/image-model.contract.d.mts +156 -0
  34. package/esm/contracts/image-model.contract.d.mts.map +1 -0
  35. package/esm/contracts/index.d.mts +8 -3
  36. package/esm/contracts/planner/index.d.mts +3 -2
  37. package/esm/contracts/planner/planner-config.type.d.mts +30 -0
  38. package/esm/contracts/planner/planner-config.type.d.mts.map +1 -1
  39. package/esm/contracts/planner/planner-execute-options.type.d.mts +13 -1
  40. package/esm/contracts/planner/planner-execute-options.type.d.mts.map +1 -1
  41. package/esm/contracts/planner/planner-snapshot.type.d.mts +77 -0
  42. package/esm/contracts/planner/planner-snapshot.type.d.mts.map +1 -0
  43. package/esm/contracts/planner/planner.contract.d.mts +21 -1
  44. package/esm/contracts/planner/planner.contract.d.mts.map +1 -1
  45. package/esm/contracts/result/base-report.type.d.mts +1 -1
  46. package/esm/contracts/result/base-report.type.d.mts.map +1 -1
  47. package/esm/contracts/result/base-report.type.mjs.map +1 -1
  48. package/esm/contracts/sdk-adapter.contract.d.mts +37 -0
  49. package/esm/contracts/sdk-adapter.contract.d.mts.map +1 -1
  50. package/esm/contracts/speech-model.contract.d.mts +97 -0
  51. package/esm/contracts/speech-model.contract.d.mts.map +1 -0
  52. package/esm/contracts/transcription-model.contract.d.mts +101 -0
  53. package/esm/contracts/transcription-model.contract.d.mts.map +1 -0
  54. package/esm/errors/agent-drift-error.d.mts +32 -0
  55. package/esm/errors/agent-drift-error.d.mts.map +1 -0
  56. package/esm/errors/agent-drift-error.mjs +31 -0
  57. package/esm/errors/agent-drift-error.mjs.map +1 -0
  58. package/esm/errors/error-code.type.d.mts +1 -1
  59. package/esm/errors/index.d.mts +2 -0
  60. package/esm/errors/index.mjs +2 -0
  61. package/esm/errors/planner-drift-error.d.mts +34 -0
  62. package/esm/errors/planner-drift-error.d.mts.map +1 -0
  63. package/esm/errors/planner-drift-error.mjs +33 -0
  64. package/esm/errors/planner-drift-error.mjs.map +1 -0
  65. package/esm/image/image-cost.d.mts +32 -0
  66. package/esm/image/image-cost.d.mts.map +1 -0
  67. package/esm/image/image-cost.mjs +55 -0
  68. package/esm/image/image-cost.mjs.map +1 -0
  69. package/esm/image/image.d.mts +92 -0
  70. package/esm/image/image.d.mts.map +1 -0
  71. package/esm/image/image.mjs +113 -0
  72. package/esm/image/image.mjs.map +1 -0
  73. package/esm/image/index.mjs +4 -0
  74. package/esm/index.d.mts +26 -4
  75. package/esm/index.mjs +20 -1
  76. package/esm/mock/index.d.mts +3 -0
  77. package/esm/mock/index.mjs +3 -0
  78. package/esm/mock/mock-config.type.d.mts +22 -0
  79. package/esm/mock/mock-config.type.d.mts.map +1 -1
  80. package/esm/mock/mock-image-model.d.mts +41 -0
  81. package/esm/mock/mock-image-model.d.mts.map +1 -0
  82. package/esm/mock/mock-image-model.mjs +52 -0
  83. package/esm/mock/mock-image-model.mjs.map +1 -0
  84. package/esm/mock/mock-sdk.d.mts +7 -1
  85. package/esm/mock/mock-sdk.d.mts.map +1 -1
  86. package/esm/mock/mock-sdk.mjs +27 -0
  87. package/esm/mock/mock-sdk.mjs.map +1 -1
  88. package/esm/mock/mock-speech-model.d.mts +31 -0
  89. package/esm/mock/mock-speech-model.d.mts.map +1 -0
  90. package/esm/mock/mock-speech-model.mjs +39 -0
  91. package/esm/mock/mock-speech-model.mjs.map +1 -0
  92. package/esm/mock/mock-transcription-model.d.mts +32 -0
  93. package/esm/mock/mock-transcription-model.d.mts.map +1 -0
  94. package/esm/mock/mock-transcription-model.mjs +36 -0
  95. package/esm/mock/mock-transcription-model.mjs.map +1 -0
  96. package/esm/planner/planner-run.d.mts +8 -0
  97. package/esm/planner/planner-run.d.mts.map +1 -1
  98. package/esm/planner/planner-run.mjs +161 -6
  99. package/esm/planner/planner-run.mjs.map +1 -1
  100. package/esm/planner/planner.d.mts.map +1 -1
  101. package/esm/planner/planner.mjs +25 -1
  102. package/esm/planner/planner.mjs.map +1 -1
  103. package/esm/planner/snapshot.mjs +95 -0
  104. package/esm/planner/snapshot.mjs.map +1 -0
  105. package/esm/rag/index.d.mts +7 -0
  106. package/esm/rag/index.mjs +7 -0
  107. package/esm/rag/loaders/errors.d.mts +19 -0
  108. package/esm/rag/loaders/errors.d.mts.map +1 -0
  109. package/esm/rag/loaders/errors.mjs +25 -0
  110. package/esm/rag/loaders/errors.mjs.map +1 -0
  111. package/esm/rag/loaders/index.mjs +7 -0
  112. package/esm/rag/loaders/load-html.d.mts +26 -0
  113. package/esm/rag/loaders/load-html.d.mts.map +1 -0
  114. package/esm/rag/loaders/load-html.mjs +138 -0
  115. package/esm/rag/loaders/load-html.mjs.map +1 -0
  116. package/esm/rag/loaders/load-pdf.d.mts +38 -0
  117. package/esm/rag/loaders/load-pdf.d.mts.map +1 -0
  118. package/esm/rag/loaders/load-pdf.mjs +150 -0
  119. package/esm/rag/loaders/load-pdf.mjs.map +1 -0
  120. package/esm/rag/loaders/load-text.d.mts +47 -0
  121. package/esm/rag/loaders/load-text.d.mts.map +1 -0
  122. package/esm/rag/loaders/load-text.mjs +60 -0
  123. package/esm/rag/loaders/load-text.mjs.map +1 -0
  124. package/esm/rag/loaders/load-web.d.mts +42 -0
  125. package/esm/rag/loaders/load-web.d.mts.map +1 -0
  126. package/esm/rag/loaders/load-web.mjs +89 -0
  127. package/esm/rag/loaders/load-web.mjs.map +1 -0
  128. package/esm/rag/loaders/loader.type.d.mts +89 -0
  129. package/esm/rag/loaders/loader.type.d.mts.map +1 -0
  130. package/esm/rag/store/pg-vector-store.d.mts +139 -0
  131. package/esm/rag/store/pg-vector-store.d.mts.map +1 -0
  132. package/esm/rag/store/pg-vector-store.mjs +328 -0
  133. package/esm/rag/store/pg-vector-store.mjs.map +1 -0
  134. package/esm/speech/index.mjs +3 -0
  135. package/esm/speech/speech.d.mts +65 -0
  136. package/esm/speech/speech.d.mts.map +1 -0
  137. package/esm/speech/speech.mjs +123 -0
  138. package/esm/speech/speech.mjs.map +1 -0
  139. package/esm/supervisor/entries.mjs +2 -2
  140. package/esm/supervisor/entries.mjs.map +1 -1
  141. package/esm/transcribe/audio-input.d.mts +47 -0
  142. package/esm/transcribe/audio-input.d.mts.map +1 -0
  143. package/esm/transcribe/audio-input.mjs +84 -0
  144. package/esm/transcribe/audio-input.mjs.map +1 -0
  145. package/esm/transcribe/index.mjs +4 -0
  146. package/esm/transcribe/transcribe.d.mts +64 -0
  147. package/esm/transcribe/transcribe.d.mts.map +1 -0
  148. package/esm/transcribe/transcribe.mjs +128 -0
  149. package/esm/transcribe/transcribe.mjs.map +1 -0
  150. package/llms-full.txt +753 -0
  151. package/llms.txt +5 -0
  152. package/package.json +3 -3
  153. package/skills/README.md +4 -0
  154. package/skills/durable-agent-runs/SKILL.md +135 -0
  155. package/skills/generate-images/SKILL.md +138 -0
  156. package/skills/generate-speech/SKILL.md +139 -0
  157. package/skills/rag-loaders-and-stores/SKILL.md +164 -0
  158. package/skills/transcribe-audio/SKILL.md +157 -0
  159. package/cjs/src-C02yzsLs.cjs.map +0 -1
package/llms-full.txt CHANGED
@@ -951,6 +951,145 @@ npm install openai
951
951
  - [`@warlock.js/ai-guard/escalate-block-to-human/SKILL.md`](@warlock.js/ai-guard/escalate-block-to-human/SKILL.md) — escalating a hard `block` to a human-review surface.
952
952
 
953
953
 
954
+ ## durable-agent-runs `@warlock.js/ai/durable-agent-runs/SKILL.md`
955
+
956
+ ---
957
+ name: durable-agent-runs
958
+ description: 'Mid-run crash-resume for agents AND planners — opt in with durable: { store, deleteOnComplete? } on the config, pass a stable runId to execute(), and call agent.resume(runId) / planner.resume(runId) after a crash to continue from the last settled trip / plan node. Reuses the ai.snapshot.{memory,pg,redis} stores; checkpoints per-trip (agent) / per-node (planner); completed trips + nodes never re-run their tools and usage is never double-counted; a drifted definition throws AgentDriftError / PlannerDriftError (bypass with { force: true }). Triggers: `durable`, `agent.resume`, `planner.resume`, `resume(runId)`, `runId`, `AgentSnapshot`, `PlannerSnapshot`, `AgentSnapshotStatus`, `PlannerSnapshotStatus`, `AgentDriftError`, `PlannerDriftError`, `computeAgentSignature`, `agent.signature`, `deleteOnComplete`, `defaultSnapshotStore`, `ai.snapshot.pg`, `ai.snapshot.memory`, `SnapshotStore`, `force: true`; ''resume an agent after a crash'', ''durable agent run'', ''continue a planner from where it crashed'', ''checkpoint agent state'', ''idempotent tool re-run on resume'', ''signature drift on resume''; typical import `import { ai } from "@warlock.js/ai"`. Skip: durable human-in-the-loop approval resume (ai.human.resume of a PendingInterrupt) — `@warlock.js/ai/durable-resume/SKILL.md`; supervisor/workflow iterate-mid-turn snapshot resume + the store contracts themselves — `@warlock.js/ai/manage-ai-stores/SKILL.md`; competing libs `temporal`, `inngest`, `restate`.'
959
+ ---
960
+
961
+ # Durable agent + planner runs — resume from the last checkpoint
962
+
963
+ Opt-in mid-run crash-resume for the two long-running primitives. Turn it on, give the run a stable `runId`, and after a process crash `resume(runId)` re-hydrates the persisted state and continues from where it stopped — never re-issuing a settled trip's model call or re-invoking a completed node's capability.
964
+
965
+ > **Not the same as [[durable-resume]].** That skill is `ai.human.resume(interruptId, decision)` — resuming a **gated tool call** hours later after a human rules (a `PendingInterrupt` in an `InterruptStore`). *This* skill is **crash-resume of an in-flight run** (an `AgentSnapshot` / `PlannerSnapshot` in a `SnapshotStore`): the process died mid-run, you restart, and continue the same trip / plan. Different trigger (a crash, not a human), different store, different verb (`agent.resume` / `planner.resume`, not `ai.human.resume`).
966
+
967
+ ## Opt in — `durable` on the config
968
+
969
+ ```ts
970
+ import { ai } from "@warlock.js/ai";
971
+
972
+ const writer = ai.agent({
973
+ name: "writer",
974
+ model,
975
+ tools: [searchTool, draftTool],
976
+ durable: {
977
+ store: ai.snapshot.pg({ client: pgPool }), // reuses the ai.snapshot.* stores
978
+ deleteOnComplete: false, // default — keep for the completed-run short-circuit + audit
979
+ },
980
+ });
981
+ ```
982
+
983
+ `durable` shape (identical on the agent and planner config):
984
+
985
+ - **`store?`** — a `SnapshotStore`. Falls back to `ai.config({ defaultSnapshotStore })`. When neither resolves, snapshot writes **silently skip** and `resume()` throws.
986
+ - **`deleteOnComplete?`** — drop the snapshot once the run completes successfully. Default `false`.
987
+
988
+ **Absent `durable` ⇒ zero behavior change** — the loop starts at trip 0 / the first node, never writes a snapshot, and runs byte-for-byte as before.
989
+
990
+ ## Run with a stable `runId`, then resume
991
+
992
+ The `runId` is the store key. Pass a stable one to `execute()` (or read the generated one off `result.report.runId`) so a later `resume()` can find the snapshot:
993
+
994
+ ```ts
995
+ const result = await writer.execute("research X", { runId: "run-42" });
996
+
997
+ // ...process crashes mid-run, restarts...
998
+
999
+ const recovered = await writer.resume("run-42");
1000
+ // continues from the next unsettled trip; `recovered.report.status === "completed"`
1001
+ ```
1002
+
1003
+ Planners are the mirror image — `durable` on the config, `runId` on `execute(goal)`, `planner.resume(runId)`:
1004
+
1005
+ ```ts
1006
+ const research = ai.planner({
1007
+ name: "research-assistant",
1008
+ model,
1009
+ capabilities: [{ name: "search", executable: searchAgent }, { name: "write", executable: writerAgent }],
1010
+ durable: { store: ai.snapshot.pg({ client: pgPool }) },
1011
+ });
1012
+
1013
+ const first = await research.execute("compare A vs B", { runId: "plan-7" });
1014
+ // ...crash...
1015
+ const done = await research.resume("plan-7");
1016
+ ```
1017
+
1018
+ ## Checkpoint granularity
1019
+
1020
+ | Primitive | Written | Contains | Resume continues at |
1021
+ |---|---|---|---|
1022
+ | **agent** | after every settled **trip** (`runTrip` end) | `messages`, `trips`, `toolCalls`, `usage`, resolved `systemPrompt` / `responseSchema`, `signature`, `status` | `trips.length` (the next trip index) |
1023
+ | **planner** | after every settled **plan node** (`executeStep` end) | the frozen `plan`, `executedSteps` ledger, `usage`, child `children` reports, `replanCount`, `signature`, `status` | the unfinished frontier (from `executedSteps`) |
1024
+
1025
+ The write happens only where the persisted arrays are mutually consistent — for the agent, after every tool a trip requested has been dispatched and its result appended. A crash **mid-trip** loses only that in-flight trip (never checkpointed), which the resume re-issues cleanly. The planner **never re-calls the planning LLM** on resume — the plan is frozen on the first run; re-asking would burn tokens and risk a plan that no longer matches the ledger. Every field on both snapshots is JSON-serializable, so they round-trip through any `ai.snapshot.{memory,pg,redis}` backend verbatim.
1026
+
1027
+ ## Idempotency — what does and doesn't re-run
1028
+
1029
+ ```ts
1030
+ // Completed run: resume is a no-op that re-returns the stored result.
1031
+ const again = await writer.resume("run-42"); // runs nothing when status === "completed"
1032
+ ```
1033
+
1034
+ - **Completed trips / nodes never re-run their tools.** On agent resume, `trips.length` is the starting trip index — earlier trips' model calls are not replayed and their tool dispatches are not re-invoked. On planner resume, a completed node's capability dispatch is skipped (the sequential skip-guard / DAG re-seed derive the completed set from `executedSteps`).
1035
+ - **Usage is never double-counted.** The running `usage` total is restored from the snapshot; only the newly-executed trips / nodes add to it.
1036
+ - **Caveat — a crash MID-trip re-runs that trip's tools.** The in-flight trip was never checkpointed, so on resume its tools fire again. **Side-effectful tools (charging a card, sending an email) must be idempotent** — the same caller-responsibility boundary the supervisor and workflow primitives document. Guard them with your own dedupe key (e.g. `${runId}:${toolCallId}`).
1037
+
1038
+ ## Drift — definition changed since the snapshot
1039
+
1040
+ Every agent / planner carries a structural `signature` (`agent.signature` — computed at factory time by `computeAgentSignature`), stamped on each snapshot. `resume()` compares the stored signature against the current definition; a mismatch throws before executing anything:
1041
+
1042
+ - **agent** covers: model name + provider, sorted tool names, `maxTrips`, whether a default `output` schema is set, `version`. It does **not** cover system-prompt text, middleware, per-event handlers, placeholders, or `modelOptions` — runtime knobs that don't change a resumable run's shape.
1043
+ - **planner** covers: name + ordered capability names. A mid-run **re-plan is NOT drift** (the plan changed, not the definition); `replanCount` is persisted so the replan budget survives a resume.
1044
+
1045
+ ```ts
1046
+ import { AgentDriftError } from "@warlock.js/ai";
1047
+
1048
+ try {
1049
+ await writer.resume("run-42");
1050
+ } catch (error) {
1051
+ if (error instanceof AgentDriftError) {
1052
+ // The definition changed (a tool was added, the model swapped). Either roll the
1053
+ // definition back, or — only when you've verified the change is snapshot-safe:
1054
+ await writer.resume("run-42", { force: true }); // bypasses the drift check
1055
+ }
1056
+ }
1057
+ ```
1058
+
1059
+ `{ force: true }` is the escape hatch (mirror `PlannerDriftError` for planners). `resume()` also throws `AgentExecutionError` / `PlannerFailedError` when no store is configured or no snapshot exists for the `runId`.
1060
+
1061
+ ## Pattern — a boot-drain resume loop
1062
+
1063
+ On restart, resume every run the store still has in flight. Snapshots carry a `status` (`"running" | "completed" | "cancelled" | "failed"`), so you only resume the live ones:
1064
+
1065
+ ```ts
1066
+ const store = ai.snapshot.pg({ client: pgPool });
1067
+ const runIds = (await store.list?.()) ?? [];
1068
+
1069
+ for (const runId of runIds) {
1070
+ const snapshot = await store.load(runId);
1071
+ if (snapshot?.status === "running") {
1072
+ await writer.resume(runId); // completed/failed snapshots short-circuit or re-throw — skip them
1073
+ }
1074
+ }
1075
+ ```
1076
+
1077
+ Pair `deleteOnComplete: true` with this loop when you don't need the completed-run audit trail — the store then holds only genuinely-unfinished runs, so the drain never touches settled ones.
1078
+
1079
+ ## Cost + testing
1080
+
1081
+ - **Checkpointing cost is one store write per settled trip / node** — a `JSONB` upsert on `pg`, an in-process `Map` set on `memory`. A failed checkpoint is surfaced via logs, not thrown: it loses resume-ability from that point but never breaks an otherwise-healthy run.
1082
+ - **Resume saves the tokens of every settled trip / node** — their model calls are not replayed. A completed-run resume spends nothing (it rebuilds the result from the snapshot). The planning LLM is never re-called on planner resume.
1083
+ - **Test with `ai.snapshot.memory()`.** Drive `execute(input, { runId })` against a flaky model that throws once, assert the tool spy was called once, flip the failure off, `resume(runId)`, and assert (a) `status === "completed"`, (b) the tool spy count is unchanged (no re-invoke), and (c) `usage.total` counts each trip's tokens exactly once. Drift is testable by mutating the definition (add a tool) between `execute` and `resume` and asserting `AgentDriftError` — then `{ force: true }` proceeds.
1084
+
1085
+ ## See also
1086
+
1087
+ - [[handle-ai-errors]] — the typed `AgentDriftError` / `PlannerDriftError` / `AgentExecutionError` / `PlannerFailedError` and how `result.error` surfaces a failed run.
1088
+ - [[manage-ai-stores]] — the `ai.snapshot.{memory,pg,redis}()` factories, the `SnapshotStore` contract, dev-owned `pg` / `redis` clients, and never-auto-migrated `schema()`.
1089
+ - [[persist-ai-data]] — supervisor / workflow snapshot resume (the sibling `iterate`-style durability) and the SnapshotStore migration notes.
1090
+ - [[durable-resume]] — the OTHER resume: `ai.human.resume` of a gated tool call (human-in-the-loop), not a crash.
1091
+
1092
+
954
1093
  ## durable-resume `@warlock.js/ai/durable-resume/SKILL.md`
955
1094
 
956
1095
  ---
@@ -1402,6 +1541,291 @@ if (report.regression && !report.regression.passed) process.exit(1);
1402
1541
  - [`@warlock.js/ai/record-replay-llm/SKILL.md`](@warlock.js/ai/record-replay-llm/SKILL.md) — `ai.vcr` for deterministic, offline eval runs
1403
1542
 
1404
1543
 
1544
+ ## generate-images `@warlock.js/ai/generate-images/SKILL.md`
1545
+
1546
+ ---
1547
+ name: generate-images
1548
+ description: 'Text-to-image via ai.image({ model: sdk.image({ name }), prompt }) — the image-OUTPUT verb (Theme I), returning the uniform never-throws { data, error, usage, report } envelope with cost-truth + panoptic observation. Models come from an adapter''s image() factory: OpenAI gpt-image-* (token-metered) / dall-e-* (per-image), Google imagen-* (per-image). Result images are a discriminated GeneratedImage = { type: "base64" } | { type: "url" }. Triggers: `ai.image`, `sdk.image`, `openai.image`, `google.image`, `ImageModelContract`, `GeneratedImage`, `ImageModelPricing`; ''generate an image'', ''text to image'', ''gpt-image'', ''dall-e'', ''imagen'', ''product thumbnail'', ''image output''; typical import `import { ai } from "@warlock.js/ai"` + `import { OpenAISDK } from "@warlock.js/ai-openai"`. Skip: image INPUT / vision attachments to a chat agent — `@warlock.js/ai/run-ai-agent/SKILL.md`; embeddings — `@warlock.js/ai/embed-text/SKILL.md`; competing libs raw `openai.images.generate`, `langchain` image tools.'
1549
+ ---
1550
+
1551
+ # Generate images — the image-output verb (`ai.image`)
1552
+
1553
+ `ai.image()` is the output counterpart to `ai.agent` for the image modality (the first verb of the output-modality track, Theme I). Prompt-in / images-out, wrapped in the same uniform result contract every executable returns — so it slots into cost dashboards and panoptic traces exactly like an agent run.
1554
+
1555
+ This is image **output** (generation). For image/PDF/audio **input** to a chat agent (vision), see [`@warlock.js/ai/run-ai-agent/SKILL.md`](@warlock.js/ai/run-ai-agent/SKILL.md).
1556
+
1557
+ ## Shape
1558
+
1559
+ ```ts
1560
+ // 1. Build an image model from an adapter's image() factory.
1561
+ const model = openai.image({ name: "gpt-image-1" }); // ImageModelContract
1562
+
1563
+ // 2. Run the verb — never throws; failures land on result.error.
1564
+ const { data, error, usage, report } = await ai.image({ model, prompt: "a red bicycle" });
1565
+
1566
+ if (error) {
1567
+ console.warn(error.code); // typed AIError (auth / rate-limit / content-filter / …)
1568
+ } else {
1569
+ for (const img of data.images) { // GeneratedImage[]
1570
+ if (img.type === "base64") save(Buffer.from(img.base64, "base64"), img.mediaType);
1571
+ else download(img.url);
1572
+ }
1573
+ }
1574
+ ```
1575
+
1576
+ `ImageModelContract` mirrors `EmbedderContract` — a peer primitive on the SDK adapter, produced by the optional `image?()` factory. An adapter without an image API simply doesn't define `image()`, so `ai.anthropic.image(...)` is a **compile-time** error, not a silent runtime failure.
1577
+
1578
+ ## The result envelope
1579
+
1580
+ ```ts
1581
+ type ImageResult = {
1582
+ type: "image";
1583
+ data?: { images: GeneratedImage[] }; // undefined on failure
1584
+ error?: AIError; // undefined on success — NEVER thrown
1585
+ usage: Usage; // tokens (gpt-image) + cost when priced
1586
+ report: ImageReport; // type:"image", model, imageCount, lineage
1587
+ };
1588
+
1589
+ type GeneratedImage =
1590
+ | { type: "base64"; base64: string; mediaType: string; revisedPrompt?: string }
1591
+ | { type: "url"; url: string; mediaType?: string; revisedPrompt?: string };
1592
+ ```
1593
+
1594
+ ## Generation options (provider-neutral)
1595
+
1596
+ ```ts
1597
+ await ai.image({
1598
+ model,
1599
+ prompt: "an isometric office desk, soft studio lighting",
1600
+ count: 2, // n images
1601
+ size: "1024x1024", // OpenAI WxH (also resolves perImageBySize pricing)
1602
+ quality: "high", // OpenAI quality tier
1603
+ aspectRatio: "16:9", // Imagen ratio
1604
+ negativePrompt: "blurry, watermark", // Imagen
1605
+ format: "png", // output container hint
1606
+ signal, // AbortSignal
1607
+ observe: collector, // route the report to an Observer (panoptic), like agents
1608
+ sessionId: "checkout-123",
1609
+ options: { background: "transparent" }, // provider-specific passthrough
1610
+ });
1611
+ ```
1612
+
1613
+ Each adapter maps the options its API supports and ignores the rest. `options` is the escape hatch for provider-specific knobs (OpenAI `background`, DALL·E `responseFormat: "url"`, Imagen `imageSize` / `personGeneration`).
1614
+
1615
+ ## OpenAI — gpt-image (token-metered) + DALL·E (per-image)
1616
+
1617
+ ```ts
1618
+ import { OpenAISDK } from "@warlock.js/ai-openai";
1619
+
1620
+ const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
1621
+
1622
+ // gpt-image-1 always returns base64 bytes; priced per TOKEN.
1623
+ const gpt = openai.image({ name: "gpt-image-1", pricing: { input: 5, output: 40 } });
1624
+
1625
+ // DALL·E 3 — per-image pricing; defaults to base64 (opt into url with options).
1626
+ const dalle = openai.image({ name: "dall-e-3", pricing: { perImage: 0.04 } });
1627
+ ```
1628
+
1629
+ A non-image model id (`openai.image({ name: "gpt-4o" })`) throws `InvalidRequestError` **at construction** — fail fast, like the embedder/vision guards.
1630
+
1631
+ ## Google — Imagen (per-image)
1632
+
1633
+ ```ts
1634
+ import { GoogleSDK } from "@warlock.js/ai-google";
1635
+
1636
+ const google = new GoogleSDK({ apiKey: process.env.GEMINI_API_KEY! });
1637
+ const imagen = google.image({ name: "imagen-4.0-generate-001", pricing: { perImage: 0.04 } });
1638
+
1639
+ const { data } = await ai.image({ model: imagen, prompt: "a watercolor lighthouse at dawn", aspectRatio: "3:4" });
1640
+ ```
1641
+
1642
+ Imagen returns base64 bytes (no hosted URL). When every candidate is safety-filtered, `ai.image` surfaces a typed `ContentFilterError` on `result.error`.
1643
+
1644
+ ## Cost-truth — one rollup, two metering models
1645
+
1646
+ `ai.image` fills `usage.cost` (a `ModelPricing`-shaped USD breakdown) so image spend folds into the **same** `Usage.cost` rollup as text — no second accounting path:
1647
+
1648
+ - **Token-metered** (gpt-image-1): `{ input, output }` USD-per-1M-tokens → standard `computeCost` against the returned token usage.
1649
+ - **Per-image** (DALL·E, Imagen): `{ perImage }` (or `perImageBySize["1792x1024"]`) × image count → `cost.output`.
1650
+
1651
+ Unpriced model → `usage.cost` stays `undefined` (honest "cost unknown", never a false zero). A pre-priced adapter response is honored, not overwritten.
1652
+
1653
+ ## Pattern — catalog thumbnail in a workflow `run` step
1654
+
1655
+ ```ts
1656
+ ai.step({
1657
+ name: "thumbnail",
1658
+ run: async (ctx) => {
1659
+ const { data, error } = await ai.image({
1660
+ model: openai.image({ name: "gpt-image-1" }),
1661
+ prompt: `product photo, white background: ${ctx.steps.extract.output.title}`,
1662
+ size: "1024x1024",
1663
+ });
1664
+ if (error) throw error; // step retry/backoff handles transient provider faults
1665
+ ctx.state.thumb = data.images[0];
1666
+ },
1667
+ });
1668
+ ```
1669
+
1670
+ ## Observability
1671
+
1672
+ The completed `ImageReport` routes to any registered `Observer` (panoptic, OTel, …) through the shared `observe` seam — pass `observe: true` (global), an `Observer` object (flow-local), or rely on observe-all. Cost + latency attribute to `report.model` for free. See [`@warlock.js/ai/observe-ai-flows/SKILL.md`](@warlock.js/ai/observe-ai-flows/SKILL.md).
1673
+
1674
+ ## Testing
1675
+
1676
+ `MockSDK({ imageResponses, imagePricing }).image({ name })` returns a deterministic `MockImageModel` — no HTTP. Script images/usage/errors and inspect `model.calls`.
1677
+
1678
+ ```ts
1679
+ import { MockSDK } from "@warlock.js/ai";
1680
+
1681
+ const mock = MockSDK({ imageResponses: [{}], imagePricing: { perImage: 0.04 } });
1682
+ const { data, usage } = await ai.image({ model: mock.image({ name: "mock-image" }), prompt: "x" });
1683
+ ```
1684
+
1685
+
1686
+ ## generate-speech `@warlock.js/ai/generate-speech/SKILL.md`
1687
+
1688
+ ---
1689
+ name: generate-speech
1690
+ description: 'Text-to-speech via ai.speech({ model: sdk.speech({ name }), text }) — the audio-OUTPUT verb (Theme I), returning the uniform never-throws { data, error, usage, report } envelope with cost-truth + panoptic observation. Models come from an adapter''s speech() factory: OpenAI tts-1 / tts-1-hd (per-character) or gpt-4o-mini-tts (per-token). Synthesized audio is a discriminated GeneratedAudio = { type: "base64"; base64; mediaType }. Options: voice / format / speed / instructions / signal. Triggers: `ai.speech`, `sdk.speech`, `openai.speech`, `SpeechModelContract`, `GeneratedAudio`, `SpeechModelPricing`, `SpeechOptions`, `MockSpeechModel`; ''text to speech'', ''TTS'', ''synthesize voice'', ''read this aloud'', ''tts-1'', ''gpt-4o-mini-tts'', ''voice narration'', ''audio output'', ''speak this text''; typical import `import { ai } from "@warlock.js/ai"` + `import { OpenAISDK } from "@warlock.js/ai-openai"`. Skip: speech-to-text / transcribing a voice note — [[transcribe-audio]]; image OUTPUT — [[generate-images]]; competing libs raw `openai.audio.speech.create`, `elevenlabs` SDK.'
1691
+ ---
1692
+
1693
+ # Generate speech — the text-to-speech verb (`ai.speech`)
1694
+
1695
+ `ai.speech()` is the audio-output counterpart to `ai.image()` on the output-modality track (Theme I). Text-in / audio-out, wrapped in the same uniform result contract every executable returns — so a synthesized voicemail slots into cost dashboards and panoptic traces exactly like an agent run.
1696
+
1697
+ This is audio **output** (TTS). For audio **input** (speech-to-text on a WhatsApp voice note or a meeting recording), see [[transcribe-audio]].
1698
+
1699
+ ## Shape
1700
+
1701
+ ```ts
1702
+ // 1. Build a speech model from an adapter's speech() factory.
1703
+ const model = openai.speech({ name: "tts-1", voice: "alloy" }); // SpeechModelContract
1704
+
1705
+ // 2. Run the verb — never throws; failures land on result.error.
1706
+ const { data, error, usage, report } = await ai.speech({ model, text: "Your order has shipped." });
1707
+
1708
+ if (error) {
1709
+ console.warn(error.code); // typed AIError (auth / rate-limit / content-filter / …)
1710
+ } else {
1711
+ const { base64, mediaType } = data.audio; // GeneratedAudio (always base64 today)
1712
+ await fs.writeFile("ship.mp3", Buffer.from(base64, "base64"));
1713
+ }
1714
+ ```
1715
+
1716
+ `SpeechModelContract` mirrors `EmbedderContract` / `ImageModelContract` — a peer primitive produced by the adapter's optional `speech?()` factory. An adapter without a TTS API simply doesn't define `speech()`, so calling it is a **compile-time** error, not a silent runtime failure. A non-TTS model id (`openai.speech({ name: "gpt-4o" })`) throws `InvalidRequestError` **at construction** — fail fast, like the embedder/image guards.
1717
+
1718
+ ## The result envelope
1719
+
1720
+ ```ts
1721
+ type SpeechResult = {
1722
+ type: "speech";
1723
+ data?: { audio: GeneratedAudio }; // undefined on failure
1724
+ error?: AIError; // undefined on success — NEVER thrown
1725
+ usage: Usage; // tokens (gpt-4o-mini-tts) + cost when priced
1726
+ report: SpeechReport; // type:"speech", model, characters, lineage
1727
+ };
1728
+
1729
+ type GeneratedAudio = {
1730
+ type: "base64";
1731
+ base64: string; // base64-encoded audio bytes
1732
+ mediaType: string; // IANA type, e.g. "audio/mpeg", "audio/wav"
1733
+ };
1734
+ ```
1735
+
1736
+ `GeneratedAudio` is a discriminated union with a single `base64` variant today — the union leaves room for a future hosted-`url` variant without a breaking change, so always branch on `audio.type` rather than assuming `base64`.
1737
+
1738
+ ## Generation options (provider-neutral)
1739
+
1740
+ ```ts
1741
+ await ai.speech({
1742
+ model,
1743
+ text: "Welcome aboard. Let's get you set up.",
1744
+ voice: "verse", // voice id/name; overrides the model's default
1745
+ format: "wav", // "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm"
1746
+ speed: 1.25, // playback multiplier (OpenAI 0.25–4.0)
1747
+ instructions: "calm, warm", // tone/delivery steering (gpt-4o-mini-tts only)
1748
+ signal, // AbortSignal
1749
+ observe: collector, // route the report to an Observer (panoptic), like agents
1750
+ sessionId: "onboarding-42", // group into a session for flat cost/trace queries
1751
+ options: { /* provider passthrough */ },
1752
+ });
1753
+ ```
1754
+
1755
+ Each adapter maps the options its API supports and forwards `options` verbatim. On OpenAI the container defaults to `mp3` (→ `audio/mpeg`); `speed` and `instructions` are only sent when set, and the default voice is `alloy` when neither the call nor the model config supplies one.
1756
+
1757
+ ## OpenAI — tts-1 (per-character) + gpt-4o-mini-tts (per-token)
1758
+
1759
+ ```ts
1760
+ import { ai } from "@warlock.js/ai";
1761
+ import { OpenAISDK } from "@warlock.js/ai-openai";
1762
+
1763
+ const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
1764
+
1765
+ // tts-1 / tts-1-hd — billed per INPUT CHARACTER.
1766
+ const classic = openai.speech({ name: "tts-1", voice: "alloy", pricing: { perMillionCharacters: 15 } });
1767
+
1768
+ // gpt-4o-mini-tts — billed per TOKEN like a chat model; supports `instructions`.
1769
+ const steered = openai.speech({ name: "gpt-4o-mini-tts", pricing: { input: 0.6, output: 12 } });
1770
+
1771
+ const { data } = await ai.speech({ model: steered, text: "Read this warmly.", instructions: "gentle" });
1772
+ ```
1773
+
1774
+ ## Cost-truth — one rollup, two metering models
1775
+
1776
+ `ai.speech` fills `usage.cost` (a USD breakdown) so TTS spend folds into the **same** `Usage.cost` rollup as text — no second accounting path:
1777
+
1778
+ - **Per-character** (`tts-1` / `tts-1-hd`): `{ perMillionCharacters }` × `report.characters` → `cost.input`. The Speech API reports no token usage, so `usage` tokens stay `{ 0, 0, 0 }` and spend is priced entirely from the input character count.
1779
+ - **Token-metered** (`gpt-4o-mini-tts`): `{ input, output }` USD-per-1M-tokens → standard `computeCost` against the returned token usage.
1780
+
1781
+ Per-character wins when both are set. An unpriced model leaves `usage.cost` **`undefined`** (honest "cost unknown", never a false zero); a pre-priced adapter response is honored, not overwritten.
1782
+
1783
+ ## Pattern — order-confirmation voice line in a workflow `run` step
1784
+
1785
+ ```ts
1786
+ ai.step({
1787
+ name: "voiceLine",
1788
+ run: async (ctx) => {
1789
+ const { data, error } = await ai.speech({
1790
+ model: openai.speech({ name: "tts-1", voice: "alloy" }),
1791
+ text: `Order ${ctx.steps.order.output.id} confirmed. Thank you!`,
1792
+ format: "mp3",
1793
+ });
1794
+ if (error) throw error; // step retry/backoff handles transient provider faults
1795
+ ctx.state.audio = data.audio; // { type:"base64", base64, mediaType:"audio/mpeg" }
1796
+ },
1797
+ });
1798
+ ```
1799
+
1800
+ ## Observability
1801
+
1802
+ The completed `SpeechReport` (with `report.characters` and cost/latency attributed to `report.model`) routes to any registered `Observer` (panoptic, OTel, …) through the shared `observe` seam — pass `observe: true` (global), an `Observer` object (flow-local), or rely on observe-all. See [[observe-ai-flows]]. Provider faults surface as typed `AIError`s on `result.error`; see [[handle-ai-errors]].
1803
+
1804
+ ## Testing
1805
+
1806
+ `MockSpeechModel(name, responses, pricing?)` is a deterministic `SpeechModelContract` double — no HTTP. Script audio/usage/errors and inspect `model.calls`. `MockSDK({ speechResponses, speechPricing }).speech({ name })` wires the same double behind a full adapter.
1807
+
1808
+ ```ts
1809
+ import { MockSpeechModel } from "@warlock.js/ai";
1810
+ import { speech } from "@warlock.js/ai";
1811
+
1812
+ const model = new MockSpeechModel("tts-1", [{}], { perMillionCharacters: 15 });
1813
+ const { data, usage } = await speech({ model, text: "abcdefghij" }); // 10 chars
1814
+ // data.audio → { type:"base64", base64:"AAAA", mediaType:"audio/mpeg" }
1815
+ // usage.cost.input → (10 * 15) / 1_000_000
1816
+ // model.calls[0] records { text, options } for assertions
1817
+ ```
1818
+
1819
+ Scripting `[{ error: new ProviderRateLimitError("slow down") }]` drives the never-throws path — `result.error` is the typed error and `result.data` is `undefined`.
1820
+
1821
+ ## See also
1822
+
1823
+ - [[transcribe-audio]] — the inverse verb (`ai.transcribe`), audio → text
1824
+ - [[generate-images]] — the sibling image-output verb (`ai.image`)
1825
+ - [[observe-ai-flows]] — routing the `SpeechReport` to panoptic / OTel
1826
+ - [[handle-ai-errors]] — the typed `AIError` taxonomy on `result.error`
1827
+
1828
+
1405
1829
  ## guard-input-output `@warlock.js/ai/guard-input-output/SKILL.md`
1406
1830
 
1407
1831
  ---
@@ -2723,6 +3147,174 @@ If you switch providers mid-project (e.g. OpenAI → Anthropic):
2723
3147
  - [`@warlock.js/ai/handle-ai-errors/SKILL.md`](@warlock.js/ai/handle-ai-errors/SKILL.md) — adapter error categorization
2724
3148
 
2725
3149
 
3150
+ ## rag-loaders-and-stores `@warlock.js/ai/rag-loaders-and-stores/SKILL.md`
3151
+
3152
+ ---
3153
+ name: rag-loaders-and-stores
3154
+ description: 'Turn any source into a RagDocument and index it in a production vector store — the document loaders ai.rag.loadText / loadHtml / loadWeb (SSRF-safe via guardedFetch) / loadPdf (lazy pdf-parse peer), plus the swappable stores ai.rag.pgVectorStore({client}) (pgvector + ensureSchema DDL + hnsw/ivfflat index) and ai.rag.cacheVectorStore(driver), both satisfying VectorStoreContract (upsert / query / removeNamespace). Loaders return the exact RagDocument[] that kb.index() consumes — no adapter. Triggers: `ai.rag.loadText`, `ai.rag.loadHtml`, `ai.rag.loadWeb`, `ai.rag.loadPdf`, `loadText`, `loadHtml`, `loadWeb`, `loadPdf`, `ai.rag.pgVectorStore`, `ai.rag.cacheVectorStore`, `pgVectorStore`, `cacheVectorStore`, `VectorStore`, `PgVectorStoreOptions`, `PgVectorStoreInstance`, `ensureSchema`, `schema()`, `RagLoaderResult`, `LoadWebOptions`, `LoadPdfOptions`, `perPage`, `OutboundPolicy`, `guardedFetch`, `hnsw`, `ivfflat`, `pgvector`, `dimensions`, `PgClientLike`, `PDF_PARSE_INSTALL_INSTRUCTIONS`; ''load a website into a knowledge base'', ''index a PDF for RAG'', ''strip HTML to text for embedding'', ''pgvector store for RAG'', ''SSRF-safe document fetch'', ''one document per PDF page'', ''swap the vector store''; typical import `import { ai } from "@warlock.js/ai"`. Skip: the chunk → embed → retrieve → rerank → cite pipeline that consumes these — `@warlock.js/ai/run-ai-rag/SKILL.md`; the raw embedder primitive — `@warlock.js/ai/embed-text/SKILL.md`; cache similarity internals — `@warlock.js/cache/use-cache-similarity/SKILL.md`; competing libs `langchain` loaders, `llamaindex` readers.'
3155
+ ---
3156
+
3157
+ # RAG loaders + vector stores — source → `RagDocument` → durable store
3158
+
3159
+ Two feature groups that bracket `ai.rag()`: **loaders** turn a source (string, raw HTML, a URL, or PDF bytes) into the exact `RagDocument[]` shape `kb.index()` consumes, and **stores** are the swappable backends that hold the embeddings. Both live on the `ai.rag.*` namespace — present the moment `@warlock.js/ai` is imported, no side-effect import, no module augmentation.
3160
+
3161
+ ## Contract — what each side produces / satisfies
3162
+
3163
+ Every loader returns `RagLoaderResult` — a plain `RagDocument[]` — so a load feeds `index()` with no adapter and callers never branch on arity (one page ⇒ 1 doc, a per-page PDF ⇒ N docs):
3164
+
3165
+ ```ts
3166
+ type RagLoaderResult = RagDocument[];
3167
+ type RagDocument = { id: string; text: string; metadata?: Record<string, unknown>; tags?: string[] };
3168
+ ```
3169
+
3170
+ Every store satisfies the three-method `VectorStore` contract (a thin narrowing of the cache `similar()` surface — NOT a new engine):
3171
+
3172
+ ```ts
3173
+ interface VectorStore {
3174
+ upsert(key: string, value: unknown, vector: number[], tags?: string[]): Promise<void>;
3175
+ query<T>(vector: number[], options: { topK: number; threshold?: number; tags?: string[] }): Promise<{ key: string; value: T; score: number }[]>;
3176
+ removeNamespace(namespace: string): Promise<void>;
3177
+ }
3178
+ ```
3179
+
3180
+ ## Loaders
3181
+
3182
+ | Loader | Input | Deps | Emits |
3183
+ |---|---|---|---|
3184
+ | `ai.rag.loadText(input, opts?)` | `string` \| `{ id, text }` \| array of either | none | one doc per non-empty item |
3185
+ | `ai.rag.loadHtml(html, opts?)` | raw HTML string | none (regex strip) | one doc, `metadata.title` from `<title>` |
3186
+ | `ai.rag.loadWeb(url, opts?)` | absolute URL | none (uses core `guardedFetch`) | one doc, SSRF-safe fetch |
3187
+ | `ai.rag.loadPdf(bytes, opts?)` | `Buffer` \| `ArrayBuffer` \| `Uint8Array` | lazy `pdf-parse` peer | one doc, or one per page with `perPage: true` |
3188
+
3189
+ Shared options (`RagLoaderOptions`): `id` (source id — falls back to the URL for web, `"document"` otherwise), `metadata` (merged **over** the loader-derived keys, so an explicit `metadata.title` always wins), and `tags` (applied to every chunk for `retrieve({ tags })` filtering). Loader-derived keys: `source`, `loader` (`"text" | "html" | "web" | "pdf"`), plus `title` / `page` / `pageCount` / `contentType` where determinable. Empty / whitespace-only / all-markup inputs emit **no** document — never a no-op record for `index()` to skip.
3190
+
3191
+ ```ts
3192
+ import { ai } from "@warlock.js/ai";
3193
+
3194
+ // Bare string, or many records → many distinctly-identified docs:
3195
+ await kb.index(ai.rag.loadText([
3196
+ { id: "faq-billing", text: "…", metadata: { section: "billing" } },
3197
+ { id: "faq-shipping", text: "…" },
3198
+ ]));
3199
+
3200
+ // Raw HTML → readable text (scripts/styles dropped, entities decoded):
3201
+ await kb.index(ai.rag.loadHtml(rawHtml, { id: "landing", tags: ["marketing"] }));
3202
+ ```
3203
+
3204
+ ### `loadWeb` is SSRF-safe — never a raw `fetch`
3205
+
3206
+ Every request goes through core's `guardedFetch` under an `OutboundPolicy`. The strict defaults (https-only, private-IP-deny on, 10s timeout, 5 MiB cap) apply even when you pass no `policy`, so an untuned call is already hardened. Tighten it per call:
3207
+
3208
+ ```ts
3209
+ await kb.index(await ai.rag.loadWeb("https://docs.example.com/guide", {
3210
+ policy: { hostAllowlist: ["docs.example.com"], maxBytes: 2_000_000, timeoutMs: 5_000 },
3211
+ tags: ["docs"],
3212
+ }));
3213
+ ```
3214
+
3215
+ HTML responses run through the same tag-strip pass as `loadHtml`; non-HTML text (`text/plain`, markdown) is used verbatim. `metadata.source` is the resolved URL, `metadata.contentType` the server-reported type. A non-OK response, a policy block, a timeout, or an over-cap body throws `OutboundPolicyError`.
3216
+
3217
+ ### `loadPdf` — lazy optional peer, page-precise citations
3218
+
3219
+ `pdf-parse` is an **optional** peer, dynamic-imported on the FIRST `loadPdf` call — importing `@warlock.js/ai` never forces it. When it is absent, the curated `PDF_PARSE_INSTALL_INSTRUCTIONS` string is thrown as a plain `Error` (a missing infra peer, not a content problem), never a raw module-resolution stack trace.
3220
+
3221
+ ```ts
3222
+ import { readFile } from "node:fs/promises";
3223
+
3224
+ // Whole PDF → one doc carrying metadata.pageCount:
3225
+ await kb.index(await ai.rag.loadPdf(await readFile("manual.pdf"), { id: "manual" }));
3226
+
3227
+ // One doc per page → citations stay page-precise (id suffixed `#p<n>`, metadata.page set):
3228
+ await kb.index(await ai.rag.loadPdf(bytes, { id: "manual", perPage: true }));
3229
+ ```
3230
+
3231
+ An image-only / scanned page has no text layer and is dropped, so a fully-scanned PDF yields zero docs (nothing to embed).
3232
+
3233
+ ## Stores
3234
+
3235
+ ### `ai.rag.cacheVectorStore(driver)` — adapt any `@warlock.js/cache` driver
3236
+
3237
+ The cache driver **is** the vector store — `upsert → set({ vector, tags })`, `query → similar()`, `removeNamespace → removeNamespace()`. A driver without similarity support throws `CacheUnsupportedError` unchanged (pointing you at the `pg` / `redis` cache drivers).
3238
+
3239
+ ```ts
3240
+ import { MemoryCacheDriver } from "@warlock.js/cache";
3241
+
3242
+ const store = ai.rag.cacheVectorStore(new MemoryCacheDriver()); // dev / tests
3243
+ ```
3244
+
3245
+ ### `ai.rag.pgVectorStore(options)` — production pgvector
3246
+
3247
+ One durable row per chunk keyed by the pipeline's dotted key, the chunk payload in a `JSONB` `value` column, the embedding in a pgvector `vector` column. Pass a live pool (`{ client }` — `@warlock.js/ai` imports **nothing**) or a `{ connectionString }` and let the store lazily `import("pg")` (the optional peer; curated install string on first use if absent). Exactly one of the two is required.
3248
+
3249
+ ```ts
3250
+ type PgVectorStoreOptions = {
3251
+ client?: PgClientLike; // a pg.Pool / pg.Client — only `query` is ever called
3252
+ connectionString?: string; // else the store builds its own Pool lazily
3253
+ table?: string; // default "warlock_ai_rag_vectors"; must be a safe identifier
3254
+ dimensions?: number; // vector(N) width in the DDL, default 1536
3255
+ index?: "hnsw" | "ivfflat" | "none"; // ANN strategy, default "hnsw"
3256
+ ivfflatLists?: number; // ivfflat only, default 100
3257
+ };
3258
+ ```
3259
+
3260
+ `schema()` (alias `ensureSchema()`) returns the reference migration DDL — `CREATE EXTENSION vector`, the table, a GIN index on `tags`, and the chosen ANN index (`USING hnsw (embedding vector_cosine_ops)`). It **only returns the string**; the framework never auto-migrates — you run it once through your own tool. Index and query MUST use the same embedding model: the `vector(N)` width is fixed at table-creation time from `dimensions`.
3261
+
3262
+ ```ts
3263
+ import { Pool } from "pg";
3264
+
3265
+ const pool = new Pool({ connectionString: process.env.DATABASE_URL });
3266
+ const store = ai.rag.pgVectorStore({ client: pool, dimensions: 1536, index: "hnsw" });
3267
+
3268
+ // Once, via your migration tooling — never auto-run:
3269
+ await pool.query(store.ensureSchema());
3270
+ ```
3271
+
3272
+ `query()` runs the cosine floor (`threshold`) and `tags` overlap filter **in SQL** (a below-floor row never crosses the wire), orders by `embedding <=> $vec`, caps at `topK`, and maps the pgvector distance back to a `[0,1]` cosine-similarity `score` — the same scale the cache store emits. `removeNamespace()` is a prefix DELETE that escapes `_` / `%` so dropping `ai.rag.docs` never also catches `ai.rag.docs2`.
3273
+
3274
+ ## Pattern — a knowledge base from a website, backed by pgvector
3275
+
3276
+ ```ts
3277
+ import { Pool } from "pg";
3278
+ import { ai } from "@warlock.js/ai";
3279
+ import { OpenAISDK } from "@warlock.js/ai-openai";
3280
+
3281
+ const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
3282
+ const pool = new Pool({ connectionString: process.env.DATABASE_URL });
3283
+
3284
+ const store = ai.rag.pgVectorStore({ client: pool, dimensions: 1536 });
3285
+ await pool.query(store.ensureSchema()); // once at boot / migration
3286
+
3287
+ const kb = ai.rag({
3288
+ name: "docs",
3289
+ embedder: openai.embedder({ name: "text-embedding-3-small" }), // 1536 dims — matches the DDL
3290
+ store,
3291
+ });
3292
+
3293
+ // Crawl a few pages (SSRF-safe) + a spec PDF into the same namespace:
3294
+ for (const url of ["https://docs.example.com/intro", "https://docs.example.com/config"]) {
3295
+ await kb.index(await ai.rag.loadWeb(url, { policy: { hostAllowlist: ["docs.example.com"] }, tags: ["docs"] }));
3296
+ }
3297
+ await kb.index(await ai.rag.loadPdf(await readFile("spec.pdf"), { id: "spec", perPage: true, tags: ["spec"] }));
3298
+
3299
+ // Now retrieve — every hit's citation traces back to its source URL / page:
3300
+ const { chunks } = await kb.retrieve("how do I configure caching?", { topK: 4, tags: ["docs"] });
3301
+ ```
3302
+
3303
+ The `embedder`'s `dimensions` MUST equal the store's `dimensions` — a mismatch is a runtime insert failure at the pgvector column, not a type error.
3304
+
3305
+ ## Cost + testing
3306
+
3307
+ - **Loaders are cheap.** `loadText` / `loadHtml` are zero-dependency string passes; `loadWeb` costs one guarded HTTP round-trip; `loadPdf` costs the `pdf-parse` parse. **None embed** — embedding cost lands entirely in `kb.index()` (batched, 96 texts per `embedMany` call). The token spend is per chunk, so `perPage` PDFs and finer chunking mean more, smaller vectors.
3308
+ - **`pgVectorStore` construction is synchronous and does no I/O** — the `pg` import + pool build are deferred to first `query`/`upsert`. Table-name validation (`/^[A-Za-z_][A-Za-z0-9_]*$/`) throws at construction, so a `table: "bad; DROP TABLE x"` fails fast.
3309
+ - **Unit-test loaders with fixtures** (a stubbed `policy.fetch` for `loadWeb`, `vi.mock("pdf-parse")` for `loadPdf` — the literal specifier is mockable). Test stores against a `FakePgClient` implementing `{ query }`, or `cacheVectorStore(new MemoryCacheDriver())` for a real end-to-end index/retrieve with no external service.
3310
+
3311
+ ## See also
3312
+
3313
+ - [[run-ai-rag]] — the chunk → embed → retrieve → rerank → cite pipeline that **consumes** these loaders and stores (`ai.rag({ embedder, store })`, `index()` / `retrieve()`).
3314
+ - [[embed-text]] — the `sdk.embedder` primitive whose `dimensions` must match the store's `vector(N)` width.
3315
+ - [`@warlock.js/cache/use-cache-similarity/SKILL.md`](@warlock.js/cache/use-cache-similarity/SKILL.md) — the cache driver `cacheVectorStore` adapts.
3316
+
3317
+
2726
3318
  ## record-replay-llm `@warlock.js/ai/record-replay-llm/SKILL.md`
2727
3319
 
2728
3320
  ---
@@ -4398,6 +4990,167 @@ const escalationAgent = ai.agent({ model, tools: [supportTool] });
4398
4990
  - [`@warlock.js/ai/define-ai-tool/SKILL.md`](@warlock.js/ai/define-ai-tool/SKILL.md) — tool artifacts side-channel
4399
4991
 
4400
4992
 
4993
+ ## transcribe-audio `@warlock.js/ai/transcribe-audio/SKILL.md`
4994
+
4995
+ ---
4996
+ name: transcribe-audio
4997
+ description: 'Speech-to-text via ai.transcribe({ model: sdk.transcribe({ name }), audio }) — the audio-INPUT verb (Theme I), returning the uniform never-throws { data, error, usage, report } envelope with cost-truth + panoptic observation. Feed it an AudioInput = { base64; mediaType; filename? } — build one with ai.audioFromFile(path) (reads disk, infers media type incl. WhatsApp .ogg/.opus) or ai.audioFromBuffer(bytes, mediaType). Models: OpenAI whisper-1 (verbose_json, per-minute, segments + durationSeconds) or gpt-4o-transcribe (json, per-token). Triggers: `ai.transcribe`, `ai.audioFromFile`, `ai.audioFromBuffer`, `sdk.transcribe`, `openai.transcribe`, `TranscriptionModelContract`, `AudioInput`, `TranscriptionSegment`, `MockTranscriptionModel`; ''speech to text'', ''transcribe audio'', ''voice note to text'', ''WhatsApp voice message'', ''whisper'', ''gpt-4o-transcribe'', ''subtitle segments'', ''audio input''; typical import `import { ai } from "@warlock.js/ai"` + `import { OpenAISDK } from "@warlock.js/ai-openai"`. Skip: text-to-speech / synthesizing a voice — [[generate-speech]]; competing libs raw `openai.audio.transcriptions.create`, `whisper.cpp`.'
4998
+ ---
4999
+
5000
+ # Transcribe audio — the speech-to-text verb (`ai.transcribe`)
5001
+
5002
+ `ai.transcribe()` is the inverse of `ai.speech()` on the modality track (Theme I). Audio-in / text-out, wrapped in the same uniform result contract every executable returns — so transcribing a support voicemail slots into cost dashboards and panoptic traces exactly like an agent run.
5003
+
5004
+ **Extracting text from an audio file NEEDS AI** — that is the `ai.transcribe` step. The file handling (`ai.audioFromFile` / `ai.audioFromBuffer`) is pure, non-AI **utility** that just packages bytes into an `AudioInput`; it does no I/O to a provider on its own.
5005
+
5006
+ This is audio **input** (STT). For audio **output** (synthesizing a voice line), see [[generate-speech]].
5007
+
5008
+ ## Shape — WhatsApp voice note → text, end to end
5009
+
5010
+ ```ts
5011
+ import { ai } from "@warlock.js/ai";
5012
+ import { OpenAISDK } from "@warlock.js/ai-openai";
5013
+
5014
+ const openai = new OpenAISDK({ apiKey: process.env.OPENAI_API_KEY! });
5015
+
5016
+ // audioFromFile reads the file + infers the media type from the extension.
5017
+ // .ogg / .opus (Android WhatsApp) and .m4a (iOS) are recognized out of the box.
5018
+ const audio = await ai.audioFromFile("./voice-note.ogg");
5019
+
5020
+ const { data, error } = await ai.transcribe({
5021
+ model: openai.transcribe({ name: "whisper-1" }),
5022
+ audio,
5023
+ language: "en", // BCP-47 hint — improves accuracy + latency
5024
+ });
5025
+
5026
+ if (error) console.warn(error.code); // typed AIError
5027
+ else console.log(data.text); // the transcript
5028
+ ```
5029
+
5030
+ `TranscriptionModelContract` mirrors `SpeechModelContract` — a peer primitive produced by the adapter's optional `transcribe?()` factory. A non-STT model id (`openai.transcribe({ name: "gpt-4o" })`) throws `InvalidRequestError` **at construction** — fail fast, like the speech/embedder guards.
5031
+
5032
+ ## The `AudioInput` shape + the two builders
5033
+
5034
+ ```ts
5035
+ type AudioInput = {
5036
+ base64: string; // base64-encoded audio bytes
5037
+ mediaType: string; // IANA type, e.g. "audio/ogg", "audio/mpeg"
5038
+ filename?: string; // helps providers infer the codec from the extension
5039
+ };
5040
+
5041
+ // From a file on disk — reads + infers media type (override for extensionless files).
5042
+ const fromDisk = await ai.audioFromFile("./meeting.m4a");
5043
+ const forced = await ai.audioFromFile("./blob", { mediaType: "audio/ogg" });
5044
+
5045
+ // From bytes you already hold (an upload buffer, a downloaded blob) — no I/O, no AI.
5046
+ const fromBytes = ai.audioFromBuffer(uploadBuffer, "audio/ogg", "note.ogg");
5047
+ ```
5048
+
5049
+ Keeping `AudioInput` as inlined base64 + explicit media type makes the verb provider-neutral and serializable — there is no `fs` coupling in core, so the same request can cross a queue or an RPC boundary.
5050
+
5051
+ ## The result envelope
5052
+
5053
+ ```ts
5054
+ type TranscriptionResult = {
5055
+ type: "transcription";
5056
+ data?: {
5057
+ text: string; // full transcript
5058
+ segments?: TranscriptionSegment[]; // timestamped, in verbose mode
5059
+ }; // undefined on failure
5060
+ error?: AIError; // undefined on success — NEVER thrown
5061
+ usage: Usage; // tokens (gpt-4o-transcribe) + cost when priced
5062
+ report: TranscriptionReport; // type:"transcription", model, durationSeconds, lineage
5063
+ };
5064
+
5065
+ type TranscriptionSegment = { text: string; start?: number; end?: number };
5066
+ ```
5067
+
5068
+ `segments` and `report.durationSeconds` appear only when the provider returns them (whisper's `verbose_json` mode). Use segments to build subtitles or to jump-to-timestamp in a player.
5069
+
5070
+ ## Transcribe options (provider-neutral)
5071
+
5072
+ ```ts
5073
+ await ai.transcribe({
5074
+ model,
5075
+ audio,
5076
+ language: "en", // BCP-47 hint
5077
+ prompt: "Names: Acme, Zoë", // priming — spelling / style hints
5078
+ format: "verbose_json", // response-format override (segments + duration)
5079
+ signal, // AbortSignal
5080
+ observe: collector, // route the report to an Observer (panoptic)
5081
+ sessionId: "ticket-88", // group into a session for flat cost/trace queries
5082
+ options: { /* provider passthrough */ },
5083
+ });
5084
+ ```
5085
+
5086
+ ## OpenAI — whisper-1 (per-minute) + gpt-4o-transcribe (per-token)
5087
+
5088
+ ```ts
5089
+ // whisper-1 — defaults to verbose_json → segments + duration; billed PER MINUTE.
5090
+ const whisper = openai.transcribe({ name: "whisper-1", pricing: { perMinute: 0.006 } });
5091
+
5092
+ // gpt-4o-transcribe — defaults to json; billed PER TOKEN like a chat model.
5093
+ const gpt = openai.transcribe({ name: "gpt-4o-transcribe", pricing: { input: 2.5, output: 10 } });
5094
+
5095
+ const { data, usage } = await ai.transcribe({ model: whisper, audio });
5096
+ // data.segments → [{ text, start, end }, …]; usage.cost from report.durationSeconds
5097
+ ```
5098
+
5099
+ The adapter wraps the base64 bytes in an uploadable via the SDK's `toFile`, using `audio.filename` (or `"audio"`) and `audio.mediaType` so the codec is declared correctly.
5100
+
5101
+ ## Cost-truth — one rollup, two metering models
5102
+
5103
+ `ai.transcribe` fills `usage.cost` so STT spend folds into the **same** `Usage.cost` rollup as text:
5104
+
5105
+ - **Per-minute** (`whisper-1`): `{ perMinute }` × `(durationSeconds / 60)` → `cost.input`. If the provider didn't report a duration, cost stays **`undefined`** (no guessing).
5106
+ - **Token-metered** (`gpt-4o-transcribe`): `{ input, output }` USD-per-1M-tokens → standard `computeCost` against the returned token usage.
5107
+
5108
+ Per-minute wins when both are set; an unpriced model leaves `usage.cost` **`undefined`** (honest "cost unknown", never a false zero).
5109
+
5110
+ ## Pattern — inbound voice-message webhook
5111
+
5112
+ ```ts
5113
+ const stt = openai.transcribe({ name: "whisper-1" });
5114
+
5115
+ async function onVoiceMessage(buffer: Buffer, mediaType: string) {
5116
+ const audio = ai.audioFromBuffer(buffer, mediaType, "inbound.ogg");
5117
+ const { data, error } = await ai.transcribe({ model: stt, audio, sessionId: "inbox" });
5118
+
5119
+ if (error) return replyWith("Sorry, I couldn't understand that audio.");
5120
+ return routeToAgent(data.text); // hand the transcript to an ai.agent for a reply
5121
+ }
5122
+ ```
5123
+
5124
+ ## Observability
5125
+
5126
+ The completed `TranscriptionReport` (with `report.durationSeconds` and cost/latency attributed to `report.model`) routes to any registered `Observer` (panoptic, OTel, …) through the shared `observe` seam — `observe: true` (global), an `Observer` (flow-local), or observe-all. See [[observe-ai-flows]]. Provider faults surface as typed `AIError`s on `result.error`; see [[handle-ai-errors]].
5127
+
5128
+ ## Testing
5129
+
5130
+ `MockTranscriptionModel(name, responses, pricing?)` is a deterministic `TranscriptionModelContract` double — no HTTP. Script text/segments/duration/usage/errors and inspect `model.calls`. `MockSDK({ transcriptionResponses, transcriptionPricing }).transcribe({ name })` wires the same double behind a full adapter.
5131
+
5132
+ ```ts
5133
+ import { MockTranscriptionModel, transcribe } from "@warlock.js/ai";
5134
+
5135
+ const AUDIO = { base64: "QUJD", mediaType: "audio/mpeg", filename: "clip.mp3" };
5136
+
5137
+ const model = new MockTranscriptionModel("whisper-1", [{ durationSeconds: 120 }], { perMinute: 0.006 });
5138
+ const { data, usage, report } = await transcribe({ model, audio: AUDIO });
5139
+ // data.text → "mock transcript"
5140
+ // usage.cost.input → (120 / 60) * 0.006 report.durationSeconds → 120
5141
+ // model.calls[0] records { audio, options } for assertions
5142
+ ```
5143
+
5144
+ Scripting `[{ error: new ProviderRateLimitError("slow down") }]` drives the never-throws path — `result.error` is the typed error and `result.data` is `undefined`.
5145
+
5146
+ ## See also
5147
+
5148
+ - [[generate-speech]] — the inverse verb (`ai.speech`), text → audio
5149
+ - [[generate-images]] — the sibling image-output verb (`ai.image`)
5150
+ - [[observe-ai-flows]] — routing the `TranscriptionReport` to panoptic / OTel
5151
+ - [[handle-ai-errors]] — the typed `AIError` taxonomy on `result.error`
5152
+
5153
+
4401
5154
  ## use-ai-memory `@warlock.js/ai/use-ai-memory/SKILL.md`
4402
5155
 
4403
5156
  ---