@gentbajko/slopify 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +63 -0
  3. package/dist/adapter-registry.js +68 -0
  4. package/dist/adapters/image/bytes.js +64 -0
  5. package/dist/adapters/image/fal.js +164 -0
  6. package/dist/adapters/image/openai.js +155 -0
  7. package/dist/adapters/image/replicate.js +183 -0
  8. package/dist/adapters/llm/claude-code.js +158 -0
  9. package/dist/adapters/llm/codex.js +126 -0
  10. package/dist/adapters/llm/openrouter.js +193 -0
  11. package/dist/adapters/llm/run-cli.js +134 -0
  12. package/dist/adapters/llm/sse-lines.js +46 -0
  13. package/dist/adapters/retry-after.js +21 -0
  14. package/dist/adapters/tts/cartesia.js +105 -0
  15. package/dist/adapters/tts/elevenlabs.js +107 -0
  16. package/dist/adapters/tts/openai.js +95 -0
  17. package/dist/edge/cli.js +36 -0
  18. package/dist/edge/events/hub.js +76 -0
  19. package/dist/edge/http/actions.js +125 -0
  20. package/dist/edge/http/app.js +72 -0
  21. package/dist/edge/http/entries.js +41 -0
  22. package/dist/edge/http/files.js +67 -0
  23. package/dist/edge/http/problem.js +65 -0
  24. package/dist/edge/http/projects.js +128 -0
  25. package/dist/edge/http/prompts.js +73 -0
  26. package/dist/edge/http/providers.js +62 -0
  27. package/dist/edge/http/settings.js +90 -0
  28. package/dist/edge/http/staging.js +104 -0
  29. package/dist/edge/http/telemetry.js +29 -0
  30. package/dist/edge/http/usage.js +19 -0
  31. package/dist/edge/open-browser.js +20 -0
  32. package/dist/kernel/clock.js +18 -0
  33. package/dist/kernel/config/index.js +32 -0
  34. package/dist/kernel/db/index.js +24 -0
  35. package/dist/kernel/db/migrate.js +59 -0
  36. package/dist/kernel/db/migrations/0001-init.sql +17 -0
  37. package/dist/kernel/db/tx.js +53 -0
  38. package/dist/kernel/events.js +1 -0
  39. package/dist/kernel/ids.js +4 -0
  40. package/dist/kernel/lock.js +73 -0
  41. package/dist/kernel/log.js +34 -0
  42. package/dist/kernel/paths.js +21 -0
  43. package/dist/kernel/pipeline.js +22 -0
  44. package/dist/kernel/ports/image.js +1 -0
  45. package/dist/kernel/ports/llm.js +1 -0
  46. package/dist/kernel/ports/model.js +34 -0
  47. package/dist/kernel/ports/registry.js +1 -0
  48. package/dist/kernel/ports/tts.js +1 -0
  49. package/dist/kernel/runner/attempt-repo.js +50 -0
  50. package/dist/kernel/runner/attempt.js +146 -0
  51. package/dist/kernel/runner/graph.js +82 -0
  52. package/dist/kernel/runner/index.js +205 -0
  53. package/dist/kernel/runner/piece-repo.js +58 -0
  54. package/dist/kernel/runner/providers.js +92 -0
  55. package/dist/kernel/version.js +14 -0
  56. package/dist/main.js +212 -0
  57. package/dist/slices/admission/model.js +6 -0
  58. package/dist/slices/admission/repo.js +178 -0
  59. package/dist/slices/admission/rules.js +201 -0
  60. package/dist/slices/admission/start.js +103 -0
  61. package/dist/slices/admission/substitute.js +80 -0
  62. package/dist/slices/article/continuation.js +81 -0
  63. package/dist/slices/article/plain.js +14 -0
  64. package/dist/slices/article/run.js +166 -0
  65. package/dist/slices/article/split.js +51 -0
  66. package/dist/slices/article/store.js +21 -0
  67. package/dist/slices/cancel/index.js +61 -0
  68. package/dist/slices/images/run.js +221 -0
  69. package/dist/slices/library/lint.js +53 -0
  70. package/dist/slices/library/model.js +13 -0
  71. package/dist/slices/library/repo.js +111 -0
  72. package/dist/slices/library/save.js +87 -0
  73. package/dist/slices/library/slots.js +86 -0
  74. package/dist/slices/narration/chunk.js +71 -0
  75. package/dist/slices/narration/concat.js +79 -0
  76. package/dist/slices/narration/run.js +280 -0
  77. package/dist/slices/reruns/cascade.js +66 -0
  78. package/dist/slices/reruns/index.js +269 -0
  79. package/dist/slices/research/planner.js +81 -0
  80. package/dist/slices/research/run.js +192 -0
  81. package/dist/slices/research/synthesis.js +48 -0
  82. package/dist/slices/settings/cli-status.js +37 -0
  83. package/dist/slices/settings/keys.js +55 -0
  84. package/dist/slices/settings/model.js +54 -0
  85. package/dist/slices/settings/playback.js +60 -0
  86. package/dist/slices/settings/readiness.js +21 -0
  87. package/dist/slices/settings/repo.js +59 -0
  88. package/dist/slices/settings/voices.js +48 -0
  89. package/dist/slices/storage/asset-name.js +16 -0
  90. package/dist/slices/storage/delete-project.js +35 -0
  91. package/dist/slices/storage/downloads.js +100 -0
  92. package/dist/slices/storage/layout.js +59 -0
  93. package/dist/slices/storage/model.js +20 -0
  94. package/dist/slices/storage/reconcile.js +102 -0
  95. package/dist/slices/storage/repo.js +104 -0
  96. package/dist/slices/storage/staging.js +224 -0
  97. package/dist/slices/telemetry/collector-client.js +51 -0
  98. package/dist/slices/telemetry/flush.js +111 -0
  99. package/dist/slices/telemetry/machine.js +33 -0
  100. package/dist/slices/telemetry/model.js +39 -0
  101. package/dist/slices/telemetry/record.js +41 -0
  102. package/dist/slices/telemetry/repo.js +73 -0
  103. package/dist/slices/telemetry/usage.js +82 -0
  104. package/dist/slices/thumbnail/by-llm.js +30 -0
  105. package/dist/slices/thumbnail/run.js +200 -0
  106. package/dist/slices/video/ffmpeg.js +230 -0
  107. package/dist/slices/video/plan.js +71 -0
  108. package/dist/slices/video/run.js +179 -0
  109. package/dist/web/app-icon.svg +1 -0
  110. package/dist/web/assets/barlow-condensed-latin-600-normal-BFJEwTuo.woff +0 -0
  111. package/dist/web/assets/barlow-condensed-latin-600-normal-DepVgxBB.woff2 +0 -0
  112. package/dist/web/assets/barlow-condensed-latin-700-normal-Dmwat-ge.woff +0 -0
  113. package/dist/web/assets/barlow-condensed-latin-700-normal-v1xN8_Wq.woff2 +0 -0
  114. package/dist/web/assets/barlow-condensed-latin-ext-600-normal-18ESti3H.woff2 +0 -0
  115. package/dist/web/assets/barlow-condensed-latin-ext-600-normal-Clv9cIcR.woff +0 -0
  116. package/dist/web/assets/barlow-condensed-latin-ext-700-normal-BIHFfxf0.woff +0 -0
  117. package/dist/web/assets/barlow-condensed-latin-ext-700-normal-CwuXbfVR.woff2 +0 -0
  118. package/dist/web/assets/barlow-condensed-vietnamese-600-normal-A5AYRdjN.woff2 +0 -0
  119. package/dist/web/assets/barlow-condensed-vietnamese-600-normal-CNlPk46_.woff +0 -0
  120. package/dist/web/assets/barlow-condensed-vietnamese-700-normal-DYeBwlKR.woff2 +0 -0
  121. package/dist/web/assets/barlow-condensed-vietnamese-700-normal-DhIzd8Tb.woff +0 -0
  122. package/dist/web/assets/barlow-latin-400-normal-fsAxiSwU.woff +0 -0
  123. package/dist/web/assets/barlow-latin-400-normal-qiz4-Cze.woff2 +0 -0
  124. package/dist/web/assets/barlow-latin-500-normal-BPAOfeC8.woff2 +0 -0
  125. package/dist/web/assets/barlow-latin-500-normal-C1h8hMer.woff +0 -0
  126. package/dist/web/assets/barlow-latin-600-normal-CNwfPWQD.woff +0 -0
  127. package/dist/web/assets/barlow-latin-600-normal-DILqtrty.woff2 +0 -0
  128. package/dist/web/assets/barlow-latin-700-normal-A9pxMQ4z.woff2 +0 -0
  129. package/dist/web/assets/barlow-latin-700-normal-__SGTsZ1.woff +0 -0
  130. package/dist/web/assets/barlow-latin-800-normal-BdVooDN4.woff +0 -0
  131. package/dist/web/assets/barlow-latin-800-normal-s1sAMnoV.woff2 +0 -0
  132. package/dist/web/assets/barlow-latin-ext-400-normal-CvBsJvxq.woff +0 -0
  133. package/dist/web/assets/barlow-latin-ext-400-normal-HxX4XjxC.woff2 +0 -0
  134. package/dist/web/assets/barlow-latin-ext-500-normal-CJPcKP2Q.woff +0 -0
  135. package/dist/web/assets/barlow-latin-ext-500-normal-DOaysfXq.woff2 +0 -0
  136. package/dist/web/assets/barlow-latin-ext-600-normal-B8NK_A3D.woff2 +0 -0
  137. package/dist/web/assets/barlow-latin-ext-600-normal-DMVRjfRT.woff +0 -0
  138. package/dist/web/assets/barlow-latin-ext-700-normal-BLuWmldJ.woff2 +0 -0
  139. package/dist/web/assets/barlow-latin-ext-700-normal-CctuGmmz.woff +0 -0
  140. package/dist/web/assets/barlow-latin-ext-800-normal-BiucknKG.woff2 +0 -0
  141. package/dist/web/assets/barlow-latin-ext-800-normal-D7I3yvUw.woff +0 -0
  142. package/dist/web/assets/barlow-vietnamese-400-normal-BFeobeCK.woff +0 -0
  143. package/dist/web/assets/barlow-vietnamese-400-normal-Dpl4UHAZ.woff2 +0 -0
  144. package/dist/web/assets/barlow-vietnamese-500-normal-GNfB7rCE.woff +0 -0
  145. package/dist/web/assets/barlow-vietnamese-500-normal-zTViEIzf.woff2 +0 -0
  146. package/dist/web/assets/barlow-vietnamese-600-normal-CA_GiK2e.woff +0 -0
  147. package/dist/web/assets/barlow-vietnamese-600-normal-DcjprdFV.woff2 +0 -0
  148. package/dist/web/assets/barlow-vietnamese-700-normal-4Jt4k04K.woff +0 -0
  149. package/dist/web/assets/barlow-vietnamese-700-normal-D6euyNzi.woff2 +0 -0
  150. package/dist/web/assets/barlow-vietnamese-800-normal-Cl1Mc_Dv.woff2 +0 -0
  151. package/dist/web/assets/barlow-vietnamese-800-normal-D0VWpbij.woff +0 -0
  152. package/dist/web/assets/index-BPqQnrpy.css +1 -0
  153. package/dist/web/assets/index-D_sWbKQi.js +81 -0
  154. package/dist/web/favicon.svg +1 -0
  155. package/dist/web/index.html +15 -0
  156. package/package.json +41 -0
@@ -0,0 +1,269 @@
1
+ import { rmSync } from "node:fs";
2
+ import { z } from "zod";
3
+ import { transact } from "../../kernel/db/tx.js";
4
+ import { allPiecesOf, deletePiece, deletePieces, setPiece, } from "../../kernel/runner/piece-repo.js";
5
+ import { projectById, resetStage, stagesOf } from "../admission/repo.js";
6
+ import { storeArticleText } from "../article/store.js";
7
+ import { outputPath } from "../storage/layout.js";
8
+ import { pieceFile } from "../storage/reconcile.js";
9
+ import { deleteOutput, outputsOf } from "../storage/repo.js";
10
+ import { storeText } from "../storage/staging.js";
11
+ import { redoPlan } from "./cascade.js";
12
+ // Only the one field this module changes is named; the rest of a piece's payload belongs
13
+ // to the stage that wrote it and travels through untouched.
14
+ const anyObject = z.record(z.string(), z.unknown());
15
+ // What an article edit replaces (step 1). `instructions` is not among them: §Q57 keeps
16
+ // the record of what was actually sent to the model, and an edit did not send anything.
17
+ const articleRoles = ["article_md", "article_txt", "sources", "glossary"];
18
+ // `logic/13` step 5 and `logic/01` §Q5: Retry is offered on a stage that stopped short,
19
+ // and a canceled stage resumes exactly as a failed one does.
20
+ export function retryStage(deps, projectId, kind) {
21
+ const loaded = load(deps, projectId);
22
+ if (!loaded.ok) {
23
+ return loaded;
24
+ }
25
+ const stage = stageOf(loaded.stages, kind);
26
+ if (stage === undefined || (stage.state !== "failed" && stage.state !== "canceled")) {
27
+ return { ok: false, reason: "not-retryable" };
28
+ }
29
+ // The pieces and the outputs stay where they are: the per-stage resume rules keep what
30
+ // finished (`logic/06` §Q54, `logic/08` §Q66, `logic/09` §Q73), and `logic/13` step 2
31
+ // keeps them through a cancel too. That is the whole of "a canceled stage is resumable".
32
+ transact(deps.db, () => {
33
+ resetStage(deps.db, stage.id);
34
+ });
35
+ return { ok: true, redone: [kind] };
36
+ }
37
+ // Steps 2, 3 and 8: Re-run audio with another voice, Re-run images, Re-render. The stage
38
+ // starts over from the project's stored configuration rather than resuming.
39
+ export function rerunStage(deps, projectId, kind) {
40
+ const loaded = load(deps, projectId);
41
+ if (!loaded.ok) {
42
+ return loaded;
43
+ }
44
+ const stage = stageOf(loaded.stages, kind);
45
+ if (stage === undefined || !rerunnable(stage.state)) {
46
+ return { ok: false, reason: "not-rerunnable" };
47
+ }
48
+ return apply(deps, projectId, { kind: "rerun", stage: kind }, loaded);
49
+ }
50
+ // Step 1: "the inline editor replaces the stored markdown; the plain-text narration source
51
+ // and the sources and glossary files are rebuilt; then audio, LLM-mode intro/outro text,
52
+ // LLM-written thumbnail, and video re-run; prompt-based images are untouched".
53
+ //
54
+ // ceiling: the LLM-mode intro and outro texts are *not* rewritten from the edited article.
55
+ // Only the article stage writes them (`logic/07` step 5), and §Q60 makes any run of that
56
+ // stage regenerate the whole article - which would throw away the edit that triggered it.
57
+ // The audio does re-narrate the kept entry text, so the video is whole; the upgrade is a
58
+ // mode on the article stage that keeps the stored article and rewrites the entries alone.
59
+ export function editArticle(deps, projectId, markdown) {
60
+ const loaded = load(deps, projectId);
61
+ if (!loaded.ok) {
62
+ return loaded;
63
+ }
64
+ const article = stageOf(loaded.stages, "article");
65
+ if (article === undefined || (article.state !== "done" && article.state !== "provided")) {
66
+ return { ok: false, reason: "no-article" };
67
+ }
68
+ const text = markdown.trim();
69
+ if (text === "") {
70
+ // §Q106 leaves a project with one output per stage; an empty article would leave the
71
+ // audio stage with nothing to narrate and no way back except another edit.
72
+ return { ok: false, reason: "empty-article" };
73
+ }
74
+ return apply(deps, projectId, { kind: "article-edit" }, loaded, () => {
75
+ const replaced = loaded.outputs.filter((output) => articleRoles.includes(output.role));
76
+ for (const output of replaced) {
77
+ deleteOutput(deps.db, output.id);
78
+ }
79
+ // The four files are written back under the names they already had, so the write is
80
+ // the replacement §Q106 asks for rather than a second version beside the first.
81
+ storeText(deps, { projectId, stageKind: "article", role: "article_md", text });
82
+ storeArticleText(deps, { projectId, markdown: text });
83
+ return replaced.map((output) => output.path);
84
+ });
85
+ }
86
+ // Step 5 with `logic/09` §Q75: one image is "removed from the set; at least one image must
87
+ // remain; video re-renders".
88
+ export function deleteImage(deps, projectId, outputId) {
89
+ const loaded = load(deps, projectId);
90
+ if (!loaded.ok) {
91
+ return loaded;
92
+ }
93
+ const image = imageOf(loaded.outputs, outputId);
94
+ if (image === undefined) {
95
+ return { ok: false, reason: "unknown-image" };
96
+ }
97
+ if (loaded.outputs.filter((output) => output.role === "image").length <= 1) {
98
+ // §Q103's invariant: "at least one image always remains". `logic/11` step 2 has no
99
+ // slideshow to compute without one.
100
+ return { ok: false, reason: "last-image" };
101
+ }
102
+ return apply(deps, projectId, { kind: "image-deleted" }, loaded, () => {
103
+ // `slices/images/run.ts` counts an image as landed only when its row and its file both
104
+ // survive, so a delete removes both - and the piece that planned it, or the next run
105
+ // of the stage would read the plan and make the image again.
106
+ deleteOutput(deps.db, image.id);
107
+ dropImagePiece(deps, loaded.stages, image.path);
108
+ return [image.path];
109
+ });
110
+ }
111
+ // Step 4: "one new call with that image's stored prompt text, replacing it in place at the
112
+ // same index (§Q103)". The piece carries that prompt and that index, so it is left alone
113
+ // and the images stage sees one image of its plan missing.
114
+ export function regenerateImage(deps, projectId, outputId) {
115
+ const loaded = load(deps, projectId);
116
+ if (!loaded.ok) {
117
+ return loaded;
118
+ }
119
+ const image = imageOf(loaded.outputs, outputId);
120
+ if (image === undefined) {
121
+ return { ok: false, reason: "unknown-image" };
122
+ }
123
+ return apply(deps, projectId, { kind: "image-regenerated" }, loaded, () => {
124
+ deleteOutput(deps.db, image.id);
125
+ forgetImageFile(deps, loaded.stages, image.path);
126
+ return [image.path];
127
+ });
128
+ }
129
+ function load(deps, projectId) {
130
+ const project = projectById(deps.db, projectId);
131
+ if (project === undefined) {
132
+ return { ok: false, reason: "no-project" };
133
+ }
134
+ const stages = stagesOf(deps.db, projectId);
135
+ // The precondition every action of `logic/12` shares: "no stage of the project is
136
+ // `running` (§Q106)". The page disables the controls; this is the other half, because
137
+ // clearing an output from under a stage that is writing it would lose both.
138
+ if (stages.some((stage) => stage.state === "running")) {
139
+ return { ok: false, reason: "running" };
140
+ }
141
+ return {
142
+ ok: true,
143
+ stages,
144
+ outputs: outputsOf(deps.db, projectId),
145
+ thumbnailSource: project.config.sources.thumbnail,
146
+ };
147
+ }
148
+ // One transaction for every row the action touches, then the filesystem. S4's lesson:
149
+ // an unlink cannot be rolled back, so it waits for the commit - and a file with no row is
150
+ // what the boot reconcile collects, where a row with no file is a broken download.
151
+ function apply(deps, projectId, action, loaded, own) {
152
+ const plan = redoPlan({ action, stages: loaded.stages, thumbnailSource: loaded.thumbnailSource });
153
+ // `own` is the action's own change - the new article, the image that goes - and it runs
154
+ // first and inside the same transaction, so it lands or rolls back with the cascade it
155
+ // triggers. The files it writes take the names they already had, which is the
156
+ // replacement §Q106 asks for rather than work a rollback would have to undo.
157
+ const orphaned = transact(deps.db, () => {
158
+ const files = own === undefined ? [] : [...own()];
159
+ for (const redo of plan) {
160
+ const stage = stageOf(loaded.stages, redo.stage);
161
+ if (stage === undefined) {
162
+ continue;
163
+ }
164
+ if (redo.clears === "all") {
165
+ files.push(...clearStage(deps, stage, loaded.outputs));
166
+ }
167
+ resetStage(deps.db, stage.id);
168
+ }
169
+ return files;
170
+ });
171
+ removeFiles(deps, projectId, orphaned);
172
+ return { ok: true, redone: plan.map((redo) => redo.stage) };
173
+ }
174
+ // Everything the stage produced: its outputs, and the resumable pieces whose payloads name
175
+ // files of their own - the audio chunks of `logic/08` §Q65, which are not outputs.
176
+ function clearStage(deps, stage, outputs) {
177
+ const files = [];
178
+ for (const output of outputs) {
179
+ if (output.stageKind !== stage.kind) {
180
+ continue;
181
+ }
182
+ files.push(output.path);
183
+ deleteOutput(deps.db, output.id);
184
+ }
185
+ for (const piece of allPiecesOf(deps.db, stage.id)) {
186
+ const file = pieceFile(piece.payload);
187
+ if (file !== undefined) {
188
+ files.push(file);
189
+ }
190
+ }
191
+ deletePieces(deps.db, stage.id);
192
+ return files;
193
+ }
194
+ // §Q103 replaces the image "in place at the same index", so the piece keeps the prompt
195
+ // and the index it was planned with and only stops naming a file: `slices/images/run.ts`
196
+ // then sees one image of its plan still to make, and the file it named can be removed.
197
+ function forgetImageFile(deps, stages, path) {
198
+ for (const piece of imagePieces(deps, stages, path)) {
199
+ if (piece.payload === null) {
200
+ continue;
201
+ }
202
+ const parsed = anyObject.safeParse(JSON.parse(piece.payload));
203
+ if (!parsed.success) {
204
+ continue;
205
+ }
206
+ const kept = Object.entries(parsed.data).filter(([name]) => name !== "file");
207
+ setPiece(deps.db, piece.id, "pending", JSON.stringify(Object.fromEntries(kept)));
208
+ }
209
+ }
210
+ function dropImagePiece(deps, stages, path) {
211
+ for (const piece of imagePieces(deps, stages, path)) {
212
+ deletePiece(deps.db, piece.id);
213
+ }
214
+ }
215
+ // The pieces of the images stage that name this file. There is one, unless a run before
216
+ // this one wrote two pieces onto the same path, which the reconcile would have to sort
217
+ // out anyway; taking all of them keeps the row set and the disk agreeing either way.
218
+ function imagePieces(deps, stages, path) {
219
+ const stage = stageOf(stages, "images");
220
+ if (stage === undefined) {
221
+ return [];
222
+ }
223
+ return allPiecesOf(deps.db, stage.id).filter((piece) => pieceFile(piece.payload) === path);
224
+ }
225
+ // Read back rather than reasoned about: an edit rewrites `article.md` under the same name
226
+ // it just deleted the row for, so what is still referenced after the commit is the only
227
+ // safe answer to what may be unlinked.
228
+ function removeFiles(deps, projectId, orphaned) {
229
+ const kept = new Set(outputsOf(deps.db, projectId).map((output) => output.path));
230
+ for (const stage of stagesOf(deps.db, projectId)) {
231
+ for (const piece of allPiecesOf(deps.db, stage.id)) {
232
+ const file = pieceFile(piece.payload);
233
+ if (file !== undefined) {
234
+ kept.add(file);
235
+ }
236
+ }
237
+ }
238
+ for (const path of new Set(orphaned)) {
239
+ if (kept.has(path)) {
240
+ continue;
241
+ }
242
+ try {
243
+ rmSync(outputPath(deps.paths, projectId, path), { force: true });
244
+ }
245
+ catch (error) {
246
+ // The row is already gone, so the next boot's reconcile removes the file. Failing
247
+ // the action here would leave the user with a stage that cannot be re-run.
248
+ deps.log.write("warn", "reruns.file", {
249
+ projectId,
250
+ detail: `${path}: ${messageOf(error)}`,
251
+ });
252
+ }
253
+ }
254
+ }
255
+ // `logic/01`: `done` → `running` is scenario 12's own transition, and a stage that failed
256
+ // or was canceled may be started over rather than resumed. `provided` and `skipped` never
257
+ // run, and `pending` has nothing to redo.
258
+ function rerunnable(state) {
259
+ return state === "done" || state === "failed" || state === "canceled";
260
+ }
261
+ function stageOf(stages, kind) {
262
+ return stages.find((stage) => stage.kind === kind);
263
+ }
264
+ function imageOf(outputs, outputId) {
265
+ return outputs.find((output) => output.id === outputId && output.role === "image");
266
+ }
267
+ function messageOf(error) {
268
+ return error instanceof Error ? error.message : String(error);
269
+ }
@@ -0,0 +1,81 @@
1
+ export function plannerMessages(brief) {
2
+ return [
3
+ {
4
+ role: "user",
5
+ content: [
6
+ "You are planning the web research for an article.",
7
+ "",
8
+ briefText(brief),
9
+ "",
10
+ // §Q53: the prompt's section guide decides the chapters when it has one, and the
11
+ // planner proposes them when it does not. There is no cap on the count.
12
+ "List the chapters to research. If the prompt sets out a section guide, take the",
13
+ "chapters from it, in the order it gives them. If it does not, propose the chapters",
14
+ "this article needs.",
15
+ "",
16
+ "Answer with one chapter title per line and nothing else: no numbering, no",
17
+ "commentary, no blank lines, no heading.",
18
+ ].join("\n"),
19
+ },
20
+ ];
21
+ }
22
+ // The answer is a list, however the model chose to punctuate it. Numbering and bullets
23
+ // are stripped because the instruction above asks for neither and models add them
24
+ // anyway; a repeated title is dropped because `logic/06`'s invariant is that no chapter
25
+ // is researched twice.
26
+ export function chaptersFrom(text) {
27
+ const seen = new Set();
28
+ const chapters = [];
29
+ for (const line of text.split("\n")) {
30
+ const title = line
31
+ .trim()
32
+ .replace(/^(?:[-*•]|\d+[.)])\s+/, "")
33
+ .trim();
34
+ const key = title.toLowerCase();
35
+ if (title === "" || seen.has(key)) {
36
+ continue;
37
+ }
38
+ seen.add(key);
39
+ chapters.push(title);
40
+ }
41
+ return chapters;
42
+ }
43
+ // §Q52, §Q53: one sub-agent per chapter, web-grounded, answering that chapter's notes
44
+ // and its sources. The whole outline travels with it so it covers its own chapter and
45
+ // leaves the neighbouring ones to the sub-agents researching them.
46
+ export function subAgentMessages(brief, chapter, outline) {
47
+ return [
48
+ {
49
+ role: "user",
50
+ content: [
51
+ "You are researching one chapter of an article on the web.",
52
+ "",
53
+ briefText(brief),
54
+ "",
55
+ "The article's chapters:",
56
+ "",
57
+ outline.map((title) => `- ${title}`).join("\n"),
58
+ "",
59
+ `Research this chapter and no other: ${chapter}`,
60
+ "",
61
+ "Search the web and answer with plain-text notes on what you found: no markdown",
62
+ "and no commentary on your own process. End with a line reading exactly",
63
+ '"Sources", then the URL of every page you used, one per line.',
64
+ ].join("\n"),
65
+ },
66
+ ];
67
+ }
68
+ // The half of every instruction that is the same for all three calls (§Q46): the
69
+ // rendered prompt the article will be written from, and the run's keyword values.
70
+ export function briefText(brief) {
71
+ const values = Object.entries(brief.values);
72
+ return [
73
+ "The article will be written from this prompt:",
74
+ "",
75
+ brief.articlePrompt,
76
+ "",
77
+ "Keyword values for this run:",
78
+ "",
79
+ values.length === 0 ? "(none)" : values.map(([name, value]) => `${name}: ${value}`).join("\n"),
80
+ ].join("\n");
81
+ }
@@ -0,0 +1,192 @@
1
+ import { z } from "zod";
2
+ import { transact } from "../../kernel/db/tx.js";
3
+ import { isProviderError } from "../../kernel/ports/model.js";
4
+ import { insertPiece, piecesOf, setPiece } from "../../kernel/runner/piece-repo.js";
5
+ import { projectById, setStageProgress } from "../admission/repo.js";
6
+ import { storeText } from "../storage/staging.js";
7
+ import { noTokens, plusUsage } from "../telemetry/model.js";
8
+ import { chaptersFrom, plannerMessages, subAgentMessages } from "./planner.js";
9
+ import { sourcedAnswer, synthesisMessages } from "./synthesis.js";
10
+ // §Q47, verbatim: "the stage fails immediately with 'web research unsupported by this
11
+ // model'; no fallback to model knowledge".
12
+ export const webResearchUnsupported = "web research unsupported by this model";
13
+ // What a chapter piece carries between runs: its title, and its notes once a sub-agent
14
+ // has answered. That is the whole of the resume of §Q54.
15
+ const chapterPayload = z.object({ title: z.string(), notes: z.string().optional() });
16
+ export async function runResearch(deps, context, providers) {
17
+ const { projectId } = context.stage;
18
+ const project = projectById(deps.db, projectId);
19
+ if (project === undefined) {
20
+ throw new Error(`project ${projectId} has no row`);
21
+ }
22
+ const choice = project.config.llm;
23
+ const articlePrompt = project.config.rendered.article;
24
+ if (choice === undefined || articlePrompt === undefined) {
25
+ // Admission refuses a run whose research is Generate without both (`logic/04`), so
26
+ // reaching here is a bug in admission rather than something the user did.
27
+ throw new Error("the run has no LLM provider or no rendered article prompt");
28
+ }
29
+ const brief = { articlePrompt, values: project.config.values };
30
+ try {
31
+ await research(deps, context, providers, choice, brief);
32
+ }
33
+ catch (error) {
34
+ // The adapter says the model cannot ground on the web and the wrapper has already
35
+ // made that terminal; this is where it becomes the sentence the user reads (§Q47).
36
+ if (isProviderError(error) && error.fault.kind === "unsupported") {
37
+ throw new Error(webResearchUnsupported);
38
+ }
39
+ throw error;
40
+ }
41
+ }
42
+ async function research(deps, context, providers, choice, brief) {
43
+ // logic/16 step 3 counts tokens per stage, so the planner, every sub-agent and the
44
+ // synthesis add into one total. A call that was aborted or that failed never answers,
45
+ // so it adds nothing (§Q112); a provider that reports no usage adds zero, never an
46
+ // estimate (§Q131).
47
+ let tokens = noTokens;
48
+ const add = (usage) => {
49
+ tokens = plusUsage(tokens, usage);
50
+ };
51
+ const chapters = await plan(deps, context, providers, choice, brief, add);
52
+ const findings = await researchChapters(deps, context, providers, choice, brief, chapters, add);
53
+ const answer = await providers.llm({
54
+ provider: choice.provider,
55
+ model: choice.model,
56
+ messages: synthesisMessages(brief, findings),
57
+ check: (given) => sourcedAnswer("the synthesis", given.text),
58
+ });
59
+ add(answer.usage);
60
+ const { projectId } = context.stage;
61
+ // Step 4: the notes and every instruction sent are stored on the project. Each
62
+ // sub-agent's own output stays on its chapter row, which is where the resume reads it.
63
+ storeText(deps, { projectId, stageKind: "research", role: "notes", text: answer.text.trim() });
64
+ storeText(deps, {
65
+ projectId,
66
+ stageKind: "research",
67
+ role: "instructions",
68
+ text: instructionsText(brief, findings),
69
+ });
70
+ deps.count("stage.completed", {
71
+ stage: "research",
72
+ provider: choice.provider,
73
+ model: choice.model,
74
+ ...tokens,
75
+ });
76
+ deps.log.write("info", "research.done", {
77
+ projectId,
78
+ stage: "research",
79
+ detail: `${String(findings.length)} chapters researched`,
80
+ });
81
+ }
82
+ // The chapter list, planned once and kept. A retry after a failure finds the rows the
83
+ // first run wrote and does not ask again (§Q54).
84
+ async function plan(deps, context, providers, choice, brief, add) {
85
+ const existing = piecesOf(deps.db, context.stage.id, "chapter");
86
+ if (existing.length > 0) {
87
+ return existing;
88
+ }
89
+ const answer = await providers.llm({
90
+ provider: choice.provider,
91
+ model: choice.model,
92
+ messages: plannerMessages(brief),
93
+ // §Q50: an empty answer, or one with no chapter in it, is a failed attempt.
94
+ check: (given) => chaptersFrom(given.text).length === 0 ? "the planner named no chapters" : undefined,
95
+ });
96
+ add(answer.usage);
97
+ // §Q53: no cap on the count. The list is the prompt's own section guide as often as
98
+ // not, and capping it would silently drop a section the article asks for.
99
+ const planned = chaptersFrom(answer.text).map((title, index) => ({
100
+ id: deps.ids.next(),
101
+ stageId: context.stage.id,
102
+ kind: "chapter",
103
+ idx: index + 1,
104
+ state: "pending",
105
+ payload: JSON.stringify({ title }),
106
+ }));
107
+ transact(deps.db, () => {
108
+ for (const piece of planned) {
109
+ insertPiece(deps.db, piece);
110
+ }
111
+ });
112
+ return planned;
113
+ }
114
+ // §Q53: "one per chapter, all in parallel". §Q54: a chapter a previous run finished is
115
+ // not researched again, and one that fails fails the whole stage while its siblings
116
+ // finish and keep their output for the next resume.
117
+ async function researchChapters(deps, context, providers, choice, brief, chapters, add) {
118
+ const outline = chapters.map((piece) => payloadOf(piece).title);
119
+ const total = chapters.length;
120
+ let done = chapters.filter((piece) => payloadOf(piece).notes !== undefined).length;
121
+ report(deps, context, done, total);
122
+ const outcomes = await Promise.all(chapters.map(async (piece) => {
123
+ const kept = payloadOf(piece);
124
+ if (kept.notes !== undefined) {
125
+ return { ok: true, finding: { title: kept.title, notes: kept.notes } };
126
+ }
127
+ setPiece(deps.db, piece.id, "running", piece.payload);
128
+ try {
129
+ const answer = await providers.forPiece(piece.id).llm({
130
+ provider: choice.provider,
131
+ model: choice.model,
132
+ messages: subAgentMessages(brief, kept.title, outline),
133
+ // §Q47: grounding is asked for explicitly, so a model without it says so
134
+ // instead of answering from what it already knows.
135
+ webSearch: true,
136
+ check: (given) => sourcedAnswer(`the researcher on "${kept.title}"`, given.text),
137
+ });
138
+ add(answer.usage);
139
+ const notes = answer.text.trim();
140
+ setPiece(deps.db, piece.id, "done", JSON.stringify({ title: kept.title, notes }));
141
+ done += 1;
142
+ report(deps, context, done, total);
143
+ return { ok: true, finding: { title: kept.title, notes } };
144
+ }
145
+ catch (error) {
146
+ // A cancel is not this chapter failing: `logic/13` §Q112 counts an aborted call
147
+ // as nothing, and the resume runs a `pending` chapter exactly as a failed one.
148
+ setPiece(deps.db, piece.id, context.signal.aborted ? "pending" : "failed", piece.payload);
149
+ return { ok: false, error };
150
+ }
151
+ }));
152
+ const findings = [];
153
+ for (const outcome of outcomes) {
154
+ if (!outcome.ok) {
155
+ throw outcome.error;
156
+ }
157
+ findings.push(outcome.finding);
158
+ }
159
+ return findings;
160
+ }
161
+ // Step 5: "k of N chapters researched". Written as well as emitted, so a page opened
162
+ // mid-stage reads the count off the row rather than waiting for the next chapter.
163
+ function report(deps, context, done, total) {
164
+ setStageProgress(deps.db, context.stage.id, done, total);
165
+ context.emit({
166
+ type: "stage.progress",
167
+ projectId: context.stage.projectId,
168
+ stage: "research",
169
+ current: done,
170
+ total,
171
+ });
172
+ }
173
+ // §Q51: every instruction sent is stored on the project. All three are pure functions of
174
+ // the brief and the findings, so a resumed run reproduces the ones it did not send.
175
+ function instructionsText(brief, findings) {
176
+ const outline = findings.map((finding) => finding.title);
177
+ const parts = [section("Planner", plannerMessages(brief))];
178
+ for (const title of outline) {
179
+ parts.push(section(`Sub-agent: ${title}`, subAgentMessages(brief, title, outline)));
180
+ }
181
+ parts.push(section("Synthesis", synthesisMessages(brief, findings)));
182
+ return `${parts.join("\n\n")}\n`;
183
+ }
184
+ function section(label, messages) {
185
+ return `=== ${label} ===\n\n${messages.map((message) => message.content).join("\n\n")}`;
186
+ }
187
+ function payloadOf(piece) {
188
+ if (piece.payload === null) {
189
+ throw new Error(`chapter ${piece.id} has no payload`);
190
+ }
191
+ return chapterPayload.parse(JSON.parse(piece.payload));
192
+ }
@@ -0,0 +1,48 @@
1
+ import { briefText } from "./planner.js";
2
+ // §Q52: "an editorial pass over every sub-agent's output that selects and organizes the
3
+ // findings; it does not concatenate". The sub-agents' notes are the material, not the
4
+ // answer, and the synthesising model is told so.
5
+ export function synthesisMessages(brief, findings) {
6
+ return [
7
+ {
8
+ role: "user",
9
+ content: [
10
+ "You are the editor of the research behind an article.",
11
+ "",
12
+ briefText(brief),
13
+ "",
14
+ "One researcher covered each chapter. Their findings follow.",
15
+ "",
16
+ findings.map(block).join("\n\n"),
17
+ "",
18
+ "Write the research notes the article will be written from. Select what the",
19
+ "article needs, organise it, and resolve what the researchers disagree on. Do not",
20
+ "concatenate their reports and do not add anything they did not find.",
21
+ "",
22
+ "Answer with plain-text notes: no markdown and no commentary on your own process.",
23
+ 'End with a line reading exactly "Sources", then the URL of every source you kept,',
24
+ "one per line, with no duplicates.",
25
+ ].join("\n"),
26
+ },
27
+ ];
28
+ }
29
+ // §Q48 and §Q55: the stored notes always end with a Sources list, and an answer without
30
+ // one "counts as a failed attempt". A heading with nothing under it is not a list, so
31
+ // the line has to be followed by something.
32
+ const heading = /^\s*(?:#{1,6}\s*)?(?:\*\*)?\s*sources(?:\s+(?:list|consulted))?\s*(?:\*\*)?\s*:?\s*$/i;
33
+ export function endsWithSources(text) {
34
+ const lines = text.split("\n");
35
+ const at = lines.findLastIndex((line) => heading.test(line));
36
+ return at !== -1 && lines.slice(at + 1).some((line) => line.trim() !== "");
37
+ }
38
+ // The one shape a stage hands `StageProviders.llm` as its `check`: an answer that arrived
39
+ // but cannot be used is a failed attempt, so the wrapper retries it (§Q50, §Q55).
40
+ export function sourcedAnswer(who, text) {
41
+ if (text.trim() === "") {
42
+ return `${who} answered with nothing`;
43
+ }
44
+ return endsWithSources(text) ? undefined : `${who} answered with no Sources list`;
45
+ }
46
+ function block(finding) {
47
+ return `--- ${finding.title} ---\n${finding.notes}`;
48
+ }
@@ -0,0 +1,37 @@
1
+ import { execFile } from "node:child_process";
2
+ // ceiling: `logic/02` §Q135 computes readiness at request time, so nothing is cached and
3
+ // a hung CLI costs the settings page this long. Both probes run in parallel, so the page
4
+ // waits 2 s in the worst case. Caching the answer for a few seconds is the upgrade if a
5
+ // slow machine makes the status line flap.
6
+ export const cliProbeTimeoutMs = 2000;
7
+ const probeOutputMax = 64 * 1024;
8
+ // The real probe: an argument array, never a shell string, so nothing in a binary name
9
+ // or an argument can be interpreted as a command. It resolves for every outcome; a
10
+ // missing CLI is an answer, not a failure.
11
+ // ceiling: POSIX only. A Windows install puts `claude` on PATH as a `.cmd` shim, which
12
+ // execFile cannot run without a shell; a shim-aware lookup is the upgrade when Windows
13
+ // is supported.
14
+ export function nodeCliProbe(binary, args, timeoutMs) {
15
+ return new Promise((resolve) => {
16
+ execFile(binary, [...args], { timeout: timeoutMs, maxBuffer: probeOutputMax, windowsHide: true }, (error, stdout) => {
17
+ resolve(error === null ? { ran: true, stdout } : { ran: false, stdout: "" });
18
+ });
19
+ });
20
+ }
21
+ export async function cliReadiness(probe, provider) {
22
+ const result = await probe(provider.binary, provider.versionArgs, cliProbeTimeoutMs);
23
+ if (!result.ran) {
24
+ return { kind: "cli", installed: false };
25
+ }
26
+ const version = versionFrom(result.stdout);
27
+ return version === undefined
28
+ ? { kind: "cli", installed: true }
29
+ : { kind: "cli", installed: true, version };
30
+ }
31
+ // `claude --version` answers "2.1.258 (Claude Code)" and `codex --version` answers
32
+ // "codex-cli 0.149.1", so the version is the first dotted number on the output rather
33
+ // than the whole line either of them prints.
34
+ export function versionFrom(stdout) {
35
+ const match = /\d+\.\d+(?:\.[0-9A-Za-z.+-]+)?/.exec(stdout);
36
+ return match?.[0];
37
+ }