@gentbajko/slopify 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +63 -0
  3. package/dist/adapter-registry.js +68 -0
  4. package/dist/adapters/image/bytes.js +64 -0
  5. package/dist/adapters/image/fal.js +164 -0
  6. package/dist/adapters/image/openai.js +155 -0
  7. package/dist/adapters/image/replicate.js +183 -0
  8. package/dist/adapters/llm/claude-code.js +158 -0
  9. package/dist/adapters/llm/codex.js +126 -0
  10. package/dist/adapters/llm/openrouter.js +193 -0
  11. package/dist/adapters/llm/run-cli.js +134 -0
  12. package/dist/adapters/llm/sse-lines.js +46 -0
  13. package/dist/adapters/retry-after.js +21 -0
  14. package/dist/adapters/tts/cartesia.js +105 -0
  15. package/dist/adapters/tts/elevenlabs.js +107 -0
  16. package/dist/adapters/tts/openai.js +95 -0
  17. package/dist/edge/cli.js +36 -0
  18. package/dist/edge/events/hub.js +76 -0
  19. package/dist/edge/http/actions.js +125 -0
  20. package/dist/edge/http/app.js +72 -0
  21. package/dist/edge/http/entries.js +41 -0
  22. package/dist/edge/http/files.js +67 -0
  23. package/dist/edge/http/problem.js +65 -0
  24. package/dist/edge/http/projects.js +128 -0
  25. package/dist/edge/http/prompts.js +73 -0
  26. package/dist/edge/http/providers.js +62 -0
  27. package/dist/edge/http/settings.js +90 -0
  28. package/dist/edge/http/staging.js +104 -0
  29. package/dist/edge/http/telemetry.js +29 -0
  30. package/dist/edge/http/usage.js +19 -0
  31. package/dist/edge/open-browser.js +20 -0
  32. package/dist/kernel/clock.js +18 -0
  33. package/dist/kernel/config/index.js +32 -0
  34. package/dist/kernel/db/index.js +24 -0
  35. package/dist/kernel/db/migrate.js +59 -0
  36. package/dist/kernel/db/migrations/0001-init.sql +17 -0
  37. package/dist/kernel/db/tx.js +53 -0
  38. package/dist/kernel/events.js +1 -0
  39. package/dist/kernel/ids.js +4 -0
  40. package/dist/kernel/lock.js +73 -0
  41. package/dist/kernel/log.js +34 -0
  42. package/dist/kernel/paths.js +21 -0
  43. package/dist/kernel/pipeline.js +22 -0
  44. package/dist/kernel/ports/image.js +1 -0
  45. package/dist/kernel/ports/llm.js +1 -0
  46. package/dist/kernel/ports/model.js +34 -0
  47. package/dist/kernel/ports/registry.js +1 -0
  48. package/dist/kernel/ports/tts.js +1 -0
  49. package/dist/kernel/runner/attempt-repo.js +50 -0
  50. package/dist/kernel/runner/attempt.js +146 -0
  51. package/dist/kernel/runner/graph.js +82 -0
  52. package/dist/kernel/runner/index.js +205 -0
  53. package/dist/kernel/runner/piece-repo.js +58 -0
  54. package/dist/kernel/runner/providers.js +92 -0
  55. package/dist/kernel/version.js +14 -0
  56. package/dist/main.js +212 -0
  57. package/dist/slices/admission/model.js +6 -0
  58. package/dist/slices/admission/repo.js +178 -0
  59. package/dist/slices/admission/rules.js +201 -0
  60. package/dist/slices/admission/start.js +103 -0
  61. package/dist/slices/admission/substitute.js +80 -0
  62. package/dist/slices/article/continuation.js +81 -0
  63. package/dist/slices/article/plain.js +14 -0
  64. package/dist/slices/article/run.js +166 -0
  65. package/dist/slices/article/split.js +51 -0
  66. package/dist/slices/article/store.js +21 -0
  67. package/dist/slices/cancel/index.js +61 -0
  68. package/dist/slices/images/run.js +221 -0
  69. package/dist/slices/library/lint.js +53 -0
  70. package/dist/slices/library/model.js +13 -0
  71. package/dist/slices/library/repo.js +111 -0
  72. package/dist/slices/library/save.js +87 -0
  73. package/dist/slices/library/slots.js +86 -0
  74. package/dist/slices/narration/chunk.js +71 -0
  75. package/dist/slices/narration/concat.js +79 -0
  76. package/dist/slices/narration/run.js +280 -0
  77. package/dist/slices/reruns/cascade.js +66 -0
  78. package/dist/slices/reruns/index.js +269 -0
  79. package/dist/slices/research/planner.js +81 -0
  80. package/dist/slices/research/run.js +192 -0
  81. package/dist/slices/research/synthesis.js +48 -0
  82. package/dist/slices/settings/cli-status.js +37 -0
  83. package/dist/slices/settings/keys.js +55 -0
  84. package/dist/slices/settings/model.js +54 -0
  85. package/dist/slices/settings/playback.js +60 -0
  86. package/dist/slices/settings/readiness.js +21 -0
  87. package/dist/slices/settings/repo.js +59 -0
  88. package/dist/slices/settings/voices.js +48 -0
  89. package/dist/slices/storage/asset-name.js +16 -0
  90. package/dist/slices/storage/delete-project.js +35 -0
  91. package/dist/slices/storage/downloads.js +100 -0
  92. package/dist/slices/storage/layout.js +59 -0
  93. package/dist/slices/storage/model.js +20 -0
  94. package/dist/slices/storage/reconcile.js +102 -0
  95. package/dist/slices/storage/repo.js +104 -0
  96. package/dist/slices/storage/staging.js +224 -0
  97. package/dist/slices/telemetry/collector-client.js +51 -0
  98. package/dist/slices/telemetry/flush.js +111 -0
  99. package/dist/slices/telemetry/machine.js +33 -0
  100. package/dist/slices/telemetry/model.js +39 -0
  101. package/dist/slices/telemetry/record.js +41 -0
  102. package/dist/slices/telemetry/repo.js +73 -0
  103. package/dist/slices/telemetry/usage.js +82 -0
  104. package/dist/slices/thumbnail/by-llm.js +30 -0
  105. package/dist/slices/thumbnail/run.js +200 -0
  106. package/dist/slices/video/ffmpeg.js +230 -0
  107. package/dist/slices/video/plan.js +71 -0
  108. package/dist/slices/video/run.js +179 -0
  109. package/dist/web/app-icon.svg +1 -0
  110. package/dist/web/assets/barlow-condensed-latin-600-normal-BFJEwTuo.woff +0 -0
  111. package/dist/web/assets/barlow-condensed-latin-600-normal-DepVgxBB.woff2 +0 -0
  112. package/dist/web/assets/barlow-condensed-latin-700-normal-Dmwat-ge.woff +0 -0
  113. package/dist/web/assets/barlow-condensed-latin-700-normal-v1xN8_Wq.woff2 +0 -0
  114. package/dist/web/assets/barlow-condensed-latin-ext-600-normal-18ESti3H.woff2 +0 -0
  115. package/dist/web/assets/barlow-condensed-latin-ext-600-normal-Clv9cIcR.woff +0 -0
  116. package/dist/web/assets/barlow-condensed-latin-ext-700-normal-BIHFfxf0.woff +0 -0
  117. package/dist/web/assets/barlow-condensed-latin-ext-700-normal-CwuXbfVR.woff2 +0 -0
  118. package/dist/web/assets/barlow-condensed-vietnamese-600-normal-A5AYRdjN.woff2 +0 -0
  119. package/dist/web/assets/barlow-condensed-vietnamese-600-normal-CNlPk46_.woff +0 -0
  120. package/dist/web/assets/barlow-condensed-vietnamese-700-normal-DYeBwlKR.woff2 +0 -0
  121. package/dist/web/assets/barlow-condensed-vietnamese-700-normal-DhIzd8Tb.woff +0 -0
  122. package/dist/web/assets/barlow-latin-400-normal-fsAxiSwU.woff +0 -0
  123. package/dist/web/assets/barlow-latin-400-normal-qiz4-Cze.woff2 +0 -0
  124. package/dist/web/assets/barlow-latin-500-normal-BPAOfeC8.woff2 +0 -0
  125. package/dist/web/assets/barlow-latin-500-normal-C1h8hMer.woff +0 -0
  126. package/dist/web/assets/barlow-latin-600-normal-CNwfPWQD.woff +0 -0
  127. package/dist/web/assets/barlow-latin-600-normal-DILqtrty.woff2 +0 -0
  128. package/dist/web/assets/barlow-latin-700-normal-A9pxMQ4z.woff2 +0 -0
  129. package/dist/web/assets/barlow-latin-700-normal-__SGTsZ1.woff +0 -0
  130. package/dist/web/assets/barlow-latin-800-normal-BdVooDN4.woff +0 -0
  131. package/dist/web/assets/barlow-latin-800-normal-s1sAMnoV.woff2 +0 -0
  132. package/dist/web/assets/barlow-latin-ext-400-normal-CvBsJvxq.woff +0 -0
  133. package/dist/web/assets/barlow-latin-ext-400-normal-HxX4XjxC.woff2 +0 -0
  134. package/dist/web/assets/barlow-latin-ext-500-normal-CJPcKP2Q.woff +0 -0
  135. package/dist/web/assets/barlow-latin-ext-500-normal-DOaysfXq.woff2 +0 -0
  136. package/dist/web/assets/barlow-latin-ext-600-normal-B8NK_A3D.woff2 +0 -0
  137. package/dist/web/assets/barlow-latin-ext-600-normal-DMVRjfRT.woff +0 -0
  138. package/dist/web/assets/barlow-latin-ext-700-normal-BLuWmldJ.woff2 +0 -0
  139. package/dist/web/assets/barlow-latin-ext-700-normal-CctuGmmz.woff +0 -0
  140. package/dist/web/assets/barlow-latin-ext-800-normal-BiucknKG.woff2 +0 -0
  141. package/dist/web/assets/barlow-latin-ext-800-normal-D7I3yvUw.woff +0 -0
  142. package/dist/web/assets/barlow-vietnamese-400-normal-BFeobeCK.woff +0 -0
  143. package/dist/web/assets/barlow-vietnamese-400-normal-Dpl4UHAZ.woff2 +0 -0
  144. package/dist/web/assets/barlow-vietnamese-500-normal-GNfB7rCE.woff +0 -0
  145. package/dist/web/assets/barlow-vietnamese-500-normal-zTViEIzf.woff2 +0 -0
  146. package/dist/web/assets/barlow-vietnamese-600-normal-CA_GiK2e.woff +0 -0
  147. package/dist/web/assets/barlow-vietnamese-600-normal-DcjprdFV.woff2 +0 -0
  148. package/dist/web/assets/barlow-vietnamese-700-normal-4Jt4k04K.woff +0 -0
  149. package/dist/web/assets/barlow-vietnamese-700-normal-D6euyNzi.woff2 +0 -0
  150. package/dist/web/assets/barlow-vietnamese-800-normal-Cl1Mc_Dv.woff2 +0 -0
  151. package/dist/web/assets/barlow-vietnamese-800-normal-D0VWpbij.woff +0 -0
  152. package/dist/web/assets/index-BPqQnrpy.css +1 -0
  153. package/dist/web/assets/index-D_sWbKQi.js +81 -0
  154. package/dist/web/favicon.svg +1 -0
  155. package/dist/web/index.html +15 -0
  156. package/package.json +41 -0
@@ -0,0 +1,34 @@
1
+ // The vocabulary that crosses the provider seam. Domain types only: a vendor's payload
2
+ // shape never leaves its adapter (01-architecture §Q10, §Q33).
3
+ export const providerFamilies = ["llm", "tts", "image"];
4
+ export const providerErrorKinds = [
5
+ "auth",
6
+ // Distinct from `auth`, which is a key the provider rejected. `logic/02` §Q13 splits
7
+ // them: a bad key runs the whole retry policy (§Q11) and an absent one "fails
8
+ // immediately, without retries", so the wrapper needs two kinds to tell apart.
9
+ "missing_key",
10
+ "rate_limit",
11
+ "refusal",
12
+ "unsupported",
13
+ "timeout",
14
+ "other",
15
+ ];
16
+ export function providerError(init) {
17
+ return Object.assign(new Error(init.message), {
18
+ fault: {
19
+ kind: init.kind,
20
+ ...(init.retryAfterMs === undefined ? {} : { retryAfterMs: init.retryAfterMs }),
21
+ },
22
+ });
23
+ }
24
+ export function isProviderError(value) {
25
+ if (!(value instanceof Error) || !("fault" in value)) {
26
+ return false;
27
+ }
28
+ const fault = value.fault;
29
+ if (typeof fault !== "object" || fault === null || !("kind" in fault)) {
30
+ return false;
31
+ }
32
+ const kinds = providerErrorKinds;
33
+ return typeof fault.kind === "string" && kinds.includes(fault.kind);
34
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,50 @@
1
+ import { z } from "zod";
2
+ import { providerErrorKinds } from "../ports/model.js";
3
+ // One row per provider call attempt (02-models, `logic/01` §Q4). The project page reads
4
+ // the last error text off these rows and the Usage page counts them, so an attempt is
5
+ // written when it opens rather than when it ends: a process that dies mid-call leaves
6
+ // the evidence behind (`logic/01` §Q7).
7
+ export const attemptOutcomes = ["ok", "canceled", ...providerErrorKinds];
8
+ const attemptRow = z.object({
9
+ id: z.string(),
10
+ stage_id: z.string(),
11
+ piece_id: z.string().nullable(),
12
+ n: z.number(),
13
+ started_at: z.string(),
14
+ ended_at: z.string().nullable(),
15
+ outcome: z.enum(attemptOutcomes).nullable(),
16
+ error_text: z.string().nullable(),
17
+ });
18
+ export function sqliteAttempts(db, ids) {
19
+ return {
20
+ start: (attempt) => {
21
+ const id = ids.next();
22
+ db.prepare("INSERT INTO attempts (id, stage_id, piece_id, n, started_at) VALUES (?, ?, ?, ?, ?)").run(id, attempt.stageId, attempt.pieceId, attempt.n, attempt.startedAt);
23
+ // `logic/01` §Q3: the stage shows the attempt count beside the error. A stage with
24
+ // pieces (twenty images) shows the count of the call being attempted, not a sum.
25
+ db.prepare("UPDATE stages SET attempt_count = ? WHERE id = ?").run(attempt.n, attempt.stageId);
26
+ return id;
27
+ },
28
+ end: (id, ended) => {
29
+ db.prepare("UPDATE attempts SET ended_at = ?, outcome = ?, error_text = ? WHERE id = ?").run(ended.endedAt, ended.outcome, ended.errorText, id);
30
+ },
31
+ };
32
+ }
33
+ export function attemptsOf(db, stageId) {
34
+ return db
35
+ .prepare("SELECT * FROM attempts WHERE stage_id = ? ORDER BY rowid")
36
+ .all(stageId)
37
+ .map((row) => toAttempt(attemptRow.parse(row)));
38
+ }
39
+ function toAttempt(row) {
40
+ return {
41
+ id: row.id,
42
+ stageId: row.stage_id,
43
+ pieceId: row.piece_id,
44
+ n: row.n,
45
+ startedAt: row.started_at,
46
+ endedAt: row.ended_at,
47
+ outcome: row.outcome,
48
+ errorText: row.error_text,
49
+ };
50
+ }
@@ -0,0 +1,146 @@
1
+ import { redact } from "../log.js";
2
+ import { isProviderError, providerError } from "../ports/model.js";
3
+ // The retry policy of `logic/01` step 6, in one place. Every provider call in the app
4
+ // goes through here: a stage slice is handed the wrapped calls of `providers.ts` and
5
+ // never an adapter, so there is no way round it.
6
+ // "attempted up to 4 times: the first attempt plus 3 retries, waiting 2 s, 8 s, 30 s
7
+ // between attempts" (`logic/01` step 6, §Q4).
8
+ export const attemptLimit = 4;
9
+ export const backoffMs = [2000, 8000, 30_000];
10
+ // "each attempt times out at 120 s ... except image-generation calls, which time out at
11
+ // 300 s" (`logic/01` step 6, §Q62; `logic/09` §Q77).
12
+ export const timeoutMs = {
13
+ llm: 120_000,
14
+ tts: 120_000,
15
+ image: 300_000,
16
+ };
17
+ // `logic/09` §Q74 and `logic/06` §Q47: a refusal and an unsupported capability are the
18
+ // provider's final answer. Retrying either would only spend the user's money again.
19
+ // `logic/02` §Q13 adds the third: a key removed mid-run leaves nothing to call with, so
20
+ // "the next attempt finds no key, fails immediately without retries" - unlike a key the
21
+ // provider rejected, which is an `auth` failure and is retried like any other (§Q11).
22
+ const terminalKinds = ["refusal", "unsupported", "missing_key"];
23
+ export async function attempt(ctx, call, opts) {
24
+ ctx.signal.throwIfAborted();
25
+ const limit = timeoutMs[opts.kind];
26
+ for (let n = 1;; n += 1) {
27
+ const id = ctx.attempts.start({
28
+ stageId: ctx.stageId,
29
+ pieceId: ctx.pieceId ?? null,
30
+ n,
31
+ startedAt: ctx.clock.now().toISOString(),
32
+ });
33
+ const window = deadline(ctx.clock, ctx.signal, limit, opts.streaming === true);
34
+ let result;
35
+ try {
36
+ result = { ok: true, value: await call(window.signal, window.progress) };
37
+ }
38
+ catch (error) {
39
+ result = { ok: false, at: error };
40
+ }
41
+ const expired = window.expired();
42
+ window.dispose();
43
+ if (result.ok) {
44
+ ctx.attempts.end(id, {
45
+ outcome: "ok",
46
+ endedAt: ctx.clock.now().toISOString(),
47
+ errorText: null,
48
+ });
49
+ return result.value;
50
+ }
51
+ if (ctx.signal.aborted) {
52
+ // `logic/13` §Q112: an aborted call counts nothing. The row is closed so the page
53
+ // is not left with an attempt that never ended, and it carries no error text: the
54
+ // user stopped it, the provider did not fail.
55
+ ctx.attempts.end(id, {
56
+ outcome: "canceled",
57
+ endedAt: ctx.clock.now().toISOString(),
58
+ errorText: null,
59
+ });
60
+ throw result.at;
61
+ }
62
+ const failure = classify(result.at, expired, opts, limit);
63
+ ctx.attempts.end(id, {
64
+ outcome: failure.fault.kind,
65
+ endedAt: ctx.clock.now().toISOString(),
66
+ errorText: failure.message,
67
+ });
68
+ ctx.log.write("warn", "provider.attempt", {
69
+ projectId: ctx.projectId,
70
+ stage: ctx.stage,
71
+ detail: `${opts.kind} attempt ${n} of ${attemptLimit} failed (${failure.fault.kind}): ${failure.message}`,
72
+ });
73
+ if (terminalKinds.includes(failure.fault.kind) || n >= attemptLimit) {
74
+ throw failure;
75
+ }
76
+ // §Q4: a 429 that names a Retry-After replaces the fixed wait with the provider's.
77
+ await ctx.clock.sleep(failure.fault.retryAfterMs ?? backoffMs[n - 1] ?? 0, ctx.signal);
78
+ }
79
+ }
80
+ // The wrapper is the only place a provider failure is named (03-standards). Everything
81
+ // downstream reads `fault.kind` and shows `message`.
82
+ function classify(error, expired, opts, limit) {
83
+ if (expired) {
84
+ const seconds = Math.round(limit / 1000);
85
+ return providerError({
86
+ kind: "timeout",
87
+ message: opts.streaming === true
88
+ ? `the provider sent nothing for ${seconds} s`
89
+ : `the provider did not answer within ${seconds} s`,
90
+ });
91
+ }
92
+ if (isProviderError(error)) {
93
+ // Rebuilt rather than rethrown so the text that reaches the stage row, the page and
94
+ // the log has been through the redactor: a provider is free to quote the key back.
95
+ return providerError({
96
+ kind: error.fault.kind,
97
+ message: redact(error.message),
98
+ ...(error.fault.retryAfterMs === undefined ? {} : { retryAfterMs: error.fault.retryAfterMs }),
99
+ });
100
+ }
101
+ return providerError({
102
+ kind: "other",
103
+ message: redact(error instanceof Error ? error.message : String(error)),
104
+ });
105
+ }
106
+ // `logic/01` §Q62: the same number of milliseconds, measured two ways. `idle` restarts
107
+ // the count on every chunk, so a long answer that keeps arriving is never cut off, and a
108
+ // stream that stalls is. Composed with AbortSignal.any so the stage's cancel still wins.
109
+ function deadline(clock, parent, ms, idle) {
110
+ const timer = new AbortController();
111
+ const signal = AbortSignal.any([parent, timer.signal]);
112
+ let due = clock.now().getTime() + ms;
113
+ let expired = false;
114
+ let disposed = false;
115
+ const watch = async () => {
116
+ while (!disposed) {
117
+ const left = due - clock.now().getTime();
118
+ if (left <= 0) {
119
+ expired = true;
120
+ timer.abort(new Error(`the provider call passed its ${Math.round(ms / 1000)} s limit`));
121
+ return;
122
+ }
123
+ try {
124
+ await clock.sleep(left, timer.signal);
125
+ }
126
+ catch {
127
+ // Disposed, or the call already ended: nothing left to time.
128
+ return;
129
+ }
130
+ }
131
+ };
132
+ void watch();
133
+ return {
134
+ signal,
135
+ progress: () => {
136
+ if (idle) {
137
+ due = clock.now().getTime() + ms;
138
+ }
139
+ },
140
+ expired: () => expired,
141
+ dispose: () => {
142
+ disposed = true;
143
+ timer.abort();
144
+ },
145
+ };
146
+ }
@@ -0,0 +1,82 @@
1
+ // logic/01 steps 2-5: research → article → {audio ∥ images ∥ thumbnail} → video.
2
+ export const deps = {
3
+ research: [],
4
+ article: ["research"],
5
+ audio: ["article"],
6
+ images: ["article"],
7
+ thumbnail: ["article"],
8
+ video: ["audio", "images", "thumbnail"],
9
+ };
10
+ // logic/01 §Q2: a dependency is released by `provided` and `skipped` exactly as by `done`.
11
+ export function satisfied(state) {
12
+ return state === "done" || state === "provided" || state === "skipped";
13
+ }
14
+ // logic/01 §Q9, in its order. Derived on every read, never stored.
15
+ export function derive(stages) {
16
+ if (stages.some((stage) => stage.state === "running")) {
17
+ return "running";
18
+ }
19
+ if (stages.some((stage) => stage.state === "canceled")) {
20
+ return "canceled";
21
+ }
22
+ if (stages.some((stage) => stage.state === "failed")) {
23
+ return "failed";
24
+ }
25
+ if (stages.some((stage) => stage.kind === "video" && stage.state === "done")) {
26
+ return "done";
27
+ }
28
+ // Extended rule, not one of §Q9's four. §Q9 offers no answer for a project that matches
29
+ // none of them, and `pending` was chosen for the window between creating a project and
30
+ // the runner claiming its first stage. `logic/13` §Q113 makes a second case reachable:
31
+ // when the only running stage stores its output in the same instant as the cancel it
32
+ // stays `done`, so nothing is `canceled` and a run the user stopped would read as one
33
+ // about to start. `logic/13` step 3 is unconditional - "the project reads `canceled`" -
34
+ // so that is what a run which started and stopped reads here.
35
+ //
36
+ // A stage in a terminal state is what tells the two apart: at creation every stage is
37
+ // `pending`, `provided` or `skipped`, and only a stage the runner carried to the end is
38
+ // `done`. Nothing restarts such a project by itself (§Q111), so `pending` is the one
39
+ // answer that is certainly wrong.
40
+ //
41
+ // ceiling: a process killed between a stage finishing and its dependent being claimed
42
+ // leaves the same rows and reads `canceled` too - `logic/01` §Q7 only reaches a stage
43
+ // found `running`. Telling that apart needs the cancel recorded on the project, which is
44
+ // the upgrade if it ever matters; §Q9 keeps status derived and stores nothing.
45
+ if (stages.some((stage) => stage.state === "done")) {
46
+ return "canceled";
47
+ }
48
+ return "pending";
49
+ }
50
+ // The thin meter under a running row on 07 Projects, "averaging stage progress"
51
+ // (uiux/screens/07-projects.md). Every stage the run actually asks for counts once: a
52
+ // finished one whole, a running one by its own progress, a waiting, failed or canceled
53
+ // one not at all.
54
+ //
55
+ // `provided` and `skipped` are left out of the average rather than counted as finished.
56
+ // Nothing was asked of them, and counting them would put a run with research off and a
57
+ // supplied article at a third of the way along before the first call was made.
58
+ export function progressOf(stages) {
59
+ const asked = stages.filter((stage) => stage.state !== "provided" && stage.state !== "skipped");
60
+ // A run made entirely of supplied files has nothing outstanding, and a division by zero
61
+ // would answer NaN.
62
+ if (asked.length === 0) {
63
+ return 1;
64
+ }
65
+ const share = asked.reduce((total, stage) => total + shareOf(stage), 0);
66
+ return share / asked.length;
67
+ }
68
+ function shareOf(stage) {
69
+ if (satisfied(stage.state)) {
70
+ return 1;
71
+ }
72
+ if (stage.state !== "running") {
73
+ return 0;
74
+ }
75
+ const total = stage.progressTotal ?? 0;
76
+ if (total <= 0) {
77
+ // A stage that has not yet said how many chapters or chunks there are. Guessing would
78
+ // be a meter that moves backwards when the count arrives.
79
+ return 0;
80
+ }
81
+ return Math.min(1, Math.max(0, (stage.progressCurrent ?? 0) / total));
82
+ }
@@ -0,0 +1,205 @@
1
+ import { derive, deps as graph, satisfied } from "./graph.js";
2
+ export function createRunner(deps) {
3
+ const inflight = new Map();
4
+ // ceiling: one entry per project ticked since boot, never evicted, so it costs a few
5
+ // dozen bytes per project of a single user's local app and is emptied by any restart.
6
+ // Evicting on a terminal state would re-announce that state on the next tick of the
7
+ // same project, so the upgrade is to drop the entry when the project is deleted.
8
+ const announced = new Map();
9
+ // The projects a cancel is walking through. It is a set rather than a flag because
10
+ // `logic/13` D14 leaves every other project untouched, and it is held for the whole of
11
+ // abortProject rather than for the abort call alone: a stage of this project that
12
+ // stores its output in the same instant as the cancel stays `done` (§Q113), and its
13
+ // hand-over to the next stage is exactly what must not happen.
14
+ const stopped = new Set();
15
+ let count = 0;
16
+ let shuttingDown = false;
17
+ const emitStage = (stage, state, reason) => {
18
+ deps.emit(stage.projectId, {
19
+ type: "stage.state",
20
+ projectId: stage.projectId,
21
+ stage: stage.kind,
22
+ state,
23
+ ...(reason === null ? {} : { failureReason: reason }),
24
+ });
25
+ };
26
+ // The global tally is the number of projects with a stage in flight, so it is read off
27
+ // the in-flight set rather than counted in the database (04-data-flow, Run step 5).
28
+ const retally = () => {
29
+ const projects = new Set();
30
+ for (const entry of inflight.values()) {
31
+ projects.add(entry.projectId);
32
+ }
33
+ if (projects.size !== count) {
34
+ count = projects.size;
35
+ deps.emitRunningCount(count);
36
+ }
37
+ };
38
+ const announce = (projectId) => {
39
+ const state = derive(deps.stages.stagesOf(projectId));
40
+ if (announced.get(projectId) === state) {
41
+ return;
42
+ }
43
+ announced.set(projectId, state);
44
+ deps.emit(projectId, { type: "project.state", projectId, state });
45
+ };
46
+ const conclude = (stage, state, reason) => {
47
+ try {
48
+ deps.stages.finish(stage.id, state, reason);
49
+ }
50
+ catch (error) {
51
+ // The page would otherwise sit on a stage that never leaves `running`; the event
52
+ // still goes out, and the next boot marks the row interrupted (logic/01 §Q7).
53
+ deps.log.write("error", "stage.finish", {
54
+ projectId: stage.projectId,
55
+ stage: stage.kind,
56
+ detail: reasonOf(error),
57
+ });
58
+ }
59
+ emitStage(stage, state, reason);
60
+ };
61
+ async function execute(stage, controller) {
62
+ try {
63
+ const run = deps.runs[stage.kind];
64
+ if (run === undefined) {
65
+ throw new Error(`no implementation is registered for the ${stage.kind} stage`);
66
+ }
67
+ await run({
68
+ stage,
69
+ signal: controller.signal,
70
+ emit: (event) => {
71
+ deps.emit(stage.projectId, event);
72
+ },
73
+ });
74
+ conclude(stage, "done", null);
75
+ }
76
+ catch (error) {
77
+ if (controller.signal.aborted) {
78
+ // logic/13 step 3 fixes the wording; a late rejection from the aborted call is
79
+ // the expected way a stage learns it was canceled, so it is not logged as a fault.
80
+ conclude(stage, "canceled", "canceled by user");
81
+ return;
82
+ }
83
+ deps.log.write("error", "stage.failed", {
84
+ projectId: stage.projectId,
85
+ stage: stage.kind,
86
+ detail: reasonOf(error),
87
+ });
88
+ conclude(stage, "failed", reasonOf(error));
89
+ }
90
+ }
91
+ function start(stage) {
92
+ const controller = new AbortController();
93
+ let release = () => { };
94
+ const entry = {
95
+ projectId: stage.projectId,
96
+ controller,
97
+ settled: new Promise((resolve) => {
98
+ release = resolve;
99
+ }),
100
+ };
101
+ // Registered before the stage body is invoked: a tick arriving while this stage is
102
+ // starting has to see it in flight.
103
+ inflight.set(stage.id, entry);
104
+ retally();
105
+ emitStage(stage, "running", null);
106
+ void execute(stage, controller)
107
+ .finally(() => {
108
+ inflight.delete(stage.id);
109
+ release();
110
+ try {
111
+ // The next stage is claimed before the tally is read, so a fan-out handing over
112
+ // to the video stage never reports the project as briefly not running.
113
+ tick(stage.projectId);
114
+ }
115
+ finally {
116
+ // stagesOf parses rows and can throw. The tally is replayed to every page that
117
+ // opens afterwards, so it is read whether or not the tick got that far.
118
+ retally();
119
+ }
120
+ })
121
+ .catch((error) => {
122
+ deps.log.write("error", "stage.execute", {
123
+ projectId: stage.projectId,
124
+ stage: stage.kind,
125
+ detail: reasonOf(error),
126
+ });
127
+ });
128
+ }
129
+ function tick(projectId) {
130
+ // A stage that finished in the same instant as the shutdown or a cancel would
131
+ // otherwise release its dependent, and the abort would then be waiting on a render
132
+ // nobody asked for - or, worse, leaving one running under a controller nobody
133
+ // aborted. The project's state still goes out, so an open page sees the run stop.
134
+ if (!shuttingDown && !stopped.has(projectId)) {
135
+ startEligible(projectId);
136
+ }
137
+ announce(projectId);
138
+ }
139
+ function startEligible(projectId) {
140
+ const stages = deps.stages.stagesOf(projectId);
141
+ const stateOf = (kind) => stages.find((stage) => stage.kind === kind)?.state ?? "pending";
142
+ for (const stage of stages) {
143
+ if (stage.state !== "pending" || inflight.has(stage.id)) {
144
+ continue;
145
+ }
146
+ if (!graph[stage.kind].every((kind) => satisfied(stateOf(kind)))) {
147
+ continue;
148
+ }
149
+ // Nothing is awaited in this loop, so the fan-out after the article starts audio,
150
+ // images and thumbnail together rather than one after another.
151
+ if (deps.stages.claim(stage.id)) {
152
+ start({ ...stage, state: "running" });
153
+ }
154
+ }
155
+ }
156
+ async function settled() {
157
+ while (inflight.size > 0) {
158
+ await Promise.all([...inflight.values()].map((entry) => entry.settled));
159
+ }
160
+ }
161
+ async function settledOf(projectId) {
162
+ for (;;) {
163
+ const waiting = [...inflight.values()].filter((entry) => entry.projectId === projectId);
164
+ if (waiting.length === 0) {
165
+ return;
166
+ }
167
+ await Promise.all(waiting.map((entry) => entry.settled));
168
+ }
169
+ }
170
+ return {
171
+ tick,
172
+ settled,
173
+ abortProject: async (projectId) => {
174
+ // Up before the first abort and down only once every stage of the project has
175
+ // stopped: in between, the tick each finishing stage fires starts nothing.
176
+ stopped.add(projectId);
177
+ try {
178
+ for (const entry of inflight.values()) {
179
+ if (entry.projectId === projectId) {
180
+ entry.controller.abort();
181
+ }
182
+ }
183
+ // §Q109's invariant: "no provider call of the project continues after cancel
184
+ // returns". Waiting here is what makes that true rather than hoped for.
185
+ await settledOf(projectId);
186
+ }
187
+ finally {
188
+ stopped.delete(projectId);
189
+ }
190
+ // The last stage's own tick announced the project while the barrier was up and the
191
+ // rows may not all have been written yet; this is the state the user asked for.
192
+ announce(projectId);
193
+ },
194
+ abortAll: async () => {
195
+ shuttingDown = true;
196
+ for (const entry of inflight.values()) {
197
+ entry.controller.abort();
198
+ }
199
+ await settled();
200
+ },
201
+ };
202
+ }
203
+ function reasonOf(error) {
204
+ return error instanceof Error ? error.message : String(error);
205
+ }
@@ -0,0 +1,58 @@
1
+ import { z } from "zod";
2
+ // One row per resumable sub-unit of a stage (02-models, `stage_pieces`): a research
3
+ // chapter, an audio chunk, an image index. It sits beside the attempt store because the
4
+ // attempt wrapper stamps `piece_id` on every row it writes and the kernel may not import
5
+ // the slices that fill these in. The payload is carried as the JSON text it is stored
6
+ // as: its shape belongs to whichever stage owns the kind, and nothing here reads inside.
7
+ export const pieceKinds = ["chapter", "chunk", "segment", "image", "prompt_written"];
8
+ export const pieceStates = ["pending", "running", "done", "failed"];
9
+ const pieceRow = z.object({
10
+ id: z.string(),
11
+ stage_id: z.string(),
12
+ kind: z.enum(pieceKinds),
13
+ idx: z.number(),
14
+ state: z.enum(pieceStates),
15
+ payload: z.string().nullable(),
16
+ });
17
+ export function insertPiece(db, piece) {
18
+ db.prepare("INSERT INTO stage_pieces (id, stage_id, kind, idx, state, payload) VALUES (?, ?, ?, ?, ?, ?)").run(piece.id, piece.stageId, piece.kind, piece.idx, piece.state, piece.payload);
19
+ }
20
+ // In `idx` order: the resume of `logic/06` §Q54 re-runs the pieces a previous attempt did
21
+ // not finish and keeps the rest, and the order they were planned in is the order the
22
+ // stage's output is assembled in.
23
+ export function piecesOf(db, stageId, kind) {
24
+ return db
25
+ .prepare("SELECT * FROM stage_pieces WHERE stage_id = ? AND kind = ? ORDER BY idx")
26
+ .all(stageId, kind)
27
+ .map((row) => toPiece(pieceRow.parse(row)));
28
+ }
29
+ // Every piece of a stage whatever its kind, for the one caller that does not care: a
30
+ // re-run throws the whole plan away, because the text a chunk was cut from and the prompt
31
+ // an image was sent with are what changed (`logic/12` §Q101, §Q104).
32
+ export function allPiecesOf(db, stageId) {
33
+ return db
34
+ .prepare("SELECT * FROM stage_pieces WHERE stage_id = ? ORDER BY kind, idx")
35
+ .all(stageId)
36
+ .map((row) => toPiece(pieceRow.parse(row)));
37
+ }
38
+ export function deletePieces(db, stageId) {
39
+ db.prepare("DELETE FROM stage_pieces WHERE stage_id = ?").run(stageId);
40
+ }
41
+ // One piece, for `logic/09` §Q75: an image the user deleted leaves no row behind, or the
42
+ // next run of the stage would look at the plan and make it again.
43
+ export function deletePiece(db, id) {
44
+ db.prepare("DELETE FROM stage_pieces WHERE id = ?").run(id);
45
+ }
46
+ export function setPiece(db, id, state, payload) {
47
+ db.prepare("UPDATE stage_pieces SET state = ?, payload = ? WHERE id = ?").run(state, payload, id);
48
+ }
49
+ function toPiece(row) {
50
+ return {
51
+ id: row.id,
52
+ stageId: row.stage_id,
53
+ kind: row.kind,
54
+ idx: row.idx,
55
+ state: row.state,
56
+ payload: row.payload,
57
+ };
58
+ }
@@ -0,0 +1,92 @@
1
+ import { attempt } from "./attempt.js";
2
+ export function stageProviders(deps, context, pieceId) {
3
+ const ctx = {
4
+ clock: deps.clock,
5
+ log: deps.log,
6
+ attempts: deps.attempts,
7
+ projectId: context.stage.projectId,
8
+ stage: context.stage.kind,
9
+ stageId: context.stage.id,
10
+ ...(pieceId === undefined ? {} : { pieceId }),
11
+ signal: context.signal,
12
+ };
13
+ return {
14
+ llm: (call, onEvent) => {
15
+ // Resolved once: an adapter reads the stored key when it builds each request, so a
16
+ // key replaced mid-run still reaches the next attempt (`logic/02` §Q16).
17
+ const port = deps.registry.llm(call.provider);
18
+ return attempt(ctx, async (signal, progress) => {
19
+ let text = "";
20
+ let usage = null;
21
+ let finishReason = null;
22
+ for await (const event of port.complete({
23
+ model: call.model,
24
+ messages: call.messages,
25
+ ...(call.webSearch === undefined ? {} : { webSearch: call.webSearch }),
26
+ signal,
27
+ })) {
28
+ // Every event is a sign of life, so the idle clock restarts here rather than
29
+ // in each slice that consumes one.
30
+ progress();
31
+ if (event.type === "delta") {
32
+ text += event.text;
33
+ }
34
+ else {
35
+ usage = event.usage;
36
+ finishReason = event.finishReason;
37
+ }
38
+ onEvent?.(event);
39
+ }
40
+ const answer = { text, usage, finishReason };
41
+ const unusable = call.check?.(answer);
42
+ if (unusable !== undefined) {
43
+ // Thrown bare: the wrapper is the one place a failure is named, and it names
44
+ // this `other`, which is retried like any other bad answer.
45
+ throw new Error(unusable);
46
+ }
47
+ return answer;
48
+ }, { kind: "llm", streaming: port.capabilities.streams });
49
+ },
50
+ tts: (call) => {
51
+ const port = deps.registry.tts(call.provider);
52
+ return attempt(ctx, async (signal, progress) => {
53
+ const spoken = await port.synthesize({
54
+ voiceId: call.voiceId,
55
+ text: call.text,
56
+ signal,
57
+ });
58
+ const reader = spoken.audio.getReader();
59
+ const chunks = [];
60
+ let total = 0;
61
+ for (;;) {
62
+ const { done, value } = await reader.read();
63
+ if (done || value === undefined) {
64
+ break;
65
+ }
66
+ progress();
67
+ chunks.push(value);
68
+ total += value.length;
69
+ }
70
+ const bytes = new Uint8Array(total);
71
+ let at = 0;
72
+ for (const chunk of chunks) {
73
+ bytes.set(chunk, at);
74
+ at += chunk.length;
75
+ }
76
+ return { bytes, container: spoken.container };
77
+ }, { kind: "tts", streaming: port.capabilities.streams });
78
+ },
79
+ image: (call) => {
80
+ const port = deps.registry.image(call.provider);
81
+ return attempt(ctx, (signal) => port.generate({
82
+ model: call.model,
83
+ prompt: call.prompt,
84
+ aspect: call.aspect,
85
+ signal,
86
+ }),
87
+ // One request, one answer: the 300 s runs over the whole call (`logic/09` §Q77).
88
+ { kind: "image" });
89
+ },
90
+ forPiece: (piece) => stageProviders(deps, context, piece),
91
+ };
92
+ }