@gentbajko/slopify 0.6.1 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/README.md +36 -1
  2. package/SUBTITLES.md +3 -3
  3. package/dist/adapter-registry.js +12 -2
  4. package/dist/adapters/image/google.js +12 -15
  5. package/dist/adapters/image/models.js +72 -0
  6. package/dist/adapters/image/openai.js +3 -5
  7. package/dist/adapters/llm/catalogue-files.js +23 -0
  8. package/dist/adapters/llm/claude-code.js +3 -4
  9. package/dist/adapters/llm/codex-models.js +46 -0
  10. package/dist/adapters/llm/codex.js +7 -10
  11. package/dist/adapters/llm/gemini-models.js +85 -0
  12. package/dist/adapters/llm/gemini-workspace.js +24 -1
  13. package/dist/adapters/llm/gemini.js +4 -7
  14. package/dist/adapters/llm/openrouter.js +12 -2
  15. package/dist/adapters/tts/cartesia.js +9 -1
  16. package/dist/adapters/tts/elevenlabs.js +33 -2
  17. package/dist/adapters/tts/inworld-async.js +150 -0
  18. package/dist/adapters/tts/inworld-text.js +32 -0
  19. package/dist/adapters/tts/inworld.js +170 -0
  20. package/dist/adapters/tts/openai.js +31 -4
  21. package/dist/assets/models.yaml +470 -0
  22. package/dist/catalog/registry.js +147 -0
  23. package/dist/catalog/schema.js +91 -0
  24. package/dist/catalog/store.js +90 -0
  25. package/dist/catalog/validate.js +43 -0
  26. package/dist/edge/cli.js +5 -0
  27. package/dist/edge/events/hub.js +9 -2
  28. package/dist/edge/events/preview-cache.js +40 -0
  29. package/dist/edge/http/actions.js +2 -0
  30. package/dist/edge/http/app.js +28 -1
  31. package/dist/edge/http/audio-preview.js +46 -0
  32. package/dist/edge/http/planning.js +101 -0
  33. package/dist/edge/http/projects.js +4 -2
  34. package/dist/edge/http/providers.js +44 -0
  35. package/dist/edge/http/update.js +57 -0
  36. package/dist/edge/update-worker.js +44 -0
  37. package/dist/kernel/audio-preview.js +188 -0
  38. package/dist/kernel/db/migrations/0003-batch-queue.sql +11 -0
  39. package/dist/kernel/ports/llm.js +1 -0
  40. package/dist/kernel/ports/text.js +26 -0
  41. package/dist/kernel/runner/providers.js +80 -28
  42. package/dist/kernel/runner/queue.js +65 -0
  43. package/dist/main.js +142 -27
  44. package/dist/model-catalog.js +37 -0
  45. package/dist/slices/admission/repo.js +6 -1
  46. package/dist/slices/admission/start.js +9 -8
  47. package/dist/slices/article/continuation.js +8 -1
  48. package/dist/slices/article/segments.js +1 -0
  49. package/dist/slices/batch/index.js +82 -0
  50. package/dist/slices/control/index.js +6 -1
  51. package/dist/slices/control/providers.js +16 -1
  52. package/dist/slices/estimate/index.js +81 -0
  53. package/dist/slices/narration/live.js +21 -0
  54. package/dist/slices/narration/plan.js +58 -0
  55. package/dist/slices/narration/run.js +15 -36
  56. package/dist/slices/research/run.js +7 -0
  57. package/dist/slices/settings/model.js +2 -0
  58. package/dist/slices/settings/models.js +89 -0
  59. package/dist/slices/storage/staging.js +2 -1
  60. package/dist/slices/subtitles/captions.js +4 -1
  61. package/dist/slices/subtitles/layout.js +19 -0
  62. package/dist/slices/subtitles/model.js +9 -0
  63. package/dist/slices/subtitles/prepare.js +6 -1
  64. package/dist/slices/thumbnail/run.js +3 -0
  65. package/dist/updater/candidate.js +28 -0
  66. package/dist/updater/forward.js +35 -0
  67. package/dist/updater/install-flow.js +26 -0
  68. package/dist/updater/install.js +50 -0
  69. package/dist/updater/model.js +24 -0
  70. package/dist/updater/plan.js +135 -0
  71. package/dist/updater/readiness.js +15 -0
  72. package/dist/updater/registry.js +19 -0
  73. package/dist/updater/service.js +128 -0
  74. package/dist/updater/worker.js +136 -0
  75. package/dist/web/assets/index-6zz8telY.css +1 -0
  76. package/dist/web/assets/index-CbYEcBOa.js +130 -0
  77. package/dist/web/index.html +2 -2
  78. package/package.json +2 -1
  79. package/dist/web/assets/index-CNYs0noB.js +0 -130
  80. package/dist/web/assets/index-Mk0bvBg-.css +0 -1
package/dist/main.js CHANGED
@@ -1,3 +1,5 @@
1
+ import { randomBytes } from "node:crypto";
2
+ import { existsSync } from "node:fs";
1
3
  import { fileURLToPath } from "node:url";
2
4
  import { serve } from "@hono/node-server";
3
5
  import ffmpegStatic from "ffmpeg-static";
@@ -5,11 +7,11 @@ import { buildRegistry } from "./adapter-registry.js";
5
7
  import { alignSubtitles } from "./adapters/alignment/index.js";
6
8
  import { prepareFfmpeg } from "./adapters/ffmpeg.js";
7
9
  import { nodeRunCli } from "./adapters/llm/run-cli.js";
8
- import { cartesiaModel } from "./adapters/tts/cartesia.js";
9
- import { elevenLabsModel } from "./adapters/tts/elevenlabs.js";
10
- import { openAiTtsModel } from "./adapters/tts/openai.js";
10
+ import { curateRegistry } from "./catalog/registry.js";
11
+ import { createCatalogueStore } from "./catalog/store.js";
11
12
  import { createHub } from "./edge/events/hub.js";
12
13
  import { createApp } from "./edge/http/app.js";
14
+ import { createAudioPreviewStore } from "./kernel/audio-preview.js";
13
15
  import { systemClock } from "./kernel/clock.js";
14
16
  import { openDb } from "./kernel/db/index.js";
15
17
  import { migrate } from "./kernel/db/migrate.js";
@@ -21,10 +23,13 @@ import { sqliteAttempts } from "./kernel/runner/attempt-repo.js";
21
23
  import { dependenciesOf } from "./kernel/runner/graph.js";
22
24
  import { createRunner } from "./kernel/runner/index.js";
23
25
  import { stageProviders } from "./kernel/runner/providers.js";
26
+ import { createProviderQueue } from "./kernel/runner/queue.js";
24
27
  import { readVersion } from "./kernel/version.js";
28
+ import { modelSources } from "./model-catalog.js";
25
29
  import { claimStage, finishStage, projectById, projectPaused, stagesOf, } from "./slices/admission/repo.js";
26
30
  import { prepareProvidedArticleSegments } from "./slices/article/provided-entries.js";
27
31
  import { runArticle } from "./slices/article/run.js";
32
+ import { pumpQueue, queueWaiting } from "./slices/batch/index.js";
28
33
  import { runImages } from "./slices/images/run.js";
29
34
  import { runNarration } from "./slices/narration/run.js";
30
35
  import { runResearch } from "./slices/research/run.js";
@@ -35,6 +40,12 @@ import { createFlusher } from "./slices/telemetry/flush.js";
35
40
  import { record } from "./slices/telemetry/record.js";
36
41
  import { runThumbnail } from "./slices/thumbnail/run.js";
37
42
  import { renderVideo } from "./slices/video/run.js";
43
+ import { watchActivation } from "./updater/candidate.js";
44
+ import { launchUpdate } from "./updater/install.js";
45
+ import { isUpdateToken } from "./updater/model.js";
46
+ import { npmCommand, updateCommitted } from "./updater/plan.js";
47
+ import { publishedVersion } from "./updater/registry.js";
48
+ import { createUpdater } from "./updater/service.js";
38
49
  // ceiling: a burst of finished stages coalesces into one delivery a second later, and the
39
50
  // collector gets ten seconds to answer before the attempt is abandoned and the events
40
51
  // stay queued.
@@ -71,32 +82,101 @@ export async function boot(config) {
71
82
  log,
72
83
  post: httpPostEvents(collectorEndpoint(process.env), collectorTimeoutMs),
73
84
  }, flushDelayMs);
74
- const registry = buildRegistry({
85
+ const catalogue = createCatalogueStore({ dataDir: paths.dataDir, fetch: globalThis.fetch });
86
+ const registry = curateRegistry(buildRegistry({
75
87
  db,
76
88
  fetch: globalThis.fetch,
77
89
  spawn: nodeRunCli,
78
90
  clock,
79
91
  probe: nodeCliProbe,
92
+ }), catalogue);
93
+ const audioPreviews = createAudioPreviewStore();
94
+ const runner = wire({
95
+ db,
96
+ paths,
97
+ clock,
98
+ ids,
99
+ log,
100
+ hub,
101
+ telemetry,
102
+ flusher,
103
+ registry,
104
+ catalogue,
105
+ ffmpeg,
106
+ audioPreviews,
107
+ });
108
+ const updateDb = db;
109
+ const oldEntry = fileURLToPath(new URL("./edge/cli.js", import.meta.url));
110
+ const workerEntry = fileURLToPath(new URL("./edge/update-worker.js", import.meta.url));
111
+ let npm;
112
+ try {
113
+ npm = await npmCommand();
114
+ }
115
+ catch {
116
+ log.write("warn", "update", {
117
+ detail: "npm is unavailable; in-app installation is disabled.",
118
+ });
119
+ }
120
+ let shutdown = async () => {
121
+ throw new Error("The server is not ready.");
122
+ };
123
+ let listeningPort = config.port;
124
+ const candidateToken = process.env.SLOPIFY_UPDATE_TOKEN ?? "";
125
+ const pendingActivation = isUpdateToken(candidateToken) && process.env.SLOPIFY_UPDATE_PENDING === "1";
126
+ const updater = createUpdater({
127
+ ...(isUpdateToken(candidateToken)
128
+ ? {
129
+ candidate: {
130
+ token: candidateToken,
131
+ pending: pendingActivation,
132
+ committed: () => updateCommitted(paths.dataDir, version, candidateToken),
133
+ },
134
+ }
135
+ : {}),
136
+ currentVersion: version,
137
+ previousUpdateFailed: process.env.SLOPIFY_UPDATE_FAILED === "1",
138
+ now: () => Date.now(),
139
+ latest: () => publishedVersion(globalThis.fetch),
140
+ unsupported: () => !existsSync(oldEntry) || !existsSync(workerEntry)
141
+ ? "Run Slopify from its installed package to use in-app updates."
142
+ : npm === undefined
143
+ ? "npm is unavailable. Install Node.js with npm to use in-app updates."
144
+ : undefined,
145
+ busy: () => updateDb.prepare("SELECT 1 FROM stages WHERE state = 'running' LIMIT 1").get() !==
146
+ undefined ||
147
+ updateDb
148
+ .prepare("SELECT id FROM projects")
149
+ .all()
150
+ .some((row) => typeof row.id === "string" && runner.hasInflight?.(row.id) === true),
151
+ report: (message) => log.write("warn", "update", { detail: message }),
152
+ install: async (next, restarting) => {
153
+ if (npm === undefined)
154
+ throw new Error("npm is unavailable.");
155
+ await launchUpdate(workerEntry, {
156
+ version: next,
157
+ token: randomBytes(32).toString("hex"),
158
+ previousVersion: version,
159
+ oldEntry,
160
+ dataDir: paths.dataDir,
161
+ cwd: process.cwd(),
162
+ host: config.host,
163
+ port: listeningPort,
164
+ npm: { file: npm.file, args: [...npm.args] },
165
+ }, async () => {
166
+ restarting();
167
+ await shutdown();
168
+ });
169
+ },
80
170
  });
81
- const runner = wire({ db, paths, clock, ids, log, hub, telemetry, flusher, registry, ffmpeg });
82
171
  const app = createApp({
83
172
  db,
84
173
  paths,
85
174
  hub,
86
175
  runner,
87
- modelsFor: (provider, family) => {
88
- if (family === "llm")
89
- return registry.llm(provider).models();
90
- if (family === "image")
91
- return registry.image(provider).models();
92
- const models = {
93
- elevenlabs: elevenLabsModel,
94
- "openai-tts": openAiTtsModel,
95
- cartesia: cartesiaModel,
96
- };
97
- const id = models[provider];
98
- return Promise.resolve(id === undefined ? [] : [{ id, name: id }]);
99
- },
176
+ updater,
177
+ audioPreviews,
178
+ ...modelSources(registry),
179
+ catalogue,
100
180
  clock,
101
181
  ids,
102
182
  log,
@@ -106,14 +186,32 @@ export async function boot(config) {
106
186
  probe: nodeCliProbe,
107
187
  });
108
188
  const server = await listen(app, config, log);
189
+ const queueTimer = setInterval(() => {
190
+ const release = updater.beginMutation();
191
+ if (!release)
192
+ return;
193
+ try {
194
+ pumpQueue(updateDb, runner);
195
+ }
196
+ catch {
197
+ log.write("error", "batch.queue", { detail: "The batch queue could not advance." });
198
+ }
199
+ finally {
200
+ release();
201
+ }
202
+ }, 1000);
203
+ listeningPort = portOf(server) ?? config.port;
109
204
  // Whatever last run left queued goes out at start. Nothing waits for
110
205
  // it, and an unreachable collector costs one refused socket.
111
206
  flusher.soon();
112
207
  const open = db;
113
- return {
114
- paths,
115
- url: urlOf(config.host, portOf(server) ?? config.port),
116
- stop: async () => {
208
+ let stopping;
209
+ let stopActivation = () => { };
210
+ shutdown = () => {
211
+ stopping ??= (async () => {
212
+ clearInterval(queueTimer);
213
+ stopActivation();
214
+ audioPreviews.close();
117
215
  try {
118
216
  // The listener goes first. Aborting the runner while the socket still accepted
119
217
  // requests let a Play arriving during the await start a stage under a fresh
@@ -132,8 +230,12 @@ export async function boot(config) {
132
230
  open.close();
133
231
  lock.release();
134
232
  }
135
- },
233
+ })();
234
+ return stopping;
136
235
  };
236
+ if (pendingActivation)
237
+ stopActivation = watchActivation(updater, candidateToken, shutdown, (message) => log.write("warn", "update", { detail: message }));
238
+ return { paths, url: urlOf(config.host, listeningPort), stop: shutdown };
137
239
  }
138
240
  catch (error) {
139
241
  db?.close();
@@ -141,7 +243,7 @@ export async function boot(config) {
141
243
  throw error;
142
244
  }
143
245
  }
144
- function wire({ db, paths, clock, ids, log, hub, telemetry, flusher, registry, ffmpeg, }) {
246
+ function wire({ audioPreviews, db, paths, clock, ids, log, hub, telemetry, flusher, registry, catalogue, ffmpeg, }) {
145
247
  // A stage counts what it did and the queue is flushed after each new event. `record`
146
248
  // swallows its own failures, so this can neither fail a stage nor widen what leaves the
147
249
  // machine - the payload allow-list is checked inside it.
@@ -157,11 +259,17 @@ function wire({ db, paths, clock, ids, log, hub, telemetry, flusher, registry, f
157
259
  const writing = { db, paths, ids, clock, log, count };
158
260
  // A stage slice is handed the wrapped calls, never the registry: every provider call
159
261
  // it makes is already inside the retry policy (kernel/runner/providers.ts).
160
- const providers = { registry, attempts: sqliteAttempts(db, ids), clock, log };
262
+ const providers = {
263
+ registry,
264
+ attempts: sqliteAttempts(db, ids),
265
+ clock,
266
+ log,
267
+ queue: createProviderQueue((provider) => catalogue.read().providers[provider]?.maxConcurrent ?? 1),
268
+ };
161
269
  return createRunner({
162
270
  stages: {
163
271
  stagesOf: (projectId) => stagesOf(db, projectId),
164
- paused: (projectId) => projectPaused(db, projectId),
272
+ paused: (projectId) => projectPaused(db, projectId) || queueWaiting(db, projectId),
165
273
  dependenciesOf: (projectId, kind) => {
166
274
  const project = projectById(db, projectId);
167
275
  return project === undefined ? [] : dependenciesOf(kind, project.config.sources);
@@ -177,7 +285,14 @@ function wire({ db, paths, clock, ids, log, hub, telemetry, flusher, registry, f
177
285
  audio: async (context) => {
178
286
  const wrapped = stageProviders(providers, context);
179
287
  await prepareProvidedArticleSegments(writing, context, wrapped);
180
- await runNarration(video, context, wrapped);
288
+ await runNarration({
289
+ ...video,
290
+ audioPreviews,
291
+ maxCharacters: (provider, model) => {
292
+ const entry = catalogue.models(provider, "tts").find((m) => m.id === model);
293
+ return entry && "tts" in entry ? entry.tts.maxCharacters : 4000;
294
+ },
295
+ }, context, wrapped);
181
296
  },
182
297
  images: (context) => runImages(writing, context, stageProviders(providers, context)),
183
298
  thumbnail: (context) => runThumbnail(writing, context, stageProviders(providers, context)),
@@ -0,0 +1,37 @@
1
+ import { falModels } from "./adapters/image/fal.js";
2
+ import { googleImageModels } from "./adapters/image/google.js";
3
+ import { openAiImageModels } from "./adapters/image/openai.js";
4
+ import { replicateModels } from "./adapters/image/replicate.js";
5
+ import { claudeCodeModels } from "./adapters/llm/claude-code.js";
6
+ import { codexModels } from "./adapters/llm/codex.js";
7
+ import { geminiModels } from "./adapters/llm/gemini.js";
8
+ import { cartesiaModels } from "./adapters/tts/cartesia.js";
9
+ import { elevenLabsModels } from "./adapters/tts/elevenlabs.js";
10
+ import { inworldModels } from "./adapters/tts/inworld.js";
11
+ import { openAiTtsModels } from "./adapters/tts/openai.js";
12
+ // Composition keeps provider-specific fallback data out of the HTTP and settings layers.
13
+ export function modelSources(registry) {
14
+ const fallbacks = {
15
+ "claude-code": claudeCodeModels,
16
+ codex: codexModels,
17
+ gemini: geminiModels,
18
+ elevenlabs: elevenLabsModels,
19
+ "openai-tts": openAiTtsModels,
20
+ cartesia: cartesiaModels,
21
+ inworld: inworldModels,
22
+ fal: falModels,
23
+ replicate: replicateModels,
24
+ "openai-image": openAiImageModels,
25
+ "google-image": googleImageModels,
26
+ };
27
+ return {
28
+ modelsFor: (provider, family) => {
29
+ if (family === "llm")
30
+ return registry.llm(provider).models();
31
+ if (family === "image")
32
+ return registry.image(provider).models();
33
+ return registry.tts(provider).models();
34
+ },
35
+ fallbackModelsFor: (provider) => Object.hasOwn(fallbacks, provider) ? (fallbacks[provider] ?? []) : [],
36
+ };
37
+ }
@@ -1,9 +1,14 @@
1
1
  import { z } from "zod";
2
2
  import { stageKinds, stageStates } from "../../kernel/pipeline.js";
3
+ import { thinkingModes } from "../../kernel/ports/llm.js";
3
4
  import { chunkModes } from "../narration/chunk.js";
4
5
  import { subtitleConfigSchema } from "../subtitles/model.js";
5
6
  import { entryModes, formats, stageSources } from "./model.js";
6
- const providerChoice = z.object({ provider: z.string(), model: z.string() });
7
+ const providerChoice = z.object({
8
+ provider: z.string(),
9
+ model: z.string(),
10
+ thinking: z.enum(thinkingModes).optional(),
11
+ });
7
12
  const entryChoice = z.object({ name: z.string(), mode: z.enum(entryModes) });
8
13
  // The shape Play posts and the shape `projects.config` holds, in one place: the second
9
14
  // is the first plus the rendered prompt texts.
@@ -13,7 +13,7 @@ export function initialState(source) {
13
13
  }
14
14
  // The project row, its six stages and the provided content all land together or not at
15
15
  // all. The caller ticks the runner after this returns, never inside it.
16
- export function startRun(deps, draft, rendered) {
16
+ export function startRun(deps, draft, rendered, retainStaged = false) {
17
17
  const id = deps.ids.next();
18
18
  const at = deps.clock.now().toISOString();
19
19
  const config = { ...draft, rendered };
@@ -49,14 +49,14 @@ export function startRun(deps, draft, rendered) {
49
49
  for (const stage of stages) {
50
50
  insertStage(deps.db, stage);
51
51
  }
52
- attachProvided(deps, id, draft, moved);
52
+ attachProvided(deps, id, draft, moved, retainStaged);
53
53
  });
54
- for (const source of moved) {
54
+ for (const source of retainStaged ? [] : moved) {
55
55
  dropStagedSource(deps, source);
56
56
  }
57
57
  return { project, stages };
58
58
  }
59
- function attachProvided(deps, projectId, draft, collected) {
59
+ function attachProvided(deps, projectId, draft, collected, retainStaged) {
60
60
  const { sources, provided } = draft;
61
61
  if (sources.research === "provide" && provided.research !== undefined) {
62
62
  storeText(deps, {
@@ -73,24 +73,25 @@ function attachProvided(deps, projectId, draft, collected) {
73
73
  storeArticleText(deps, { projectId, markdown: provided.article.trim() });
74
74
  }
75
75
  if (sources.audio === "provide") {
76
- attach(deps, projectId, "audio", provided.audio, "audio_body", collected);
76
+ attach(deps, projectId, "audio", provided.audio, "audio_body", collected, retainStaged);
77
77
  }
78
78
  if (sources.thumbnail === "provide") {
79
- attach(deps, projectId, "thumbnail", provided.thumbnail, "thumbnail", collected);
79
+ attach(deps, projectId, "thumbnail", provided.thumbnail, "thumbnail", collected, retainStaged);
80
80
  }
81
81
  if (sources.images === "provide") {
82
82
  // Slideshow order is the order the user left the list in.
83
83
  for (const [index, stagedFileId] of (provided.images ?? []).entries()) {
84
- attach(deps, projectId, "images", stagedFileId, "image", collected, index + 1);
84
+ attach(deps, projectId, "images", stagedFileId, "image", collected, retainStaged, index + 1);
85
85
  }
86
86
  }
87
87
  }
88
- function attach(deps, projectId, kind, stagedFileId, role, collected, index) {
88
+ function attach(deps, projectId, kind, stagedFileId, role, collected, retainStaged, index) {
89
89
  if (stagedFileId === undefined) {
90
90
  throw new Error(`the ${kind} stage is provided but carries no staged file`);
91
91
  }
92
92
  const result = attachStagedFile(deps, {
93
93
  stagedFileId,
94
+ retainStaged,
94
95
  projectId,
95
96
  role,
96
97
  ...(index === undefined ? {} : { index }),
@@ -43,7 +43,13 @@ export async function writeArticle(providers, choice, brief, onDelta) {
43
43
  onDelta(event.text);
44
44
  }
45
45
  };
46
- let answer = await providers.llm({ provider: choice.provider, model: choice.model, messages: base, check: written }, stream);
46
+ let answer = await providers.llm({
47
+ provider: choice.provider,
48
+ model: choice.model,
49
+ ...(choice.thinking === undefined ? {} : { thinking: choice.thinking }),
50
+ messages: base,
51
+ check: written,
52
+ }, stream);
47
53
  const pieces = [answer.text];
48
54
  let tokens = plusUsage(noTokens, answer.usage);
49
55
  for (let n = 1; truncated(answer) && n <= continuationLimit; n += 1) {
@@ -52,6 +58,7 @@ export async function writeArticle(providers, choice, brief, onDelta) {
52
58
  answer = await providers.llm({
53
59
  provider: choice.provider,
54
60
  model: choice.model,
61
+ ...(choice.thinking === undefined ? {} : { thinking: choice.thinking }),
55
62
  messages,
56
63
  // The last continuation allowed is the one that has to end the article: a fourth
57
64
  // truncation is a failed attempt, which is the wrapper's to retry. The
@@ -18,6 +18,7 @@ export async function writeSegment(providers, choice, config, category, article,
18
18
  const answer = await providers.llm({
19
19
  provider: choice.provider,
20
20
  model: choice.model,
21
+ ...(choice.thinking === undefined ? {} : { thinking: choice.thinking }),
21
22
  messages,
22
23
  check: (given) => given.text.trim() === "" ? `the ${category} answered with nothing` : undefined,
23
24
  });
@@ -0,0 +1,82 @@
1
+ import { z } from "zod";
2
+ import { transact } from "../../kernel/db/tx.js";
3
+ import { derive } from "../../kernel/runner/graph.js";
4
+ import { projectPaused, stagesOf } from "../admission/repo.js";
5
+ import { startRun } from "../admission/start.js";
6
+ import { stagingPath } from "../storage/layout.js";
7
+ import { deleteStagedFile, stagedFileById } from "../storage/repo.js";
8
+ import { dropStagedSource } from "../storage/staging.js";
9
+ const queueRow = z.object({
10
+ projectId: z.string(),
11
+ batchId: z.string(),
12
+ position: z.number(),
13
+ state: z.enum(["queued", "active", "finished"]),
14
+ });
15
+ export function queueEntries(db, batchId) {
16
+ return db
17
+ .prepare(`SELECT project_id AS projectId, batch_id AS batchId, position, state
18
+ FROM project_queue ${batchId === undefined ? "WHERE state != 'finished'" : "WHERE batch_id = ?"}
19
+ ORDER BY position`)
20
+ .all(...(batchId === undefined ? [] : [batchId]))
21
+ .map((row) => queueRow.parse(row));
22
+ }
23
+ export function queueWaiting(db, projectId) {
24
+ return (db
25
+ .prepare("SELECT 1 FROM project_queue WHERE project_id = ? AND state = 'queued'")
26
+ .get(projectId) !== undefined);
27
+ }
28
+ export function batchExists(db, id) {
29
+ return db.prepare("SELECT 1 FROM batches WHERE id = ?").get(id) !== undefined;
30
+ }
31
+ export function enqueueBatch(deps, batchId, runs) {
32
+ if (batchExists(deps.db, batchId))
33
+ return queueEntries(deps.db, batchId);
34
+ const sources = new Set();
35
+ transact(deps.db, () => {
36
+ deps.db
37
+ .prepare("INSERT INTO batches(id, created_at) VALUES (?, ?)")
38
+ .run(batchId, deps.clock.now().toISOString());
39
+ const used = new Set();
40
+ for (const { draft, rendered } of runs) {
41
+ const { project } = startRun(deps, draft, rendered, true);
42
+ deps.db
43
+ .prepare("INSERT INTO project_queue(project_id, batch_id) VALUES (?, ?)")
44
+ .run(project.id, batchId);
45
+ if (draft.sources.audio === "provide" && draft.provided.audio)
46
+ used.add(draft.provided.audio);
47
+ if (draft.sources.thumbnail === "provide" && draft.provided.thumbnail)
48
+ used.add(draft.provided.thumbnail);
49
+ if (draft.sources.images === "provide")
50
+ for (const id of draft.provided.images ?? [])
51
+ used.add(id);
52
+ }
53
+ for (const id of used) {
54
+ const file = stagedFileById(deps.db, id);
55
+ if (file)
56
+ sources.add(stagingPath(deps.paths, file.path));
57
+ deleteStagedFile(deps.db, id);
58
+ }
59
+ });
60
+ for (const source of sources)
61
+ dropStagedSource(deps, source);
62
+ return queueEntries(deps.db, batchId);
63
+ }
64
+ // Only one batch video runs at once. A paused item holds its place; a failed or
65
+ // canceled item releases the next video once its provider calls have drained.
66
+ export function pumpQueue(db, runner) {
67
+ for (const entry of queueEntries(db)) {
68
+ const stages = stagesOf(db, entry.projectId);
69
+ const status = derive(stages, projectPaused(db, entry.projectId));
70
+ if (runner.hasInflight?.(entry.projectId))
71
+ return;
72
+ if (["done", "failed", "canceled"].includes(status)) {
73
+ db.prepare("UPDATE project_queue SET state = 'finished' WHERE project_id = ?").run(entry.projectId);
74
+ continue;
75
+ }
76
+ if (status === "paused")
77
+ return;
78
+ db.prepare("UPDATE project_queue SET state = 'active' WHERE project_id = ?").run(entry.projectId);
79
+ runner.tick(entry.projectId);
80
+ return;
81
+ }
82
+ }
@@ -15,6 +15,7 @@ async function providerFields(deps, changes) {
15
15
  providers: await deps.providers(),
16
16
  voices: listVoices(deps.db),
17
17
  modelsFor: deps.modelsFor,
18
+ allowsCustomModels: deps.allowsCustomModels,
18
19
  }))(),
19
20
  new Promise((_resolve, reject) => {
20
21
  timeout = setTimeout(() => reject(new Error("Provider catalog timed out")), providerCheckTimeoutMs);
@@ -136,13 +137,17 @@ export function changeProviders(deps, id, changes) {
136
137
  (changes.audio.provider !== project.config.audio?.provider ||
137
138
  changes.audio.model !== project.config.audio?.model ||
138
139
  changes.audio.voice !== project.config.audio?.voice);
140
+ const chunkingChanged = changes.chunking !== undefined &&
141
+ (changes.chunking.mode !== (project.config.chunking?.mode ?? "whole") ||
142
+ (changes.chunking.words ?? 500) !== (project.config.chunking?.words ?? 500));
139
143
  const orphaned = transact(deps.db, () => {
140
- const files = audioChanged ? clearUnfinishedAudio(deps, id) : [];
144
+ const files = audioChanged || chunkingChanged ? clearUnfinishedAudio(deps, id) : [];
141
145
  updateProjectConfig(deps.db, id, {
142
146
  ...project.config,
143
147
  ...(changes.llm === undefined ? {} : { llm: changes.llm }),
144
148
  ...(changes.audio === undefined ? {} : { audio: changes.audio }),
145
149
  ...(changes.images === undefined ? {} : { images: changes.images }),
150
+ ...(changes.chunking === undefined ? {} : { chunking: changes.chunking }),
146
151
  }, deps.clock.now().toISOString());
147
152
  return files;
148
153
  });
@@ -1,8 +1,11 @@
1
1
  import { z } from "zod";
2
+ import { thinkingModes } from "../../kernel/ports/llm.js";
3
+ import { chunkModes } from "../narration/chunk.js";
2
4
  const choice = z
3
5
  .object({
4
6
  provider: z.string().trim().min(1).max(200),
5
7
  model: z.string().trim().min(1).max(200),
8
+ thinking: z.enum(thinkingModes).optional(),
6
9
  })
7
10
  .strict();
8
11
  export const providerChangesSchema = z
@@ -10,6 +13,10 @@ export const providerChangesSchema = z
10
13
  llm: choice.optional(),
11
14
  audio: choice.extend({ voice: z.string().trim().min(1).max(200) }).optional(),
12
15
  images: choice.optional(),
16
+ chunking: z
17
+ .object({ mode: z.enum(chunkModes), words: z.number().int().min(1).max(10000).optional() })
18
+ .strict()
19
+ .optional(),
13
20
  })
14
21
  .strict()
15
22
  .refine((value) => Object.keys(value).length > 0, "Pick at least one provider to update.");
@@ -50,10 +57,18 @@ export async function validateProviderChanges(changes, current) {
50
57
  const picked = changes[key];
51
58
  if (picked === undefined || fields.some((field) => field.field === key))
52
59
  continue;
60
+ if (current.allowsCustomModels?.(picked.provider) === true)
61
+ continue;
53
62
  const models = await current.modelsFor(picked.provider, families[key]);
54
- if (!models.some((model) => model.id === picked.model)) {
63
+ const model = models.find((model) => model.id === picked.model);
64
+ if (!model) {
55
65
  fields.push({ field: `${key}.model`, message: "Choose a model supported by this provider." });
56
66
  }
67
+ else if (key === "llm" &&
68
+ picked.thinking &&
69
+ !model.thinkingModes?.includes(picked.thinking)) {
70
+ fields.push({ field: `${key}.thinking`, message: "Choose a supported thinking setting." });
71
+ }
57
72
  }
58
73
  return fields;
59
74
  }
@@ -0,0 +1,81 @@
1
+ export function estimateRun(draft, rendered, expectedWords, catalogue) {
2
+ const rows = [];
3
+ const model = (family, choice) => choice
4
+ ? catalogue?.models(choice.provider, family).find((m) => m.id === choice.model)
5
+ : undefined;
6
+ const llm = model("llm", draft.llm);
7
+ const tts = model("tts", draft.audio);
8
+ const image = model("image", draft.images);
9
+ const generatedArticle = draft.sources.article === "generate";
10
+ const articleChars = generatedArticle ? expectedWords * 6 : (draft.provided.article?.length ?? 0);
11
+ const promptChars = Object.values(rendered).reduce((sum, value) => sum + value.length, 0);
12
+ const row = (stage, amount, detail, variable = false) => {
13
+ rows.push({
14
+ stage,
15
+ low: amount === undefined ? null : amount * (variable ? 0.5 : 1),
16
+ high: amount === undefined ? null : amount * (variable ? 1.5 : 1),
17
+ detail,
18
+ });
19
+ };
20
+ const textCost = (input, output) => {
21
+ const p = llm?.pricing;
22
+ return p?.inputPerMillionTokens === undefined || p.outputPerMillionTokens === undefined
23
+ ? undefined
24
+ : ((input / 4) * p.inputPerMillionTokens + (output / 4) * p.outputPerMillionTokens) / 1000000;
25
+ };
26
+ const textNote = llm?.pricing.note ??
27
+ "CLI subscription or model pricing is unavailable; usage may be billed by your account.";
28
+ if (draft.sources.research === "generate")
29
+ row("Research", textCost(promptChars + 6000, 12000), `Assumes about 2,000 words of notes; web search fees are excluded. ${textNote}`, true);
30
+ else
31
+ row("Research", 0, "Provided or off; no generation charge.");
32
+ if (generatedArticle)
33
+ row("Article", textCost(promptChars + (draft.sources.research === "generate" ? 12000 : 0), articleChars), `${expectedWords.toLocaleString()} expected words. ${textNote}`, true);
34
+ else
35
+ row("Article", 0, "Provided article; no generation charge.");
36
+ if (draft.intro?.mode === "llm" || draft.outro?.mode === "llm")
37
+ row("Intro / outro text", textCost(promptChars + articleChars, 2400), textNote, true);
38
+ if (draft.sources.audio === "generate") {
39
+ const extras = ["intro", "outro"].reduce((n, key) => n + (rendered[key]?.length ?? 0), 0);
40
+ const llmExtras = (draft.intro?.mode === "llm" ? 1200 : 0) + (draft.outro?.mode === "llm" ? 1200 : 0);
41
+ const chars = articleChars + extras + llmExtras;
42
+ const p = tts?.pricing;
43
+ const amount = p?.perMillionCharacters === undefined
44
+ ? p?.perMinute === undefined
45
+ ? undefined
46
+ : (chars / 6 / 150) * p.perMinute
47
+ : (chars / 1000000) * p.perMillionCharacters;
48
+ row("Narration", amount, `About ${chars.toLocaleString()} characters. ${p?.note ?? "Account pricing is unknown."}`, generatedArticle || llmExtras > 0 || p?.perMinute !== undefined);
49
+ }
50
+ else
51
+ row("Narration", 0, "Provided or off; no generation charge.");
52
+ const images = draft.sources.images === "generate" ? draft.imagePrompts.reduce((n, p) => n + p.number, 0) : 0;
53
+ row("Images", images === 0
54
+ ? 0
55
+ : image?.pricing.perImage === undefined
56
+ ? undefined
57
+ : images * image.pricing.perImage, images === 0
58
+ ? "Provided or off."
59
+ : `${images} images. ${image?.pricing.note ?? "Model or account pricing is unknown."}`);
60
+ const thumbnail = ["from_prompt", "prompt_by_llm"].includes(draft.sources.thumbnail);
61
+ row("Thumbnail", thumbnail ? image?.pricing.perImage : 0, thumbnail
62
+ ? (image?.pricing.note ?? "Model or account pricing is unknown.")
63
+ : "Provided or off.");
64
+ if (draft.sources.thumbnail === "prompt_by_llm")
65
+ row("Thumbnail prompt", textCost(promptChars + articleChars, 1200), textNote, true);
66
+ row("Export / subtitles", 0, "Local processing; no API fee.");
67
+ return {
68
+ currency: "USD",
69
+ rows,
70
+ expectedWords,
71
+ catalogueDate: catalogue?.status().updatedAt ?? null,
72
+ low: rows.reduce((n, r) => n + (r.low ?? 0), 0),
73
+ high: rows.reduce((n, r) => n + (r.high ?? 0), 0),
74
+ unknown: rows.filter((r) => r.low === null).length,
75
+ assumptions: [
76
+ "Planning estimate, not a spending limit. Generated lengths use a ±50% range; actual output can exceed it.",
77
+ "Assumes roughly 6 characters per word, 4 characters per token, and 150 spoken words per minute.",
78
+ "Retries, extra reasoning tokens, web tools, taxes, discounts and included credits are excluded. Unknown charges are not counted as zero.",
79
+ ],
80
+ };
81
+ }
@@ -0,0 +1,21 @@
1
+ export function observeNarration(store, projectId, key, label) {
2
+ if (store === undefined)
3
+ return undefined;
4
+ let sink;
5
+ return (event) => {
6
+ switch (event.type) {
7
+ case "start":
8
+ sink = store.begin(projectId, key, label);
9
+ break;
10
+ case "chunk":
11
+ sink?.append(event.bytes);
12
+ break;
13
+ case "complete":
14
+ sink?.complete();
15
+ break;
16
+ case "interrupted":
17
+ sink?.interrupt();
18
+ break;
19
+ }
20
+ };
21
+ }