@gentbajko/slopify 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/llm/codex.js +3 -0
- package/dist/adapters/llm/gemini-workspace.js +24 -1
- package/dist/adapters/llm/gemini.js +1 -1
- package/dist/adapters/llm/openrouter.js +1 -0
- package/dist/assets/models.yaml +470 -0
- package/dist/catalog/registry.js +147 -0
- package/dist/catalog/schema.js +91 -0
- package/dist/catalog/store.js +90 -0
- package/dist/catalog/validate.js +43 -0
- package/dist/edge/http/actions.js +1 -1
- package/dist/edge/http/app.js +4 -0
- package/dist/edge/http/open-folder.js +61 -0
- package/dist/edge/http/planning.js +101 -0
- package/dist/edge/http/projects.js +4 -2
- package/dist/edge/http/providers.js +28 -0
- package/dist/edge/open-folder.js +25 -0
- package/dist/kernel/db/migrations/0003-batch-queue.sql +11 -0
- package/dist/kernel/ports/llm.js +1 -0
- package/dist/kernel/ports/text.js +26 -0
- package/dist/kernel/runner/providers.js +8 -6
- package/dist/kernel/runner/queue.js +65 -0
- package/dist/main.js +43 -6
- package/dist/slices/admission/repo.js +13 -2
- package/dist/slices/admission/start.js +9 -8
- package/dist/slices/article/continuation.js +8 -1
- package/dist/slices/article/segments.js +1 -0
- package/dist/slices/batch/index.js +82 -0
- package/dist/slices/control/index.js +4 -1
- package/dist/slices/control/providers.js +18 -1
- package/dist/slices/estimate/index.js +81 -0
- package/dist/slices/narration/chunk.js +21 -22
- package/dist/slices/narration/plan.js +58 -0
- package/dist/slices/narration/run.js +6 -27
- package/dist/slices/research/run.js +4 -0
- package/dist/slices/storage/staging.js +2 -1
- package/dist/slices/thumbnail/run.js +3 -0
- package/dist/web/assets/index-DkC4WXhk.css +1 -0
- package/dist/web/assets/index-GszhBVNn.js +130 -0
- package/dist/web/index.html +2 -2
- package/package.json +2 -1
- package/dist/web/assets/index-6QIQ-TzO.css +0 -1
- package/dist/web/assets/index-CNgWL-ct.js +0 -130
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
// One app-wide queue: no more than five calls, with lower provider limits.
|
|
2
|
+
// Waiting does not start an attempt or its idle timer. FIFO among eligible calls.
|
|
3
|
+
export function createProviderQueue(limitFor) {
|
|
4
|
+
const active = new Map();
|
|
5
|
+
const waiting = [];
|
|
6
|
+
let total = 0;
|
|
7
|
+
function drain() {
|
|
8
|
+
while (total < 5) {
|
|
9
|
+
const at = waiting.findIndex((w) => (active.get(w.provider) ?? 0) <
|
|
10
|
+
Math.max(1, Math.min(5, Math.floor(limitFor(w.provider)) || 1)));
|
|
11
|
+
if (at < 0)
|
|
12
|
+
return;
|
|
13
|
+
const next = waiting.splice(at, 1)[0];
|
|
14
|
+
if (!next)
|
|
15
|
+
return;
|
|
16
|
+
next.signal.removeEventListener("abort", next.cancel);
|
|
17
|
+
if (next.signal.aborted) {
|
|
18
|
+
next.cancel();
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
total++;
|
|
22
|
+
active.set(next.provider, (active.get(next.provider) ?? 0) + 1);
|
|
23
|
+
next.start();
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return {
|
|
27
|
+
run: (provider, signal, work) => {
|
|
28
|
+
if (signal.aborted)
|
|
29
|
+
return Promise.reject(signal.reason);
|
|
30
|
+
return new Promise((resolve, reject) => {
|
|
31
|
+
const waiter = {
|
|
32
|
+
provider,
|
|
33
|
+
signal,
|
|
34
|
+
cancel: () => {
|
|
35
|
+
const at = waiting.indexOf(waiter);
|
|
36
|
+
if (at >= 0)
|
|
37
|
+
waiting.splice(at, 1);
|
|
38
|
+
signal.removeEventListener("abort", waiter.cancel);
|
|
39
|
+
reject(signal.reason);
|
|
40
|
+
},
|
|
41
|
+
start: () => {
|
|
42
|
+
void Promise.resolve()
|
|
43
|
+
.then(() => {
|
|
44
|
+
signal.throwIfAborted();
|
|
45
|
+
return work();
|
|
46
|
+
})
|
|
47
|
+
.then(resolve, reject)
|
|
48
|
+
.finally(() => {
|
|
49
|
+
total--;
|
|
50
|
+
const left = (active.get(provider) ?? 1) - 1;
|
|
51
|
+
if (left)
|
|
52
|
+
active.set(provider, left);
|
|
53
|
+
else
|
|
54
|
+
active.delete(provider);
|
|
55
|
+
drain();
|
|
56
|
+
});
|
|
57
|
+
},
|
|
58
|
+
};
|
|
59
|
+
waiting.push(waiter);
|
|
60
|
+
signal.addEventListener("abort", waiter.cancel, { once: true });
|
|
61
|
+
drain();
|
|
62
|
+
});
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
package/dist/main.js
CHANGED
|
@@ -7,8 +7,11 @@ import { buildRegistry } from "./adapter-registry.js";
|
|
|
7
7
|
import { alignSubtitles } from "./adapters/alignment/index.js";
|
|
8
8
|
import { prepareFfmpeg } from "./adapters/ffmpeg.js";
|
|
9
9
|
import { nodeRunCli } from "./adapters/llm/run-cli.js";
|
|
10
|
+
import { curateRegistry } from "./catalog/registry.js";
|
|
11
|
+
import { createCatalogueStore } from "./catalog/store.js";
|
|
10
12
|
import { createHub } from "./edge/events/hub.js";
|
|
11
13
|
import { createApp } from "./edge/http/app.js";
|
|
14
|
+
import { openFolder } from "./edge/open-folder.js";
|
|
12
15
|
import { createAudioPreviewStore } from "./kernel/audio-preview.js";
|
|
13
16
|
import { systemClock } from "./kernel/clock.js";
|
|
14
17
|
import { openDb } from "./kernel/db/index.js";
|
|
@@ -21,11 +24,13 @@ import { sqliteAttempts } from "./kernel/runner/attempt-repo.js";
|
|
|
21
24
|
import { dependenciesOf } from "./kernel/runner/graph.js";
|
|
22
25
|
import { createRunner } from "./kernel/runner/index.js";
|
|
23
26
|
import { stageProviders } from "./kernel/runner/providers.js";
|
|
27
|
+
import { createProviderQueue } from "./kernel/runner/queue.js";
|
|
24
28
|
import { readVersion } from "./kernel/version.js";
|
|
25
29
|
import { modelSources } from "./model-catalog.js";
|
|
26
30
|
import { claimStage, finishStage, projectById, projectPaused, stagesOf, } from "./slices/admission/repo.js";
|
|
27
31
|
import { prepareProvidedArticleSegments } from "./slices/article/provided-entries.js";
|
|
28
32
|
import { runArticle } from "./slices/article/run.js";
|
|
33
|
+
import { pumpQueue, queueWaiting } from "./slices/batch/index.js";
|
|
29
34
|
import { runImages } from "./slices/images/run.js";
|
|
30
35
|
import { runNarration } from "./slices/narration/run.js";
|
|
31
36
|
import { runResearch } from "./slices/research/run.js";
|
|
@@ -78,13 +83,14 @@ export async function boot(config) {
|
|
|
78
83
|
log,
|
|
79
84
|
post: httpPostEvents(collectorEndpoint(process.env), collectorTimeoutMs),
|
|
80
85
|
}, flushDelayMs);
|
|
81
|
-
const
|
|
86
|
+
const catalogue = createCatalogueStore({ dataDir: paths.dataDir, fetch: globalThis.fetch });
|
|
87
|
+
const registry = curateRegistry(buildRegistry({
|
|
82
88
|
db,
|
|
83
89
|
fetch: globalThis.fetch,
|
|
84
90
|
spawn: nodeRunCli,
|
|
85
91
|
clock,
|
|
86
92
|
probe: nodeCliProbe,
|
|
87
|
-
});
|
|
93
|
+
}), catalogue);
|
|
88
94
|
const audioPreviews = createAudioPreviewStore();
|
|
89
95
|
const runner = wire({
|
|
90
96
|
db,
|
|
@@ -96,6 +102,7 @@ export async function boot(config) {
|
|
|
96
102
|
telemetry,
|
|
97
103
|
flusher,
|
|
98
104
|
registry,
|
|
105
|
+
catalogue,
|
|
99
106
|
ffmpeg,
|
|
100
107
|
audioPreviews,
|
|
101
108
|
});
|
|
@@ -163,6 +170,7 @@ export async function boot(config) {
|
|
|
163
170
|
},
|
|
164
171
|
});
|
|
165
172
|
const app = createApp({
|
|
173
|
+
openFolder,
|
|
166
174
|
db,
|
|
167
175
|
paths,
|
|
168
176
|
hub,
|
|
@@ -170,6 +178,7 @@ export async function boot(config) {
|
|
|
170
178
|
updater,
|
|
171
179
|
audioPreviews,
|
|
172
180
|
...modelSources(registry),
|
|
181
|
+
catalogue,
|
|
173
182
|
clock,
|
|
174
183
|
ids,
|
|
175
184
|
log,
|
|
@@ -179,6 +188,20 @@ export async function boot(config) {
|
|
|
179
188
|
probe: nodeCliProbe,
|
|
180
189
|
});
|
|
181
190
|
const server = await listen(app, config, log);
|
|
191
|
+
const queueTimer = setInterval(() => {
|
|
192
|
+
const release = updater.beginMutation();
|
|
193
|
+
if (!release)
|
|
194
|
+
return;
|
|
195
|
+
try {
|
|
196
|
+
pumpQueue(updateDb, runner);
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
log.write("error", "batch.queue", { detail: "The batch queue could not advance." });
|
|
200
|
+
}
|
|
201
|
+
finally {
|
|
202
|
+
release();
|
|
203
|
+
}
|
|
204
|
+
}, 1000);
|
|
182
205
|
listeningPort = portOf(server) ?? config.port;
|
|
183
206
|
// Whatever last run left queued goes out at start. Nothing waits for
|
|
184
207
|
// it, and an unreachable collector costs one refused socket.
|
|
@@ -188,6 +211,7 @@ export async function boot(config) {
|
|
|
188
211
|
let stopActivation = () => { };
|
|
189
212
|
shutdown = () => {
|
|
190
213
|
stopping ??= (async () => {
|
|
214
|
+
clearInterval(queueTimer);
|
|
191
215
|
stopActivation();
|
|
192
216
|
audioPreviews.close();
|
|
193
217
|
try {
|
|
@@ -221,7 +245,7 @@ export async function boot(config) {
|
|
|
221
245
|
throw error;
|
|
222
246
|
}
|
|
223
247
|
}
|
|
224
|
-
function wire({ audioPreviews, db, paths, clock, ids, log, hub, telemetry, flusher, registry, ffmpeg, }) {
|
|
248
|
+
function wire({ audioPreviews, db, paths, clock, ids, log, hub, telemetry, flusher, registry, catalogue, ffmpeg, }) {
|
|
225
249
|
// A stage counts what it did and the queue is flushed after each new event. `record`
|
|
226
250
|
// swallows its own failures, so this can neither fail a stage nor widen what leaves the
|
|
227
251
|
// machine - the payload allow-list is checked inside it.
|
|
@@ -237,11 +261,17 @@ function wire({ audioPreviews, db, paths, clock, ids, log, hub, telemetry, flush
|
|
|
237
261
|
const writing = { db, paths, ids, clock, log, count };
|
|
238
262
|
// A stage slice is handed the wrapped calls, never the registry: every provider call
|
|
239
263
|
// it makes is already inside the retry policy (kernel/runner/providers.ts).
|
|
240
|
-
const providers = {
|
|
264
|
+
const providers = {
|
|
265
|
+
registry,
|
|
266
|
+
attempts: sqliteAttempts(db, ids),
|
|
267
|
+
clock,
|
|
268
|
+
log,
|
|
269
|
+
queue: createProviderQueue((provider) => catalogue.read().providers[provider]?.maxConcurrent ?? 1),
|
|
270
|
+
};
|
|
241
271
|
return createRunner({
|
|
242
272
|
stages: {
|
|
243
273
|
stagesOf: (projectId) => stagesOf(db, projectId),
|
|
244
|
-
paused: (projectId) => projectPaused(db, projectId),
|
|
274
|
+
paused: (projectId) => projectPaused(db, projectId) || queueWaiting(db, projectId),
|
|
245
275
|
dependenciesOf: (projectId, kind) => {
|
|
246
276
|
const project = projectById(db, projectId);
|
|
247
277
|
return project === undefined ? [] : dependenciesOf(kind, project.config.sources);
|
|
@@ -257,7 +287,14 @@ function wire({ audioPreviews, db, paths, clock, ids, log, hub, telemetry, flush
|
|
|
257
287
|
audio: async (context) => {
|
|
258
288
|
const wrapped = stageProviders(providers, context);
|
|
259
289
|
await prepareProvidedArticleSegments(writing, context, wrapped);
|
|
260
|
-
await runNarration({
|
|
290
|
+
await runNarration({
|
|
291
|
+
...video,
|
|
292
|
+
audioPreviews,
|
|
293
|
+
maxCharacters: (provider, model) => {
|
|
294
|
+
const entry = catalogue.models(provider, "tts").find((m) => m.id === model);
|
|
295
|
+
return entry && "tts" in entry ? entry.tts.maxCharacters : 4000;
|
|
296
|
+
},
|
|
297
|
+
}, context, wrapped);
|
|
261
298
|
},
|
|
262
299
|
images: (context) => runImages(writing, context, stageProviders(providers, context)),
|
|
263
300
|
thumbnail: (context) => runThumbnail(writing, context, stageProviders(providers, context)),
|
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { stageKinds, stageStates } from "../../kernel/pipeline.js";
|
|
3
|
+
import { thinkingModes } from "../../kernel/ports/llm.js";
|
|
3
4
|
import { chunkModes } from "../narration/chunk.js";
|
|
4
5
|
import { subtitleConfigSchema } from "../subtitles/model.js";
|
|
5
6
|
import { entryModes, formats, stageSources } from "./model.js";
|
|
6
|
-
const providerChoice = z.object({
|
|
7
|
+
const providerChoice = z.object({
|
|
8
|
+
provider: z.string(),
|
|
9
|
+
model: z.string(),
|
|
10
|
+
thinking: z.enum(thinkingModes).optional(),
|
|
11
|
+
});
|
|
7
12
|
const entryChoice = z.object({ name: z.string(), mode: z.enum(entryModes) });
|
|
8
13
|
// The shape Play posts and the shape `projects.config` holds, in one place: the second
|
|
9
14
|
// is the first plus the rendered prompt texts.
|
|
@@ -38,7 +43,13 @@ export const runDraftSchema = z.object({
|
|
|
38
43
|
}),
|
|
39
44
|
// Optional until Play carries the control; unknown keys are stripped by this schema, so
|
|
40
45
|
// a mode not listed here would never reach the audio stage.
|
|
41
|
-
chunking: z
|
|
46
|
+
chunking: z
|
|
47
|
+
.object({
|
|
48
|
+
mode: z.enum(chunkModes),
|
|
49
|
+
words: z.number().optional(),
|
|
50
|
+
characters: z.number().int().min(1).max(1000000).optional(),
|
|
51
|
+
})
|
|
52
|
+
.optional(),
|
|
42
53
|
silenceGapSeconds: z.number(),
|
|
43
54
|
subtitles: subtitleConfigSchema.optional(),
|
|
44
55
|
});
|
|
@@ -13,7 +13,7 @@ export function initialState(source) {
|
|
|
13
13
|
}
|
|
14
14
|
// The project row, its six stages and the provided content all land together or not at
|
|
15
15
|
// all. The caller ticks the runner after this returns, never inside it.
|
|
16
|
-
export function startRun(deps, draft, rendered) {
|
|
16
|
+
export function startRun(deps, draft, rendered, retainStaged = false) {
|
|
17
17
|
const id = deps.ids.next();
|
|
18
18
|
const at = deps.clock.now().toISOString();
|
|
19
19
|
const config = { ...draft, rendered };
|
|
@@ -49,14 +49,14 @@ export function startRun(deps, draft, rendered) {
|
|
|
49
49
|
for (const stage of stages) {
|
|
50
50
|
insertStage(deps.db, stage);
|
|
51
51
|
}
|
|
52
|
-
attachProvided(deps, id, draft, moved);
|
|
52
|
+
attachProvided(deps, id, draft, moved, retainStaged);
|
|
53
53
|
});
|
|
54
|
-
for (const source of moved) {
|
|
54
|
+
for (const source of retainStaged ? [] : moved) {
|
|
55
55
|
dropStagedSource(deps, source);
|
|
56
56
|
}
|
|
57
57
|
return { project, stages };
|
|
58
58
|
}
|
|
59
|
-
function attachProvided(deps, projectId, draft, collected) {
|
|
59
|
+
function attachProvided(deps, projectId, draft, collected, retainStaged) {
|
|
60
60
|
const { sources, provided } = draft;
|
|
61
61
|
if (sources.research === "provide" && provided.research !== undefined) {
|
|
62
62
|
storeText(deps, {
|
|
@@ -73,24 +73,25 @@ function attachProvided(deps, projectId, draft, collected) {
|
|
|
73
73
|
storeArticleText(deps, { projectId, markdown: provided.article.trim() });
|
|
74
74
|
}
|
|
75
75
|
if (sources.audio === "provide") {
|
|
76
|
-
attach(deps, projectId, "audio", provided.audio, "audio_body", collected);
|
|
76
|
+
attach(deps, projectId, "audio", provided.audio, "audio_body", collected, retainStaged);
|
|
77
77
|
}
|
|
78
78
|
if (sources.thumbnail === "provide") {
|
|
79
|
-
attach(deps, projectId, "thumbnail", provided.thumbnail, "thumbnail", collected);
|
|
79
|
+
attach(deps, projectId, "thumbnail", provided.thumbnail, "thumbnail", collected, retainStaged);
|
|
80
80
|
}
|
|
81
81
|
if (sources.images === "provide") {
|
|
82
82
|
// Slideshow order is the order the user left the list in.
|
|
83
83
|
for (const [index, stagedFileId] of (provided.images ?? []).entries()) {
|
|
84
|
-
attach(deps, projectId, "images", stagedFileId, "image", collected, index + 1);
|
|
84
|
+
attach(deps, projectId, "images", stagedFileId, "image", collected, retainStaged, index + 1);
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
87
|
}
|
|
88
|
-
function attach(deps, projectId, kind, stagedFileId, role, collected, index) {
|
|
88
|
+
function attach(deps, projectId, kind, stagedFileId, role, collected, retainStaged, index) {
|
|
89
89
|
if (stagedFileId === undefined) {
|
|
90
90
|
throw new Error(`the ${kind} stage is provided but carries no staged file`);
|
|
91
91
|
}
|
|
92
92
|
const result = attachStagedFile(deps, {
|
|
93
93
|
stagedFileId,
|
|
94
|
+
retainStaged,
|
|
94
95
|
projectId,
|
|
95
96
|
role,
|
|
96
97
|
...(index === undefined ? {} : { index }),
|
|
@@ -43,7 +43,13 @@ export async function writeArticle(providers, choice, brief, onDelta) {
|
|
|
43
43
|
onDelta(event.text);
|
|
44
44
|
}
|
|
45
45
|
};
|
|
46
|
-
let answer = await providers.llm({
|
|
46
|
+
let answer = await providers.llm({
|
|
47
|
+
provider: choice.provider,
|
|
48
|
+
model: choice.model,
|
|
49
|
+
...(choice.thinking === undefined ? {} : { thinking: choice.thinking }),
|
|
50
|
+
messages: base,
|
|
51
|
+
check: written,
|
|
52
|
+
}, stream);
|
|
47
53
|
const pieces = [answer.text];
|
|
48
54
|
let tokens = plusUsage(noTokens, answer.usage);
|
|
49
55
|
for (let n = 1; truncated(answer) && n <= continuationLimit; n += 1) {
|
|
@@ -52,6 +58,7 @@ export async function writeArticle(providers, choice, brief, onDelta) {
|
|
|
52
58
|
answer = await providers.llm({
|
|
53
59
|
provider: choice.provider,
|
|
54
60
|
model: choice.model,
|
|
61
|
+
...(choice.thinking === undefined ? {} : { thinking: choice.thinking }),
|
|
55
62
|
messages,
|
|
56
63
|
// The last continuation allowed is the one that has to end the article: a fourth
|
|
57
64
|
// truncation is a failed attempt, which is the wrapper's to retry. The
|
|
@@ -18,6 +18,7 @@ export async function writeSegment(providers, choice, config, category, article,
|
|
|
18
18
|
const answer = await providers.llm({
|
|
19
19
|
provider: choice.provider,
|
|
20
20
|
model: choice.model,
|
|
21
|
+
...(choice.thinking === undefined ? {} : { thinking: choice.thinking }),
|
|
21
22
|
messages,
|
|
22
23
|
check: (given) => given.text.trim() === "" ? `the ${category} answered with nothing` : undefined,
|
|
23
24
|
});
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { transact } from "../../kernel/db/tx.js";
|
|
3
|
+
import { derive } from "../../kernel/runner/graph.js";
|
|
4
|
+
import { projectPaused, stagesOf } from "../admission/repo.js";
|
|
5
|
+
import { startRun } from "../admission/start.js";
|
|
6
|
+
import { stagingPath } from "../storage/layout.js";
|
|
7
|
+
import { deleteStagedFile, stagedFileById } from "../storage/repo.js";
|
|
8
|
+
import { dropStagedSource } from "../storage/staging.js";
|
|
9
|
+
const queueRow = z.object({
|
|
10
|
+
projectId: z.string(),
|
|
11
|
+
batchId: z.string(),
|
|
12
|
+
position: z.number(),
|
|
13
|
+
state: z.enum(["queued", "active", "finished"]),
|
|
14
|
+
});
|
|
15
|
+
export function queueEntries(db, batchId) {
|
|
16
|
+
return db
|
|
17
|
+
.prepare(`SELECT project_id AS projectId, batch_id AS batchId, position, state
|
|
18
|
+
FROM project_queue ${batchId === undefined ? "WHERE state != 'finished'" : "WHERE batch_id = ?"}
|
|
19
|
+
ORDER BY position`)
|
|
20
|
+
.all(...(batchId === undefined ? [] : [batchId]))
|
|
21
|
+
.map((row) => queueRow.parse(row));
|
|
22
|
+
}
|
|
23
|
+
export function queueWaiting(db, projectId) {
|
|
24
|
+
return (db
|
|
25
|
+
.prepare("SELECT 1 FROM project_queue WHERE project_id = ? AND state = 'queued'")
|
|
26
|
+
.get(projectId) !== undefined);
|
|
27
|
+
}
|
|
28
|
+
export function batchExists(db, id) {
|
|
29
|
+
return db.prepare("SELECT 1 FROM batches WHERE id = ?").get(id) !== undefined;
|
|
30
|
+
}
|
|
31
|
+
export function enqueueBatch(deps, batchId, runs) {
|
|
32
|
+
if (batchExists(deps.db, batchId))
|
|
33
|
+
return queueEntries(deps.db, batchId);
|
|
34
|
+
const sources = new Set();
|
|
35
|
+
transact(deps.db, () => {
|
|
36
|
+
deps.db
|
|
37
|
+
.prepare("INSERT INTO batches(id, created_at) VALUES (?, ?)")
|
|
38
|
+
.run(batchId, deps.clock.now().toISOString());
|
|
39
|
+
const used = new Set();
|
|
40
|
+
for (const { draft, rendered } of runs) {
|
|
41
|
+
const { project } = startRun(deps, draft, rendered, true);
|
|
42
|
+
deps.db
|
|
43
|
+
.prepare("INSERT INTO project_queue(project_id, batch_id) VALUES (?, ?)")
|
|
44
|
+
.run(project.id, batchId);
|
|
45
|
+
if (draft.sources.audio === "provide" && draft.provided.audio)
|
|
46
|
+
used.add(draft.provided.audio);
|
|
47
|
+
if (draft.sources.thumbnail === "provide" && draft.provided.thumbnail)
|
|
48
|
+
used.add(draft.provided.thumbnail);
|
|
49
|
+
if (draft.sources.images === "provide")
|
|
50
|
+
for (const id of draft.provided.images ?? [])
|
|
51
|
+
used.add(id);
|
|
52
|
+
}
|
|
53
|
+
for (const id of used) {
|
|
54
|
+
const file = stagedFileById(deps.db, id);
|
|
55
|
+
if (file)
|
|
56
|
+
sources.add(stagingPath(deps.paths, file.path));
|
|
57
|
+
deleteStagedFile(deps.db, id);
|
|
58
|
+
}
|
|
59
|
+
});
|
|
60
|
+
for (const source of sources)
|
|
61
|
+
dropStagedSource(deps, source);
|
|
62
|
+
return queueEntries(deps.db, batchId);
|
|
63
|
+
}
|
|
64
|
+
// Only one batch video runs at once. A paused item holds its place; a failed or
|
|
65
|
+
// canceled item releases the next video once its provider calls have drained.
|
|
66
|
+
export function pumpQueue(db, runner) {
|
|
67
|
+
for (const entry of queueEntries(db)) {
|
|
68
|
+
const stages = stagesOf(db, entry.projectId);
|
|
69
|
+
const status = derive(stages, projectPaused(db, entry.projectId));
|
|
70
|
+
if (runner.hasInflight?.(entry.projectId))
|
|
71
|
+
return;
|
|
72
|
+
if (["done", "failed", "canceled"].includes(status)) {
|
|
73
|
+
db.prepare("UPDATE project_queue SET state = 'finished' WHERE project_id = ?").run(entry.projectId);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (status === "paused")
|
|
77
|
+
return;
|
|
78
|
+
db.prepare("UPDATE project_queue SET state = 'active' WHERE project_id = ?").run(entry.projectId);
|
|
79
|
+
runner.tick(entry.projectId);
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { transact } from "../../kernel/db/tx.js";
|
|
2
2
|
import { derive, satisfied } from "../../kernel/runner/graph.js";
|
|
3
3
|
import { finishStage, projectById, resetStage, setProjectPaused, stagesOf, updateProjectConfig, } from "../admission/repo.js";
|
|
4
|
+
import { sameChunking } from "../narration/chunk.js";
|
|
4
5
|
import { clearUnfinishedAudio } from "../reruns/index.js";
|
|
5
6
|
import { providers as providerCatalog } from "../settings/model.js";
|
|
6
7
|
import { hasKey, listVoices } from "../settings/repo.js";
|
|
@@ -137,13 +138,15 @@ export function changeProviders(deps, id, changes) {
|
|
|
137
138
|
(changes.audio.provider !== project.config.audio?.provider ||
|
|
138
139
|
changes.audio.model !== project.config.audio?.model ||
|
|
139
140
|
changes.audio.voice !== project.config.audio?.voice);
|
|
141
|
+
const chunkingChanged = changes.chunking !== undefined && !sameChunking(changes.chunking, project.config.chunking);
|
|
140
142
|
const orphaned = transact(deps.db, () => {
|
|
141
|
-
const files = audioChanged ? clearUnfinishedAudio(deps, id) : [];
|
|
143
|
+
const files = audioChanged || chunkingChanged ? clearUnfinishedAudio(deps, id) : [];
|
|
142
144
|
updateProjectConfig(deps.db, id, {
|
|
143
145
|
...project.config,
|
|
144
146
|
...(changes.llm === undefined ? {} : { llm: changes.llm }),
|
|
145
147
|
...(changes.audio === undefined ? {} : { audio: changes.audio }),
|
|
146
148
|
...(changes.images === undefined ? {} : { images: changes.images }),
|
|
149
|
+
...(changes.chunking === undefined ? {} : { chunking: changes.chunking }),
|
|
147
150
|
}, deps.clock.now().toISOString());
|
|
148
151
|
return files;
|
|
149
152
|
});
|
|
@@ -1,8 +1,11 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { thinkingModes } from "../../kernel/ports/llm.js";
|
|
3
|
+
import { chunkModes } from "../narration/chunk.js";
|
|
2
4
|
const choice = z
|
|
3
5
|
.object({
|
|
4
6
|
provider: z.string().trim().min(1).max(200),
|
|
5
7
|
model: z.string().trim().min(1).max(200),
|
|
8
|
+
thinking: z.enum(thinkingModes).optional(),
|
|
6
9
|
})
|
|
7
10
|
.strict();
|
|
8
11
|
export const providerChangesSchema = z
|
|
@@ -10,6 +13,14 @@ export const providerChangesSchema = z
|
|
|
10
13
|
llm: choice.optional(),
|
|
11
14
|
audio: choice.extend({ voice: z.string().trim().min(1).max(200) }).optional(),
|
|
12
15
|
images: choice.optional(),
|
|
16
|
+
chunking: z
|
|
17
|
+
.object({
|
|
18
|
+
mode: z.enum(chunkModes),
|
|
19
|
+
words: z.number().int().min(1).max(10000).optional(),
|
|
20
|
+
characters: z.number().int().min(1).max(1000000).optional(),
|
|
21
|
+
})
|
|
22
|
+
.strict()
|
|
23
|
+
.optional(),
|
|
13
24
|
})
|
|
14
25
|
.strict()
|
|
15
26
|
.refine((value) => Object.keys(value).length > 0, "Pick at least one provider to update.");
|
|
@@ -53,9 +64,15 @@ export async function validateProviderChanges(changes, current) {
|
|
|
53
64
|
if (current.allowsCustomModels?.(picked.provider) === true)
|
|
54
65
|
continue;
|
|
55
66
|
const models = await current.modelsFor(picked.provider, families[key]);
|
|
56
|
-
|
|
67
|
+
const model = models.find((model) => model.id === picked.model);
|
|
68
|
+
if (!model) {
|
|
57
69
|
fields.push({ field: `${key}.model`, message: "Choose a model supported by this provider." });
|
|
58
70
|
}
|
|
71
|
+
else if (key === "llm" &&
|
|
72
|
+
picked.thinking &&
|
|
73
|
+
!model.thinkingModes?.includes(picked.thinking)) {
|
|
74
|
+
fields.push({ field: `${key}.thinking`, message: "Choose a supported thinking setting." });
|
|
75
|
+
}
|
|
59
76
|
}
|
|
60
77
|
return fields;
|
|
61
78
|
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
export function estimateRun(draft, rendered, expectedWords, catalogue) {
|
|
2
|
+
const rows = [];
|
|
3
|
+
const model = (family, choice) => choice
|
|
4
|
+
? catalogue?.models(choice.provider, family).find((m) => m.id === choice.model)
|
|
5
|
+
: undefined;
|
|
6
|
+
const llm = model("llm", draft.llm);
|
|
7
|
+
const tts = model("tts", draft.audio);
|
|
8
|
+
const image = model("image", draft.images);
|
|
9
|
+
const generatedArticle = draft.sources.article === "generate";
|
|
10
|
+
const articleChars = generatedArticle ? expectedWords * 6 : (draft.provided.article?.length ?? 0);
|
|
11
|
+
const promptChars = Object.values(rendered).reduce((sum, value) => sum + value.length, 0);
|
|
12
|
+
const row = (stage, amount, detail, variable = false) => {
|
|
13
|
+
rows.push({
|
|
14
|
+
stage,
|
|
15
|
+
low: amount === undefined ? null : amount * (variable ? 0.5 : 1),
|
|
16
|
+
high: amount === undefined ? null : amount * (variable ? 1.5 : 1),
|
|
17
|
+
detail,
|
|
18
|
+
});
|
|
19
|
+
};
|
|
20
|
+
const textCost = (input, output) => {
|
|
21
|
+
const p = llm?.pricing;
|
|
22
|
+
return p?.inputPerMillionTokens === undefined || p.outputPerMillionTokens === undefined
|
|
23
|
+
? undefined
|
|
24
|
+
: ((input / 4) * p.inputPerMillionTokens + (output / 4) * p.outputPerMillionTokens) / 1000000;
|
|
25
|
+
};
|
|
26
|
+
const textNote = llm?.pricing.note ??
|
|
27
|
+
"CLI subscription or model pricing is unavailable; usage may be billed by your account.";
|
|
28
|
+
if (draft.sources.research === "generate")
|
|
29
|
+
row("Research", textCost(promptChars + 6000, 12000), `Assumes about 2,000 words of notes; web search fees are excluded. ${textNote}`, true);
|
|
30
|
+
else
|
|
31
|
+
row("Research", 0, "Provided or off; no generation charge.");
|
|
32
|
+
if (generatedArticle)
|
|
33
|
+
row("Article", textCost(promptChars + (draft.sources.research === "generate" ? 12000 : 0), articleChars), `${expectedWords.toLocaleString()} expected words. ${textNote}`, true);
|
|
34
|
+
else
|
|
35
|
+
row("Article", 0, "Provided article; no generation charge.");
|
|
36
|
+
if (draft.intro?.mode === "llm" || draft.outro?.mode === "llm")
|
|
37
|
+
row("Intro / outro text", textCost(promptChars + articleChars, 2400), textNote, true);
|
|
38
|
+
if (draft.sources.audio === "generate") {
|
|
39
|
+
const extras = ["intro", "outro"].reduce((n, key) => n + (rendered[key]?.length ?? 0), 0);
|
|
40
|
+
const llmExtras = (draft.intro?.mode === "llm" ? 1200 : 0) + (draft.outro?.mode === "llm" ? 1200 : 0);
|
|
41
|
+
const chars = articleChars + extras + llmExtras;
|
|
42
|
+
const p = tts?.pricing;
|
|
43
|
+
const amount = p?.perMillionCharacters === undefined
|
|
44
|
+
? p?.perMinute === undefined
|
|
45
|
+
? undefined
|
|
46
|
+
: (chars / 6 / 150) * p.perMinute
|
|
47
|
+
: (chars / 1000000) * p.perMillionCharacters;
|
|
48
|
+
row("Narration", amount, `About ${chars.toLocaleString()} characters. ${p?.note ?? "Account pricing is unknown."}`, generatedArticle || llmExtras > 0 || p?.perMinute !== undefined);
|
|
49
|
+
}
|
|
50
|
+
else
|
|
51
|
+
row("Narration", 0, "Provided or off; no generation charge.");
|
|
52
|
+
const images = draft.sources.images === "generate" ? draft.imagePrompts.reduce((n, p) => n + p.number, 0) : 0;
|
|
53
|
+
row("Images", images === 0
|
|
54
|
+
? 0
|
|
55
|
+
: image?.pricing.perImage === undefined
|
|
56
|
+
? undefined
|
|
57
|
+
: images * image.pricing.perImage, images === 0
|
|
58
|
+
? "Provided or off."
|
|
59
|
+
: `${images} images. ${image?.pricing.note ?? "Model or account pricing is unknown."}`);
|
|
60
|
+
const thumbnail = ["from_prompt", "prompt_by_llm"].includes(draft.sources.thumbnail);
|
|
61
|
+
row("Thumbnail", thumbnail ? image?.pricing.perImage : 0, thumbnail
|
|
62
|
+
? (image?.pricing.note ?? "Model or account pricing is unknown.")
|
|
63
|
+
: "Provided or off.");
|
|
64
|
+
if (draft.sources.thumbnail === "prompt_by_llm")
|
|
65
|
+
row("Thumbnail prompt", textCost(promptChars + articleChars, 1200), textNote, true);
|
|
66
|
+
row("Export / subtitles", 0, "Local processing; no API fee.");
|
|
67
|
+
return {
|
|
68
|
+
currency: "USD",
|
|
69
|
+
rows,
|
|
70
|
+
expectedWords,
|
|
71
|
+
catalogueDate: catalogue?.status().updatedAt ?? null,
|
|
72
|
+
low: rows.reduce((n, r) => n + (r.low ?? 0), 0),
|
|
73
|
+
high: rows.reduce((n, r) => n + (r.high ?? 0), 0),
|
|
74
|
+
unknown: rows.filter((r) => r.low === null).length,
|
|
75
|
+
assumptions: [
|
|
76
|
+
"Planning estimate, not a spending limit. Generated lengths use a ±50% range; actual output can exceed it.",
|
|
77
|
+
"Assumes roughly 6 characters per word, 4 characters per token, and 150 spoken words per minute.",
|
|
78
|
+
"Retries, extra reasoning tokens, web tools, taxes, discounts and included credits are excluded. Unknown charges are not counted as zero.",
|
|
79
|
+
],
|
|
80
|
+
};
|
|
81
|
+
}
|
|
@@ -1,12 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
// paragraph, and every ~N words is consecutive chunks each ending at the last sentence
|
|
3
|
-
// boundary at or before N words, N defaulting to 500.
|
|
4
|
-
//
|
|
5
|
-
// Pure, and the only place the rule lives. Chunking sits in the functional core beside
|
|
6
|
-
// substitution and the render plan, so it is testable with no I/O and `run.ts` does nothing
|
|
7
|
-
// but call it.
|
|
8
|
-
export const chunkModes = ["whole", "paragraph", "words"];
|
|
1
|
+
export const chunkModes = ["whole", "paragraph", "words", "characters"];
|
|
9
2
|
export const defaultChunkWords = 500;
|
|
3
|
+
export const defaultChunkCharacters = 3000;
|
|
10
4
|
// A run created before Play carried the control sends the whole text as one request,
|
|
11
5
|
// which is the first case and the one that adds nothing the user did not ask for.
|
|
12
6
|
export const defaultChunking = { mode: "whole" };
|
|
@@ -20,7 +14,9 @@ export function chunkNarration(text, chunking) {
|
|
|
20
14
|
case "paragraph":
|
|
21
15
|
return nonEmpty(text.split(paragraphBreak));
|
|
22
16
|
case "words":
|
|
23
|
-
return
|
|
17
|
+
return sentenceRuns(text, chunking.words ?? defaultChunkWords, wordsIn);
|
|
18
|
+
case "characters":
|
|
19
|
+
return sentenceRuns(text, chunking.characters ?? defaultChunkCharacters, (part) => Array.from(part.trim()).length);
|
|
24
20
|
}
|
|
25
21
|
}
|
|
26
22
|
// What "N words" counts: runs of non-space. Exported because it is half of the rule -
|
|
@@ -28,29 +24,32 @@ export function chunkNarration(text, chunking) {
|
|
|
28
24
|
export function wordsIn(text) {
|
|
29
25
|
return text.split(/\s+/).filter((word) => word !== "").length;
|
|
30
26
|
}
|
|
31
|
-
function
|
|
32
|
-
|
|
33
|
-
|
|
27
|
+
export function sameChunking(left, right) {
|
|
28
|
+
const a = left ?? defaultChunking;
|
|
29
|
+
const b = right ?? defaultChunking;
|
|
30
|
+
if (a.mode !== b.mode)
|
|
31
|
+
return false;
|
|
32
|
+
if (a.mode === "words")
|
|
33
|
+
return (a.words ?? defaultChunkWords) === (b.words ?? defaultChunkWords);
|
|
34
|
+
if (a.mode === "characters")
|
|
35
|
+
return (a.characters ?? defaultChunkCharacters) === (b.characters ?? defaultChunkCharacters);
|
|
36
|
+
return true;
|
|
37
|
+
}
|
|
38
|
+
function sentenceRuns(text, budget, measure) {
|
|
34
39
|
const limit = Math.max(1, Math.floor(budget));
|
|
35
40
|
const chunks = [];
|
|
36
41
|
let current = "";
|
|
37
|
-
let count = 0;
|
|
38
42
|
for (const sentence of sentences(text)) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
// past the budget starts the next chunk instead of being split.
|
|
42
|
-
if (count > 0 && count + words > limit) {
|
|
43
|
+
// Measure the joined text so spaces between sentences count toward a character budget.
|
|
44
|
+
if (current.trim() && measure(current + sentence) > limit) {
|
|
43
45
|
chunks.push(current);
|
|
44
46
|
current = "";
|
|
45
|
-
count = 0;
|
|
46
47
|
}
|
|
47
48
|
current += sentence;
|
|
48
|
-
count += words;
|
|
49
49
|
}
|
|
50
50
|
chunks.push(current);
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
// limit is what refuses it, as an error on the stage.
|
|
51
|
+
// Like word chunking, a sentence longer than the budget stays whole on its own.
|
|
52
|
+
// The provider-specific planning layer applies any hard request limit afterwards.
|
|
54
53
|
return nonEmpty(chunks);
|
|
55
54
|
}
|
|
56
55
|
// ceiling: segmented as English. `Intl.Segmenter` is the platform's own sentence breaker
|