@gentbajko/slopify 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapter-registry.js +11 -12
- package/dist/adapters/image/bytes.js +1 -1
- package/dist/adapters/image/fal.js +28 -41
- package/dist/adapters/image/openai.js +19 -24
- package/dist/adapters/image/replicate.js +33 -43
- package/dist/adapters/llm/claude-code.js +16 -19
- package/dist/adapters/llm/codex.js +14 -15
- package/dist/adapters/llm/openrouter.js +13 -16
- package/dist/adapters/llm/run-cli.js +9 -11
- package/dist/adapters/llm/sse-lines.js +1 -2
- package/dist/adapters/retry-after.js +6 -6
- package/dist/adapters/tts/cartesia.js +5 -5
- package/dist/adapters/tts/elevenlabs.js +14 -19
- package/dist/adapters/tts/openai.js +9 -10
- package/dist/edge/cli.js +0 -0
- package/dist/edge/http/actions.js +8 -9
- package/dist/edge/http/app.js +3 -3
- package/dist/edge/http/entries.js +1 -1
- package/dist/edge/http/projects.js +7 -7
- package/dist/edge/http/prompts.js +6 -6
- package/dist/edge/http/providers.js +4 -4
- package/dist/edge/http/settings.js +2 -2
- package/dist/edge/http/staging.js +1 -1
- package/dist/edge/http/telemetry.js +6 -7
- package/dist/edge/http/usage.js +5 -5
- package/dist/kernel/config/index.js +1 -1
- package/dist/kernel/db/tx.js +9 -12
- package/dist/kernel/log.js +5 -5
- package/dist/kernel/pipeline.js +7 -9
- package/dist/kernel/ports/model.js +3 -4
- package/dist/kernel/runner/attempt-repo.js +5 -6
- package/dist/kernel/runner/attempt.js +16 -23
- package/dist/kernel/runner/graph.js +18 -29
- package/dist/kernel/runner/index.js +27 -33
- package/dist/kernel/runner/piece-repo.js +9 -11
- package/dist/kernel/runner/providers.js +5 -7
- package/dist/kernel/version.js +3 -3
- package/dist/main.js +10 -11
- package/dist/slices/admission/model.js +2 -2
- package/dist/slices/admission/repo.js +6 -7
- package/dist/slices/admission/rules.js +12 -15
- package/dist/slices/admission/start.js +7 -8
- package/dist/slices/admission/substitute.js +11 -11
- package/dist/slices/article/continuation.js +9 -9
- package/dist/slices/article/plain.js +5 -5
- package/dist/slices/article/run.js +25 -25
- package/dist/slices/article/split.js +1 -1
- package/dist/slices/article/store.js +5 -5
- package/dist/slices/cancel/index.js +8 -8
- package/dist/slices/images/run.js +32 -34
- package/dist/slices/library/lint.js +3 -3
- package/dist/slices/library/model.js +7 -7
- package/dist/slices/library/repo.js +3 -3
- package/dist/slices/library/save.js +7 -7
- package/dist/slices/library/slots.js +5 -5
- package/dist/slices/narration/chunk.js +12 -12
- package/dist/slices/narration/concat.js +12 -13
- package/dist/slices/narration/run.js +46 -49
- package/dist/slices/reruns/cascade.js +10 -11
- package/dist/slices/reruns/index.js +36 -37
- package/dist/slices/research/planner.js +4 -5
- package/dist/slices/research/run.js +22 -23
- package/dist/slices/research/synthesis.js +5 -5
- package/dist/slices/settings/cli-status.js +9 -10
- package/dist/slices/settings/keys.js +8 -9
- package/dist/slices/settings/model.js +5 -5
- package/dist/slices/settings/readiness.js +3 -4
- package/dist/slices/settings/repo.js +1 -1
- package/dist/slices/settings/voices.js +3 -4
- package/dist/slices/storage/asset-name.js +2 -2
- package/dist/slices/storage/delete-project.js +3 -5
- package/dist/slices/storage/downloads.js +4 -4
- package/dist/slices/storage/layout.js +5 -6
- package/dist/slices/storage/model.js +1 -1
- package/dist/slices/storage/reconcile.js +6 -7
- package/dist/slices/storage/repo.js +1 -1
- package/dist/slices/storage/staging.js +12 -15
- package/dist/slices/telemetry/collector-client.js +2 -2
- package/dist/slices/telemetry/flush.js +8 -8
- package/dist/slices/telemetry/machine.js +6 -7
- package/dist/slices/telemetry/model.js +11 -13
- package/dist/slices/telemetry/record.js +7 -7
- package/dist/slices/telemetry/repo.js +4 -4
- package/dist/slices/telemetry/usage.js +5 -5
- package/dist/slices/thumbnail/by-llm.js +2 -2
- package/dist/slices/thumbnail/run.js +30 -32
- package/dist/slices/video/ffmpeg.js +6 -6
- package/dist/slices/video/plan.js +9 -9
- package/dist/slices/video/run.js +19 -20
- package/dist/web/assets/index-77hb2Mr0.js +81 -0
- package/dist/web/index.html +1 -1
- package/package.json +1 -1
- package/dist/web/assets/index-D_sWbKQi.js +0 -81
package/dist/main.js
CHANGED
|
@@ -81,7 +81,7 @@ export async function boot(config) {
|
|
|
81
81
|
probe: nodeCliProbe,
|
|
82
82
|
});
|
|
83
83
|
const server = await listen(app, config, log);
|
|
84
|
-
//
|
|
84
|
+
// Whatever last run left queued goes out at start. Nothing waits for
|
|
85
85
|
// it, and an unreachable collector costs one refused socket.
|
|
86
86
|
flusher.soon();
|
|
87
87
|
const open = db;
|
|
@@ -101,7 +101,7 @@ export async function boot(config) {
|
|
|
101
101
|
// The pending timer is cancelled rather than awaited: a shutdown must not wait
|
|
102
102
|
// on the collector. A flush already in flight may land after the database
|
|
103
103
|
// closes and fail to mark its batch delivered, which costs one re-send that
|
|
104
|
-
// the collector deduplicates by event id
|
|
104
|
+
// the collector deduplicates by event id.
|
|
105
105
|
flusher.stop();
|
|
106
106
|
log.write("info", "shutdown");
|
|
107
107
|
open.close();
|
|
@@ -120,9 +120,9 @@ function wire({ db, paths, clock, ids, log, hub, telemetry, flusher, registry })
|
|
|
120
120
|
// Resolved once at boot rather than per render, so a machine with no usable binary
|
|
121
121
|
// fails at start with one message instead of on every project's last stage.
|
|
122
122
|
const ffmpeg = resolveFfmpeg(process.env, ffmpegStatic);
|
|
123
|
-
//
|
|
124
|
-
//
|
|
125
|
-
//
|
|
123
|
+
// A stage counts what it did and the queue is flushed after each new event. `record`
|
|
124
|
+
// swallows its own failures, so this can neither fail a stage nor widen what leaves the
|
|
125
|
+
// machine - the payload allow-list is checked inside it.
|
|
126
126
|
const count = (type, counters) => {
|
|
127
127
|
record(telemetry, type, counters);
|
|
128
128
|
flusher.soon();
|
|
@@ -152,11 +152,10 @@ function wire({ db, paths, clock, ids, log, hub, telemetry, flusher, registry })
|
|
|
152
152
|
thumbnail: (context) => runThumbnail(writing, context, stageProviders(providers, context)),
|
|
153
153
|
video: (context) => renderVideo(video, context),
|
|
154
154
|
},
|
|
155
|
-
//
|
|
156
|
-
//
|
|
157
|
-
//
|
|
158
|
-
//
|
|
159
|
-
// nothing: the kernel may not import a slice (03-conventions).
|
|
155
|
+
// Counting is finer than a stage reaching `done` - each intro and outro text, each
|
|
156
|
+
// narrated segment - and the counters are provider names, token usage and durations
|
|
157
|
+
// only the stage slice ever sees. Each slice records its own units through `count`
|
|
158
|
+
// above, and the runner counts nothing: the kernel may not import a slice.
|
|
160
159
|
emit: (projectId, event) => {
|
|
161
160
|
hub.emit(projectId, event);
|
|
162
161
|
},
|
|
@@ -203,7 +202,7 @@ export function urlOf(host, port) {
|
|
|
203
202
|
return `http://${host.includes(":") ? `[${host}]` : host}:${port}`;
|
|
204
203
|
}
|
|
205
204
|
// A stage can only be `running` at boot if the previous process died mid-run;
|
|
206
|
-
// nothing auto-resumes, the user retries by hand
|
|
205
|
+
// nothing auto-resumes, the user retries by hand.
|
|
207
206
|
export function markInterruptedStages(db, clock) {
|
|
208
207
|
const result = db
|
|
209
208
|
.prepare("UPDATE stages SET state = 'failed', failure_reason = 'interrupted', finished_at = ? WHERE state = 'running'")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// The format is the kernel's: the image port asks for the same two aspects.
|
|
2
2
|
export { formats } from "../../kernel/pipeline.js";
|
|
3
|
-
//
|
|
4
|
-
//
|
|
3
|
+
// Generate, Provide or Off for most stages, plus the thumbnail's two Generate modes. Video
|
|
4
|
+
// is always `generate`.
|
|
5
5
|
export const stageSources = ["generate", "provide", "off", "from_prompt", "prompt_by_llm"];
|
|
6
6
|
export const entryModes = ["text", "llm"];
|
|
@@ -5,7 +5,7 @@ import { entryModes, formats, stageSources } from "./model.js";
|
|
|
5
5
|
const providerChoice = z.object({ provider: z.string(), model: z.string() });
|
|
6
6
|
const entryChoice = z.object({ name: z.string(), mode: z.enum(entryModes) });
|
|
7
7
|
// The shape Play posts and the shape `projects.config` holds, in one place: the second
|
|
8
|
-
// is the first plus the rendered prompt texts
|
|
8
|
+
// is the first plus the rendered prompt texts.
|
|
9
9
|
export const runDraftSchema = z.object({
|
|
10
10
|
title: z.string(),
|
|
11
11
|
format: z.enum(formats),
|
|
@@ -35,8 +35,8 @@ export const runDraftSchema = z.object({
|
|
|
35
35
|
images: z.array(z.string()).optional(),
|
|
36
36
|
thumbnail: z.string().optional(),
|
|
37
37
|
}),
|
|
38
|
-
//
|
|
39
|
-
//
|
|
38
|
+
// Optional until Play carries the control; unknown keys are stripped by this schema, so
|
|
39
|
+
// a mode not listed here would never reach the audio stage.
|
|
40
40
|
chunking: z.object({ mode: z.enum(chunkModes), words: z.number().optional() }).optional(),
|
|
41
41
|
silenceGapSeconds: z.number(),
|
|
42
42
|
});
|
|
@@ -87,7 +87,7 @@ export function projectById(db, id) {
|
|
|
87
87
|
export function projectExists(db, id) {
|
|
88
88
|
return db.prepare("SELECT 1 FROM projects WHERE id = ?").get(id) !== undefined;
|
|
89
89
|
}
|
|
90
|
-
//
|
|
90
|
+
// Newest first.
|
|
91
91
|
export function listProjects(db) {
|
|
92
92
|
return db
|
|
93
93
|
.prepare("SELECT * FROM projects ORDER BY created_at DESC, id DESC")
|
|
@@ -137,9 +137,8 @@ export function claimStage(db, stageId, at) {
|
|
|
137
137
|
export function finishStage(db, stageId, state, failureReason, at) {
|
|
138
138
|
db.prepare("UPDATE stages SET state = ?, failure_reason = ?, finished_at = ? WHERE id = ?").run(state, failureReason, at, stageId);
|
|
139
139
|
}
|
|
140
|
-
//
|
|
141
|
-
//
|
|
142
|
-
// run before it, and the fresh attempt budget of `logic/01` §Q5.
|
|
140
|
+
// A stage put back to `pending` by a re-run, a cascade or a retry starts again from a clean row
|
|
141
|
+
// - no error text, no progress from the run before it, and a fresh attempt budget.
|
|
143
142
|
export function resetStage(db, stageId) {
|
|
144
143
|
db.prepare("UPDATE stages SET state = 'pending', failure_reason = NULL, attempt_count = 0, progress_current = NULL, progress_total = NULL, started_at = NULL, finished_at = NULL WHERE id = ?").run(stageId);
|
|
145
144
|
}
|
|
@@ -3,15 +3,14 @@ export const titleMax = 200;
|
|
|
3
3
|
export const valueMax = 200;
|
|
4
4
|
export const numberPerPromptMax = 20;
|
|
5
5
|
export const imagesPerRunMax = 60;
|
|
6
|
-
// ceiling:
|
|
7
|
-
//
|
|
6
|
+
// ceiling: the default gap is 3 s and nothing fixes an upper or lower bound, so the range
|
|
7
|
+
// is this module's. A wider gap is a settings change, not a schema change.
|
|
8
8
|
export const silenceGapSecondsMax = 30;
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
// itself only asks whether a source is on its stage's list.
|
|
9
|
+
// The legal source for each stage; video is always generated. Exported because Play's
|
|
10
|
+
// source switches offer exactly these and nothing else, so the segmented control and the
|
|
11
|
+
// refusal below it are the same list and a stage whose legal set changes cannot leave a
|
|
12
|
+
// switch offering something the server refuses. The order is the order the segments are
|
|
13
|
+
// drawn in; the rule itself only asks whether a source is on its stage's list.
|
|
15
14
|
export const allowedSources = {
|
|
16
15
|
research: ["off", "generate", "provide"],
|
|
17
16
|
article: ["generate", "provide"],
|
|
@@ -38,8 +37,7 @@ export function admit(input) {
|
|
|
38
37
|
});
|
|
39
38
|
}
|
|
40
39
|
}
|
|
41
|
-
//
|
|
42
|
-
// something in the run actually asks an LLM for text.
|
|
40
|
+
// The LLM row is required only when something in the run actually asks an LLM for text.
|
|
43
41
|
const needsLlm = sources.research === "generate" ||
|
|
44
42
|
sources.article === "generate" ||
|
|
45
43
|
sources.thumbnail === "prompt_by_llm" ||
|
|
@@ -82,8 +80,8 @@ export function admit(input) {
|
|
|
82
80
|
}
|
|
83
81
|
return fields.length === 0 ? { ok: true, draft } : { ok: false, fields };
|
|
84
82
|
}
|
|
85
|
-
//
|
|
86
|
-
//
|
|
83
|
+
// Research only feeds article writing, so a provided article hides it.
|
|
84
|
+
// Video is generated whatever the form said.
|
|
87
85
|
function normalise(draft) {
|
|
88
86
|
const sources = { ...draft.sources, video: "generate" };
|
|
89
87
|
if (sources.article === "provide") {
|
|
@@ -100,7 +98,7 @@ function normalise(draft) {
|
|
|
100
98
|
}
|
|
101
99
|
function checkImagePrompts(draft, fields) {
|
|
102
100
|
if (draft.imagePrompts.length === 0) {
|
|
103
|
-
//
|
|
101
|
+
// A run always has an image source.
|
|
104
102
|
fields.push({ field: "imagePrompts", message: "Tick at least one image prompt." });
|
|
105
103
|
return;
|
|
106
104
|
}
|
|
@@ -159,7 +157,7 @@ function checkProvided(draft, staged, fields) {
|
|
|
159
157
|
}
|
|
160
158
|
}
|
|
161
159
|
}
|
|
162
|
-
//
|
|
160
|
+
// A run never starts with provided content that is missing or still copying.
|
|
163
161
|
function checkFile(staged, id, kind, field, missing, fields) {
|
|
164
162
|
if (id === undefined || id === "") {
|
|
165
163
|
fields.push({ field, message: missing });
|
|
@@ -174,7 +172,6 @@ function checkFile(staged, id, kind, field, missing, fields) {
|
|
|
174
172
|
fields.push({ field, message: "This upload is still copying." });
|
|
175
173
|
}
|
|
176
174
|
}
|
|
177
|
-
// logic/03 step 4 and §Q26.
|
|
178
175
|
function checkValues(draft, requiredSlots, fields) {
|
|
179
176
|
for (const name of requiredSlots) {
|
|
180
177
|
const value = Object.hasOwn(draft.values, name) ? draft.values[name] : undefined;
|
|
@@ -3,7 +3,7 @@ import { stageKinds } from "../../kernel/pipeline.js";
|
|
|
3
3
|
import { storeArticleText } from "../article/store.js";
|
|
4
4
|
import { attachStagedFile, dropStagedSource, storeText } from "../storage/staging.js";
|
|
5
5
|
import { insertProject, insertStage } from "./repo.js";
|
|
6
|
-
//
|
|
6
|
+
// Provide → `provided` with its output attached; Off → `skipped`;
|
|
7
7
|
// everything else → `pending`.
|
|
8
8
|
export function initialState(source) {
|
|
9
9
|
if (source === "provide") {
|
|
@@ -11,9 +11,8 @@ export function initialState(source) {
|
|
|
11
11
|
}
|
|
12
12
|
return source === "off" ? "skipped" : "pending";
|
|
13
13
|
}
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
// this returns, never inside it.
|
|
14
|
+
// The project row, its six stages and the provided content all land together or not at
|
|
15
|
+
// all. The caller ticks the runner after this returns, never inside it.
|
|
17
16
|
export function startRun(deps, draft, rendered) {
|
|
18
17
|
const id = deps.ids.next();
|
|
19
18
|
const at = deps.clock.now().toISOString();
|
|
@@ -66,9 +65,9 @@ function attachProvided(deps, projectId, draft, collected) {
|
|
|
66
65
|
});
|
|
67
66
|
}
|
|
68
67
|
if (sources.article === "provide" && provided.article !== undefined) {
|
|
69
|
-
//
|
|
70
|
-
//
|
|
71
|
-
//
|
|
68
|
+
// The end-matter split runs when the article becomes done *or provided*, so a pasted
|
|
69
|
+
// Sources Consulted list is cut into its own file here exactly as the article stage cuts a
|
|
70
|
+
// written one, and never reaches the narration.
|
|
72
71
|
storeArticleText(deps, { projectId, markdown: provided.article.trim() });
|
|
73
72
|
}
|
|
74
73
|
if (sources.audio === "provide") {
|
|
@@ -78,7 +77,7 @@ function attachProvided(deps, projectId, draft, collected) {
|
|
|
78
77
|
attach(deps, projectId, "thumbnail", provided.thumbnail, "thumbnail", collected);
|
|
79
78
|
}
|
|
80
79
|
if (sources.images === "provide") {
|
|
81
|
-
//
|
|
80
|
+
// Slideshow order is the order the user left the list in.
|
|
82
81
|
for (const [index, stagedFileId] of (provided.images ?? []).entries()) {
|
|
83
82
|
attach(deps, projectId, "images", stagedFileId, "image", collected, index + 1);
|
|
84
83
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
//
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
1
|
+
// The slot grammar: `{{` … `}}`, whitespace immediately inside the braces stripped, the
|
|
2
|
+
// name case-sensitive and free to hold anything but `{`, `}` and a newline (so
|
|
3
|
+
// `{{Middle of Words}}` is one name). There is no escape syntax. Hand-rolled on purpose: a
|
|
4
|
+
// template library would bring its own grammar.
|
|
5
5
|
export const slotLintKinds = ["unclosed", "empty", "nested"];
|
|
6
6
|
export const fieldGroups = ["common", "text", "image"];
|
|
7
7
|
// A well-formed slot: no brace and no newline between the delimiters.
|
|
@@ -37,9 +37,9 @@ export function detectSlots(body) {
|
|
|
37
37
|
}
|
|
38
38
|
return { names, errors };
|
|
39
39
|
}
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
40
|
+
// One field per distinct name; Common when a name is used on both sides, otherwise Text or
|
|
41
|
+
// Image. The order is first appearance, which is why both sides arrive as ordered lists of
|
|
42
|
+
// bodies rather than as sets.
|
|
43
43
|
export function collectFields(textBodies, imageBodies) {
|
|
44
44
|
const text = namesOf(textBodies);
|
|
45
45
|
const image = namesOf(imageBodies);
|
|
@@ -54,12 +54,12 @@ export function collectFields(textBodies, imageBodies) {
|
|
|
54
54
|
}
|
|
55
55
|
return fields;
|
|
56
56
|
}
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
57
|
+
// One pass, so a value that itself contains `{{x}}` lands in the output verbatim and is never
|
|
58
|
+
// looked at again. A name with no value is left as it was written; admission refuses the run
|
|
59
|
+
// before this can reach a provider.
|
|
60
60
|
export function render(body, values) {
|
|
61
61
|
return body.replace(slot, (whole, inside) => {
|
|
62
|
-
// A slot name may be anything but a brace or a newline
|
|
62
|
+
// A slot name may be anything but a brace or a newline, so "constructor" and
|
|
63
63
|
// "toString" are ordinary names; a plain lookup would answer them from Object's
|
|
64
64
|
// prototype and splice a function's source into a prompt bound for a provider.
|
|
65
65
|
const name = inside.trim();
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { noTokens, plusUsage } from "../telemetry/model.js";
|
|
2
|
-
//
|
|
2
|
+
// At most 3 continuations; still unfinished after the third is a failed attempt.
|
|
3
3
|
export const continuationLimit = 3;
|
|
4
4
|
// The provider's own word for "I stopped because I ran out of room", which is what every
|
|
5
5
|
// adapter maps its finish reason to (`kernel/ports/llm.ts`).
|
|
6
6
|
const truncatedReason = "length";
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
7
|
+
// A fixed 'Research notes' header followed by the notes, then the rendered article prompt;
|
|
8
|
+
// without research, the rendered prompt alone. One user message, and no parameters of the app's
|
|
9
|
+
// own: the provider's defaults are used.
|
|
10
10
|
export function articleMessages(brief) {
|
|
11
11
|
const notes = brief.notes?.trim();
|
|
12
12
|
const content = notes === undefined || notes === ""
|
|
@@ -16,9 +16,9 @@ export function articleMessages(brief) {
|
|
|
16
16
|
}
|
|
17
17
|
// The article so far goes back as the assistant turn it was, so the model continues its
|
|
18
18
|
// own answer rather than being asked to write a second article. Nothing is inserted at
|
|
19
|
-
// the seam - the pieces are concatenated exactly as they arrived
|
|
20
|
-
//
|
|
21
|
-
//
|
|
19
|
+
// the seam - the pieces are concatenated exactly as they arrived, and the stored text is
|
|
20
|
+
// the model's, never edited by the app - so the instruction has to carry the whole of the
|
|
21
|
+
// rule that keeps the seam invisible.
|
|
22
22
|
export function continuationMessages(base, soFar) {
|
|
23
23
|
return [
|
|
24
24
|
...base,
|
|
@@ -54,7 +54,7 @@ export async function writeArticle(providers, choice, brief, onDelta) {
|
|
|
54
54
|
model: choice.model,
|
|
55
55
|
messages,
|
|
56
56
|
// The last continuation allowed is the one that has to end the article: a fourth
|
|
57
|
-
// truncation is a failed attempt, which is the wrapper's to retry
|
|
57
|
+
// truncation is a failed attempt, which is the wrapper's to retry. The
|
|
58
58
|
// loop therefore never sees a truncated answer with its budget spent.
|
|
59
59
|
check: n === continuationLimit ? finished : written,
|
|
60
60
|
}, stream);
|
|
@@ -66,7 +66,7 @@ export async function writeArticle(providers, choice, brief, onDelta) {
|
|
|
66
66
|
function truncated(answer) {
|
|
67
67
|
return answer.finishReason === truncatedReason;
|
|
68
68
|
}
|
|
69
|
-
//
|
|
69
|
+
// Empty response → failed attempt.
|
|
70
70
|
function written(answer) {
|
|
71
71
|
return answer.text.trim() === "" ? "the article answered with nothing" : undefined;
|
|
72
72
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { remark } from "remark";
|
|
2
2
|
import remarkGfm from "remark-gfm";
|
|
3
3
|
import stripMarkdown from "strip-markdown";
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
4
|
+
// The article is stored twice, as the markdown the model wrote and as the plain text the
|
|
5
|
+
// narration is read from, so no TTS voice ever says a hash or a bracket. remark parses and
|
|
6
|
+
// strip-markdown drops the formatting; the GFM extension is what makes a table a table rather
|
|
7
|
+
// than a paragraph full of pipes, and what takes a footnote marker out of the middle of a
|
|
8
|
+
// sentence.
|
|
9
9
|
export function plainText(markdown) {
|
|
10
10
|
// Built per call: a shared processor would be a module-level singleton, and building
|
|
11
11
|
// one costs a few microseconds against a call that just parsed an article.
|
|
@@ -17,29 +17,29 @@ export async function runArticle(deps, context, providers) {
|
|
|
17
17
|
const choice = project.config.llm;
|
|
18
18
|
const articlePrompt = project.config.rendered.article;
|
|
19
19
|
if (choice === undefined || articlePrompt === undefined) {
|
|
20
|
-
// Admission refuses a run whose article is Generate without both
|
|
20
|
+
// Admission refuses a run whose article is Generate without both, so
|
|
21
21
|
// reaching here is a bug in admission rather than something the user did.
|
|
22
22
|
throw new Error("the run has no LLM provider or no rendered article prompt");
|
|
23
23
|
}
|
|
24
24
|
const notes = researchNotes(deps, projectId);
|
|
25
25
|
const brief = { articlePrompt, ...(notes === undefined ? {} : { notes }) };
|
|
26
26
|
const written = await writeArticle(providers, choice, brief, (text) => {
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
27
|
+
// The page shows the article as it is written. The idle timeout is restarted by the wrapper
|
|
28
|
+
// on the same events, not here. ceiling: the partial text of a failed attempt is discarded,
|
|
29
|
+
// but the deltas already sent cannot be unsent, so a retry mid-stream leaves the page
|
|
30
|
+
// appending the second telling under the first. What the project keeps is still the
|
|
31
|
+
// successful attempt's text alone; the upgrade is an event telling the page to start the
|
|
32
|
+
// article again.
|
|
33
33
|
context.emit({ type: "article.delta", projectId, text });
|
|
34
34
|
});
|
|
35
|
-
//
|
|
36
|
-
//
|
|
35
|
+
// The markdown exactly as the model produced it, and the plain-text narration source.
|
|
36
|
+
// The end matter is kept out of the narration and stored beside the article instead.
|
|
37
37
|
const store = { projectId, stageKind: "article" };
|
|
38
38
|
storeText(deps, { ...store, role: "article_md", text: written.markdown });
|
|
39
39
|
const narration = storeArticleText(deps, { projectId, markdown: written.markdown });
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
40
|
+
// The article and each entry text are counted as units of their own, so the article's
|
|
41
|
+
// event goes out as soon as its text is stored, carrying the tokens of the first call
|
|
42
|
+
// and its continuations.
|
|
43
43
|
deps.count("stage.completed", {
|
|
44
44
|
stage: "article",
|
|
45
45
|
provider: choice.provider,
|
|
@@ -51,9 +51,9 @@ export async function runArticle(deps, context, providers) {
|
|
|
51
51
|
const segment = await writeSegment(providers, choice, project.config, category, narration, sent);
|
|
52
52
|
if (segment !== undefined) {
|
|
53
53
|
keepSegment(deps, context, segment, index + 1);
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
//
|
|
54
|
+
// One event per intro/outro text, named by its segment. A text-mode entry is
|
|
55
|
+
// rendered rather than written, so it made no call, names no provider and reports
|
|
56
|
+
// zero tokens rather than an estimate.
|
|
57
57
|
deps.count("stage.completed", {
|
|
58
58
|
stage: "article",
|
|
59
59
|
segment: category,
|
|
@@ -62,7 +62,7 @@ export async function runArticle(deps, context, providers) {
|
|
|
62
62
|
});
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
|
-
//
|
|
65
|
+
// The exact messages sent, the continuations and the entry calls among them.
|
|
66
66
|
storeText(deps, { ...store, role: "instructions", text: instructionsText(sent) });
|
|
67
67
|
deps.log.write("info", "article.done", {
|
|
68
68
|
projectId,
|
|
@@ -70,9 +70,9 @@ export async function runArticle(deps, context, providers) {
|
|
|
70
70
|
detail: `${String(written.sent.length - 1)} continuations, ${String(narration.length)} characters to narrate`,
|
|
71
71
|
});
|
|
72
72
|
}
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
//
|
|
73
|
+
// For each picked entry in LLM mode, one call with the filled entry as instruction plus the
|
|
74
|
+
// title, keyword values, and the plain-text article ... Text-mode entries are stored as
|
|
75
|
+
// rendered, with no call.
|
|
76
76
|
async function writeSegment(providers, choice, config, category, article, sent) {
|
|
77
77
|
const picked = config[category];
|
|
78
78
|
if (picked === undefined) {
|
|
@@ -92,7 +92,7 @@ async function writeSegment(providers, choice, config, category, article, sent)
|
|
|
92
92
|
provider: choice.provider,
|
|
93
93
|
model: choice.model,
|
|
94
94
|
messages,
|
|
95
|
-
//
|
|
95
|
+
// An entry that answers with nothing is a failed attempt like any
|
|
96
96
|
// other, and the wrapper is what retries it.
|
|
97
97
|
check: (given) => given.text.trim() === "" ? `the ${category} answered with nothing` : undefined,
|
|
98
98
|
});
|
|
@@ -125,8 +125,8 @@ function segmentMessages(body, config, article) {
|
|
|
125
125
|
// a failure replaces what it wrote rather than colliding with it.
|
|
126
126
|
function keepSegment(deps, context, segment, idx) {
|
|
127
127
|
// Written out rather than stringified whole: the caller hands in what the call cost as
|
|
128
|
-
// well, and the
|
|
129
|
-
// about the model that wrote it.
|
|
128
|
+
// well, and the narration stage is what reads the piece - it carries the text to speak,
|
|
129
|
+
// nothing about the model that wrote it.
|
|
130
130
|
const payload = JSON.stringify({
|
|
131
131
|
category: segment.category,
|
|
132
132
|
name: segment.name,
|
|
@@ -147,9 +147,9 @@ function keepSegment(deps, context, segment, idx) {
|
|
|
147
147
|
}
|
|
148
148
|
setPiece(deps.db, existing.id, "done", payload);
|
|
149
149
|
}
|
|
150
|
-
// Research writes its notes as an output of its own
|
|
151
|
-
//
|
|
152
|
-
//
|
|
150
|
+
// Research writes its notes as an output of its own, and a provided research stage stores the
|
|
151
|
+
// pasted text the same way, so one lookup covers both. No row means research was Off or
|
|
152
|
+
// skipped, and the article is written from the prompt alone.
|
|
153
153
|
function researchNotes(deps, projectId) {
|
|
154
154
|
const notes = outputsOf(deps.db, projectId).find((output) => output.role === "notes");
|
|
155
155
|
if (notes === undefined) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { remark } from "remark";
|
|
2
2
|
import remarkGfm from "remark-gfm";
|
|
3
3
|
import { plainText } from "./plain.js";
|
|
4
|
-
//
|
|
4
|
+
// The two end-matter sections. The comparison is on the heading's own text, so "## Sources
|
|
5
5
|
// Consulted", "# SOURCES CONSULTED" and "**Sources Consulted:**" are one heading and
|
|
6
6
|
// "## Sources Consulted and Further Reading" is not.
|
|
7
7
|
const endHeadings = {
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import { storeText } from "../storage/staging.js";
|
|
2
2
|
import { plainText } from "./plain.js";
|
|
3
3
|
import { splitEndMatter } from "./split.js";
|
|
4
|
-
// The narration source it stored, so a caller that needs the article as it will be spoken
|
|
5
|
-
//
|
|
4
|
+
// The narration source it stored, so a caller that needs the article as it will be spoken -
|
|
5
|
+
// the intro and outro are written from it - does not read the file back.
|
|
6
6
|
export function storeArticleText(deps, input) {
|
|
7
7
|
const end = splitEndMatter(input.markdown);
|
|
8
8
|
const store = { projectId: input.projectId, stageKind: "article" };
|
|
9
|
-
//
|
|
10
|
-
//
|
|
9
|
+
// The narration source of an article is always plain text, so no voice ever says a hash or a
|
|
10
|
+
// bracket, whoever wrote the markdown.
|
|
11
11
|
const narration = plainText(end.body);
|
|
12
12
|
storeText(deps, { ...store, role: "article_txt", text: narration });
|
|
13
|
-
//
|
|
13
|
+
// No such heading means no file, not an empty one.
|
|
14
14
|
if (end.sources.trim() !== "") {
|
|
15
15
|
storeText(deps, { ...store, role: "sources", text: end.sources });
|
|
16
16
|
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { derive } from "../../kernel/runner/graph.js";
|
|
2
2
|
import { finishStage, projectExists, stagesOf } from "../admission/repo.js";
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
//
|
|
3
|
+
// Cancel on the project header. Every in-flight call of this project is aborted at once, every
|
|
4
|
+
// `done` output and every finished piece is kept for the resume, and the project sits
|
|
5
|
+
// `canceled` until the user retries a stage.
|
|
6
|
+
// The cancel rules and the stage transition table, in the words the stage row carries.
|
|
7
7
|
export const canceledByUser = "canceled by user";
|
|
8
8
|
export async function cancelProject(deps, projectId) {
|
|
9
9
|
if (!projectExists(deps.db, projectId)) {
|
|
@@ -12,14 +12,14 @@ export async function cancelProject(deps, projectId) {
|
|
|
12
12
|
const before = stagesOf(deps.db, projectId);
|
|
13
13
|
const running = before.filter((stage) => stage.state === "running").map((stage) => stage.kind);
|
|
14
14
|
if (running.length === 0) {
|
|
15
|
-
//
|
|
15
|
+
// A second click is a no-op. Nothing is aborted and no state changes, so
|
|
16
16
|
// the page is simply told what the project already reads.
|
|
17
17
|
return { ok: true, canceled: [], state: derive(before) };
|
|
18
18
|
}
|
|
19
|
-
//
|
|
19
|
+
// Nothing waits for a response. The runner holds the controllers and its own
|
|
20
20
|
// barrier, so a stage that finishes during this does not release its dependents.
|
|
21
21
|
await deps.abort(projectId);
|
|
22
|
-
//
|
|
22
|
+
// The invariant: after cancel completes no stage of the project is `running`.
|
|
23
23
|
// The runner writes that row as each aborted stage unwinds; this is the path where it
|
|
24
24
|
// could not - a failed write is logged there and the run is left mid-flight otherwise.
|
|
25
25
|
const after = stagesOf(deps.db, projectId);
|
|
@@ -51,7 +51,7 @@ export async function cancelProject(deps, projectId) {
|
|
|
51
51
|
}
|
|
52
52
|
return {
|
|
53
53
|
ok: true,
|
|
54
|
-
//
|
|
54
|
+
// A stage whose output was stored in the same instant as the cancel stays
|
|
55
55
|
// `done`, so what was stopped is read back rather than assumed from what was running.
|
|
56
56
|
canceled: settled
|
|
57
57
|
.filter((stage) => stage.state === "canceled" && running.includes(stage.kind))
|