@gentbajko/slopify 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/edge/events/hub.js +4 -0
- package/dist/edge/http/actions.js +68 -14
- package/dist/edge/http/projects.js +17 -10
- package/dist/kernel/db/migrations/0002-project-controls.sql +4 -0
- package/dist/kernel/pipeline.js +10 -5
- package/dist/kernel/runner/graph.js +18 -18
- package/dist/kernel/runner/index.js +10 -6
- package/dist/kernel/runner/piece-repo.js +8 -1
- package/dist/main.js +30 -3
- package/dist/slices/admission/model.js +2 -2
- package/dist/slices/admission/repo.js +20 -4
- package/dist/slices/admission/rules.js +26 -12
- package/dist/slices/admission/start.js +3 -1
- package/dist/slices/article/provided-entries.js +62 -0
- package/dist/slices/article/run.js +63 -101
- package/dist/slices/article/segments.js +73 -0
- package/dist/slices/cancel/index.js +27 -4
- package/dist/slices/control/index.js +155 -0
- package/dist/slices/control/lock.js +19 -0
- package/dist/slices/control/providers.js +59 -0
- package/dist/slices/library/slots.js +3 -1
- package/dist/slices/reruns/cascade.js +11 -7
- package/dist/slices/reruns/index.js +23 -1
- package/dist/slices/storage/layout.js +2 -0
- package/dist/slices/storage/model.js +1 -0
- package/dist/slices/video/audio-export.js +66 -0
- package/dist/slices/video/audio-inputs.js +32 -0
- package/dist/slices/video/ffmpeg.js +5 -5
- package/dist/slices/video/plan.js +9 -8
- package/dist/slices/video/run.js +25 -141
- package/dist/slices/video/write-export.js +98 -0
- package/dist/web/assets/index-C1fsrIP5.css +1 -0
- package/dist/web/assets/index-DwJ_XTl3.js +81 -0
- package/dist/web/index.html +2 -2
- package/package.json +1 -1
- package/dist/web/assets/index-BZvZKvIv.js +0 -81
- package/dist/web/assets/index-cpjfvs5E.css +0 -1
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { readFileSync } from "node:fs";
|
|
2
|
+
import { piecesOf } from "../../kernel/runner/piece-repo.js";
|
|
3
|
+
import { projectById, stagesOf } from "../admission/repo.js";
|
|
4
|
+
import { outputPath } from "../storage/layout.js";
|
|
5
|
+
import { outputsOf } from "../storage/repo.js";
|
|
6
|
+
import { storeText } from "../storage/staging.js";
|
|
7
|
+
import { categories, instructionsText, keepSegment, writeSegment } from "./segments.js";
|
|
8
|
+
// A provided Article never runs its writer. Prepare its selected entries as part
|
|
9
|
+
// of Audio's cancellable work, retaining Article ownership of the resulting text.
|
|
10
|
+
export async function prepareProvidedArticleSegments(deps, audioContext, providers) {
|
|
11
|
+
const { projectId } = audioContext.stage;
|
|
12
|
+
const project = projectById(deps.db, projectId);
|
|
13
|
+
if (!project)
|
|
14
|
+
throw new Error(`project ${projectId} has no row`);
|
|
15
|
+
const { config } = project;
|
|
16
|
+
if (config.sources.article !== "provide" || config.sources.audio !== "generate")
|
|
17
|
+
return;
|
|
18
|
+
if (!config.intro && !config.outro)
|
|
19
|
+
return;
|
|
20
|
+
audioContext.signal.throwIfAborted();
|
|
21
|
+
const article = stagesOf(deps.db, projectId).find((stage) => stage.kind === "article");
|
|
22
|
+
if (!article)
|
|
23
|
+
throw new Error(`project ${projectId} has no article stage`);
|
|
24
|
+
const outputs = outputsOf(deps.db, projectId);
|
|
25
|
+
const plain = outputs.find((output) => output.role === "article_txt");
|
|
26
|
+
if (!plain)
|
|
27
|
+
throw new Error("the run has no provided article text");
|
|
28
|
+
const narration = readFileSync(outputPath(deps.paths, projectId, plain.path), "utf8");
|
|
29
|
+
const saved = outputs.find((output) => output.stageKind === "article" && output.role === "instructions");
|
|
30
|
+
let instructions = saved
|
|
31
|
+
? readFileSync(outputPath(deps.paths, projectId, saved.path), "utf8")
|
|
32
|
+
: "";
|
|
33
|
+
const completed = piecesOf(deps.db, article.id, "segment");
|
|
34
|
+
for (const [index, category] of categories.entries()) {
|
|
35
|
+
audioContext.signal.throwIfAborted();
|
|
36
|
+
if (completed.some((piece) => piece.idx === index + 1 && piece.state === "done"))
|
|
37
|
+
continue;
|
|
38
|
+
const sent = [];
|
|
39
|
+
const segment = await writeSegment(providers, config.llm, config, category, narration, sent);
|
|
40
|
+
audioContext.signal.throwIfAborted();
|
|
41
|
+
if (!segment)
|
|
42
|
+
continue;
|
|
43
|
+
if (sent.length > 0) {
|
|
44
|
+
instructions += `${instructions === "" ? "" : "\n"}${instructionsText(sent)}`;
|
|
45
|
+
storeText(deps, {
|
|
46
|
+
projectId,
|
|
47
|
+
stageKind: "article",
|
|
48
|
+
role: "instructions",
|
|
49
|
+
text: instructions,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
keepSegment(deps, { stage: article }, segment, index + 1);
|
|
53
|
+
deps.count("stage.completed", {
|
|
54
|
+
stage: "article",
|
|
55
|
+
segment: category,
|
|
56
|
+
...(segment.mode === "llm" && config.llm
|
|
57
|
+
? { provider: config.llm.provider, model: config.llm.model }
|
|
58
|
+
: {}),
|
|
59
|
+
...segment.tokens,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
import { readFileSync } from "node:fs";
|
|
1
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { messageRoles } from "../../kernel/ports/llm.js";
|
|
2
4
|
import { insertPiece, piecesOf, setPiece } from "../../kernel/runner/piece-repo.js";
|
|
3
5
|
import { projectById } from "../admission/repo.js";
|
|
4
6
|
import { outputPath } from "../storage/layout.js";
|
|
5
7
|
import { outputsOf } from "../storage/repo.js";
|
|
6
8
|
import { storeText } from "../storage/staging.js";
|
|
7
|
-
import { noTokens, plusUsage } from "../telemetry/model.js";
|
|
8
9
|
import { writeArticle } from "./continuation.js";
|
|
10
|
+
import { categories, instructionsText, keepSegment, writeSegment } from "./segments.js";
|
|
9
11
|
import { storeArticleText } from "./store.js";
|
|
10
|
-
const categories = ["intro", "outro"];
|
|
11
12
|
export async function runArticle(deps, context, providers) {
|
|
12
13
|
const { projectId } = context.stage;
|
|
13
14
|
const project = projectById(deps.db, projectId);
|
|
@@ -23,34 +24,19 @@ export async function runArticle(deps, context, providers) {
|
|
|
23
24
|
}
|
|
24
25
|
const notes = researchNotes(deps, projectId);
|
|
25
26
|
const brief = { articlePrompt, ...(notes === undefined ? {} : { notes }) };
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
// but the deltas already sent cannot be unsent, so a retry mid-stream leaves the page
|
|
30
|
-
// appending the second telling under the first. What the project keeps is still the
|
|
31
|
-
// successful attempt's text alone; the upgrade is an event telling the page to start the
|
|
32
|
-
// article again.
|
|
33
|
-
context.emit({ type: "article.delta", projectId, text });
|
|
34
|
-
});
|
|
35
|
-
// The markdown exactly as the model produced it, and the plain-text narration source.
|
|
36
|
-
// The end matter is kept out of the narration and stored beside the article instead.
|
|
37
|
-
const store = { projectId, stageKind: "article" };
|
|
38
|
-
storeText(deps, { ...store, role: "article_md", text: written.markdown });
|
|
39
|
-
const narration = storeArticleText(deps, { projectId, markdown: written.markdown });
|
|
40
|
-
// The article and each entry text are counted as units of their own, so the article's
|
|
41
|
-
// event goes out as soon as its text is stored, carrying the tokens of the first call
|
|
42
|
-
// and its continuations.
|
|
43
|
-
deps.count("stage.completed", {
|
|
44
|
-
stage: "article",
|
|
45
|
-
provider: choice.provider,
|
|
46
|
-
model: choice.model,
|
|
47
|
-
...written.tokens,
|
|
48
|
-
});
|
|
49
|
-
const sent = [...written.sent];
|
|
27
|
+
const completed = await articleBody(deps, context, providers, choice, brief);
|
|
28
|
+
const { narration } = completed;
|
|
29
|
+
const sent = [...completed.sent];
|
|
50
30
|
for (const [index, category] of categories.entries()) {
|
|
31
|
+
if (project.config.sources.audio !== "generate")
|
|
32
|
+
break;
|
|
33
|
+
if (completed.resumed &&
|
|
34
|
+
piecesOf(deps.db, context.stage.id, "segment").some((piece) => piece.idx === index + 1 && piece.state === "done"))
|
|
35
|
+
continue;
|
|
51
36
|
const segment = await writeSegment(providers, choice, project.config, category, narration, sent);
|
|
52
37
|
if (segment !== undefined) {
|
|
53
38
|
keepSegment(deps, context, segment, index + 1);
|
|
39
|
+
checkpointArticle(deps, context, sent);
|
|
54
40
|
// One event per intro/outro text, named by its segment. A text-mode entry is
|
|
55
41
|
// rendered rather than written, so it made no call, names no provider and reports
|
|
56
42
|
// zero tokens rather than an estimate.
|
|
@@ -63,89 +49,72 @@ export async function runArticle(deps, context, providers) {
|
|
|
63
49
|
}
|
|
64
50
|
}
|
|
65
51
|
// The exact messages sent, the continuations and the entry calls among them.
|
|
66
|
-
storeText(deps, {
|
|
52
|
+
storeText(deps, {
|
|
53
|
+
projectId,
|
|
54
|
+
stageKind: "article",
|
|
55
|
+
role: "instructions",
|
|
56
|
+
text: instructionsText(sent),
|
|
57
|
+
});
|
|
67
58
|
deps.log.write("info", "article.done", {
|
|
68
59
|
projectId,
|
|
69
60
|
stage: "article",
|
|
70
|
-
detail: `${String(
|
|
61
|
+
detail: `${String(narration.length)} characters to narrate`,
|
|
71
62
|
});
|
|
72
63
|
}
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
const
|
|
86
|
-
|
|
87
|
-
|
|
64
|
+
// Once the article is stored, a pause during end matter resumes that end matter
|
|
65
|
+
// instead of buying and replacing the article again. Re-run clears this checkpoint.
|
|
66
|
+
const articleCheckpoint = z.object({
|
|
67
|
+
sent: z.array(z.object({
|
|
68
|
+
label: z.string(),
|
|
69
|
+
messages: z.array(z.object({ role: z.enum(messageRoles), content: z.string() })),
|
|
70
|
+
})),
|
|
71
|
+
});
|
|
72
|
+
async function articleBody(deps, context, providers, choice, brief) {
|
|
73
|
+
const { projectId } = context.stage;
|
|
74
|
+
const saved = piecesOf(deps.db, context.stage.id, "article_written")[0];
|
|
75
|
+
const outputs = outputsOf(deps.db, projectId);
|
|
76
|
+
const plain = outputs.find((output) => output.role === "article_txt");
|
|
77
|
+
const markdown = outputs.find((output) => output.role === "article_md");
|
|
78
|
+
if (saved?.payload !== null &&
|
|
79
|
+
saved?.payload !== undefined &&
|
|
80
|
+
plain &&
|
|
81
|
+
markdown &&
|
|
82
|
+
existsSync(outputPath(deps.paths, projectId, plain.path)) &&
|
|
83
|
+
existsSync(outputPath(deps.paths, projectId, markdown.path))) {
|
|
84
|
+
return {
|
|
85
|
+
narration: readFileSync(outputPath(deps.paths, projectId, plain.path), "utf8"),
|
|
86
|
+
sent: articleCheckpoint.parse(JSON.parse(saved.payload)).sent,
|
|
87
|
+
resumed: true,
|
|
88
|
+
};
|
|
88
89
|
}
|
|
89
|
-
const
|
|
90
|
-
|
|
91
|
-
|
|
90
|
+
const written = await writeArticle(providers, choice, brief, (text) => {
|
|
91
|
+
context.emit({ type: "article.delta", projectId, text });
|
|
92
|
+
});
|
|
93
|
+
storeText(deps, { projectId, stageKind: "article", role: "article_md", text: written.markdown });
|
|
94
|
+
const narration = storeArticleText(deps, { projectId, markdown: written.markdown });
|
|
95
|
+
checkpointArticle(deps, context, written.sent);
|
|
96
|
+
deps.count("stage.completed", {
|
|
97
|
+
stage: "article",
|
|
92
98
|
provider: choice.provider,
|
|
93
99
|
model: choice.model,
|
|
94
|
-
|
|
95
|
-
// An entry that answers with nothing is a failed attempt like any
|
|
96
|
-
// other, and the wrapper is what retries it.
|
|
97
|
-
check: (given) => given.text.trim() === "" ? `the ${category} answered with nothing` : undefined,
|
|
100
|
+
...written.tokens,
|
|
98
101
|
});
|
|
99
|
-
return {
|
|
102
|
+
return { narration, sent: written.sent, resumed: false };
|
|
100
103
|
}
|
|
101
|
-
function
|
|
102
|
-
const
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
body,
|
|
108
|
-
"",
|
|
109
|
-
`Video title: ${config.title}`,
|
|
110
|
-
"",
|
|
111
|
-
"Keyword values for this run:",
|
|
112
|
-
"",
|
|
113
|
-
values.length === 0
|
|
114
|
-
? "(none)"
|
|
115
|
-
: values.map(([name, value]) => `${name}: ${value}`).join("\n"),
|
|
116
|
-
"",
|
|
117
|
-
"The article this video narrates:",
|
|
118
|
-
"",
|
|
119
|
-
article,
|
|
120
|
-
].join("\n"),
|
|
121
|
-
},
|
|
122
|
-
];
|
|
123
|
-
}
|
|
124
|
-
// The segment rows are keyed by `idx` within the stage, so a stage that runs again after
|
|
125
|
-
// a failure replaces what it wrote rather than colliding with it.
|
|
126
|
-
function keepSegment(deps, context, segment, idx) {
|
|
127
|
-
// Written out rather than stringified whole: the caller hands in what the call cost as
|
|
128
|
-
// well, and the narration stage is what reads the piece - it carries the text to speak,
|
|
129
|
-
// nothing about the model that wrote it.
|
|
130
|
-
const payload = JSON.stringify({
|
|
131
|
-
category: segment.category,
|
|
132
|
-
name: segment.name,
|
|
133
|
-
mode: segment.mode,
|
|
134
|
-
text: segment.text,
|
|
135
|
-
});
|
|
136
|
-
const existing = piecesOf(deps.db, context.stage.id, "segment").find((piece) => piece.idx === idx);
|
|
137
|
-
if (existing === undefined) {
|
|
104
|
+
function checkpointArticle(deps, context, sent) {
|
|
105
|
+
const existing = piecesOf(deps.db, context.stage.id, "article_written")[0];
|
|
106
|
+
const payload = JSON.stringify({ sent });
|
|
107
|
+
if (existing)
|
|
108
|
+
setPiece(deps.db, existing.id, "done", payload);
|
|
109
|
+
else
|
|
138
110
|
insertPiece(deps.db, {
|
|
139
111
|
id: deps.ids.next(),
|
|
140
112
|
stageId: context.stage.id,
|
|
141
|
-
kind: "
|
|
142
|
-
idx,
|
|
113
|
+
kind: "article_written",
|
|
114
|
+
idx: 1,
|
|
143
115
|
state: "done",
|
|
144
116
|
payload,
|
|
145
117
|
});
|
|
146
|
-
return;
|
|
147
|
-
}
|
|
148
|
-
setPiece(deps.db, existing.id, "done", payload);
|
|
149
118
|
}
|
|
150
119
|
// Research writes its notes as an output of its own, and a provided research stage stores the
|
|
151
120
|
// pasted text the same way, so one lookup covers both. No row means research was Off or
|
|
@@ -157,10 +126,3 @@ function researchNotes(deps, projectId) {
|
|
|
157
126
|
}
|
|
158
127
|
return readFileSync(outputPath(deps.paths, projectId, notes.path), "utf8");
|
|
159
128
|
}
|
|
160
|
-
function instructionsText(sent) {
|
|
161
|
-
const parts = sent.map((one) => `=== ${one.label} ===\n\n${one.messages.map((message) => message.content).join("\n\n")}`);
|
|
162
|
-
return `${parts.join("\n\n")}\n`;
|
|
163
|
-
}
|
|
164
|
-
function label(category) {
|
|
165
|
-
return category === "intro" ? "Intro" : "Outro";
|
|
166
|
-
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { insertPiece, piecesOf, setPiece } from "../../kernel/runner/piece-repo.js";
|
|
2
|
+
import { noTokens, plusUsage } from "../telemetry/model.js";
|
|
3
|
+
export const categories = ["intro", "outro"];
|
|
4
|
+
export async function writeSegment(providers, choice, config, category, article, sent) {
|
|
5
|
+
const picked = config[category];
|
|
6
|
+
if (picked === undefined)
|
|
7
|
+
return undefined;
|
|
8
|
+
const body = config.rendered[category];
|
|
9
|
+
if (body === undefined)
|
|
10
|
+
throw new Error(`the run has no rendered ${category} text`);
|
|
11
|
+
const common = { category, name: picked.name, mode: picked.mode };
|
|
12
|
+
if (picked.mode === "text")
|
|
13
|
+
return { ...common, text: body, tokens: noTokens };
|
|
14
|
+
if (choice === undefined)
|
|
15
|
+
throw new Error(`the ${category} has no LLM provider or model`);
|
|
16
|
+
const messages = segmentMessages(body, config, article);
|
|
17
|
+
sent.push({ label: category === "intro" ? "Intro" : "Outro", messages });
|
|
18
|
+
const answer = await providers.llm({
|
|
19
|
+
provider: choice.provider,
|
|
20
|
+
model: choice.model,
|
|
21
|
+
messages,
|
|
22
|
+
check: (given) => given.text.trim() === "" ? `the ${category} answered with nothing` : undefined,
|
|
23
|
+
});
|
|
24
|
+
return { ...common, text: answer.text.trim(), tokens: plusUsage(noTokens, answer.usage) };
|
|
25
|
+
}
|
|
26
|
+
function segmentMessages(body, config, article) {
|
|
27
|
+
const values = Object.entries(config.values);
|
|
28
|
+
return [
|
|
29
|
+
{
|
|
30
|
+
role: "user",
|
|
31
|
+
content: [
|
|
32
|
+
body,
|
|
33
|
+
"",
|
|
34
|
+
`Video title: ${config.title}`,
|
|
35
|
+
"",
|
|
36
|
+
"Keyword values for this run:",
|
|
37
|
+
"",
|
|
38
|
+
values.length === 0
|
|
39
|
+
? "(none)"
|
|
40
|
+
: values.map(([name, value]) => `${name}: ${value}`).join("\n"),
|
|
41
|
+
"",
|
|
42
|
+
"The article this video narrates:",
|
|
43
|
+
"",
|
|
44
|
+
article,
|
|
45
|
+
].join("\n"),
|
|
46
|
+
},
|
|
47
|
+
];
|
|
48
|
+
}
|
|
49
|
+
export function keepSegment(deps, context, segment, idx) {
|
|
50
|
+
const payload = JSON.stringify({
|
|
51
|
+
category: segment.category,
|
|
52
|
+
name: segment.name,
|
|
53
|
+
mode: segment.mode,
|
|
54
|
+
text: segment.text,
|
|
55
|
+
});
|
|
56
|
+
const existing = piecesOf(deps.db, context.stage.id, "segment").find((piece) => piece.idx === idx);
|
|
57
|
+
if (existing === undefined) {
|
|
58
|
+
insertPiece(deps.db, {
|
|
59
|
+
id: deps.ids.next(),
|
|
60
|
+
stageId: context.stage.id,
|
|
61
|
+
kind: "segment",
|
|
62
|
+
idx,
|
|
63
|
+
state: "done",
|
|
64
|
+
payload,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
else
|
|
68
|
+
setPiece(deps.db, existing.id, "done", payload);
|
|
69
|
+
}
|
|
70
|
+
export function instructionsText(sent) {
|
|
71
|
+
const parts = sent.map((one) => `=== ${one.label} ===\n\n${one.messages.map((message) => message.content).join("\n\n")}`);
|
|
72
|
+
return `${parts.join("\n\n")}\n`;
|
|
73
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { derive } from "../../kernel/runner/graph.js";
|
|
2
|
-
import { finishStage, projectExists, stagesOf } from "../admission/repo.js";
|
|
2
|
+
import { finishStage, projectExists, projectPaused, setProjectPaused, stagesOf, } from "../admission/repo.js";
|
|
3
3
|
// Cancel on the project header. Every in-flight call of this project is aborted at once, every
|
|
4
4
|
// `done` output and every finished piece is kept for the resume, and the project sits
|
|
5
5
|
// `canceled` until the user retries a stage.
|
|
@@ -10,15 +10,21 @@ export async function cancelProject(deps, projectId) {
|
|
|
10
10
|
return { ok: false, reason: "no-project" };
|
|
11
11
|
}
|
|
12
12
|
const before = stagesOf(deps.db, projectId);
|
|
13
|
+
const paused = projectPaused(deps.db, projectId);
|
|
14
|
+
if (paused) {
|
|
15
|
+
setProjectPaused(deps.db, projectId, false, deps.clock.now().toISOString());
|
|
16
|
+
deps.emit(projectId, { type: "project.updated", projectId });
|
|
17
|
+
}
|
|
13
18
|
const running = before.filter((stage) => stage.state === "running").map((stage) => stage.kind);
|
|
14
|
-
if (running.length === 0) {
|
|
19
|
+
if (running.length === 0 && !paused) {
|
|
15
20
|
// A second click is a no-op. Nothing is aborted and no state changes, so
|
|
16
21
|
// the page is simply told what the project already reads.
|
|
17
22
|
return { ok: true, canceled: [], state: derive(before) };
|
|
18
23
|
}
|
|
19
24
|
// Nothing waits for a response. The runner holds the controllers and its own
|
|
20
25
|
// barrier, so a stage that finishes during this does not release its dependents.
|
|
21
|
-
|
|
26
|
+
if (running.length > 0)
|
|
27
|
+
await deps.abort(projectId);
|
|
22
28
|
// The invariant: after cancel completes no stage of the project is `running`.
|
|
23
29
|
// The runner writes that row as each aborted stage unwinds; this is the path where it
|
|
24
30
|
// could not - a failed write is logged there and the run is left mid-flight otherwise.
|
|
@@ -41,9 +47,26 @@ export async function cancelProject(deps, projectId) {
|
|
|
41
47
|
failureReason: canceledByUser,
|
|
42
48
|
});
|
|
43
49
|
}
|
|
50
|
+
// An abort can race a successful stage completion. Mark one remaining pending
|
|
51
|
+
// stage canceled so cancellation is explicit, rather than guessing from any
|
|
52
|
+
// completed stage (independent images can now finish before Article).
|
|
53
|
+
const stopped = stagesOf(deps.db, projectId);
|
|
54
|
+
const waiting = stopped.find((stage) => stage.state === "pending");
|
|
55
|
+
if (!stopped.some((stage) => stage.state === "canceled") && waiting !== undefined) {
|
|
56
|
+
finishStage(deps.db, waiting.id, "canceled", canceledByUser, deps.clock.now().toISOString());
|
|
57
|
+
deps.emit(projectId, {
|
|
58
|
+
type: "stage.state",
|
|
59
|
+
projectId,
|
|
60
|
+
stage: waiting.kind,
|
|
61
|
+
state: "canceled",
|
|
62
|
+
failureReason: canceledByUser,
|
|
63
|
+
});
|
|
64
|
+
}
|
|
44
65
|
const settled = stagesOf(deps.db, projectId);
|
|
45
66
|
const state = derive(settled);
|
|
46
|
-
if (
|
|
67
|
+
if (paused ||
|
|
68
|
+
after.some((stage) => stage.state === "running") ||
|
|
69
|
+
(waiting !== undefined && !stopped.some((stage) => stage.state === "canceled"))) {
|
|
47
70
|
// Only when the sweep above moved a row: the runner already announced the state it
|
|
48
71
|
// left the project in, and repeating it would tell every open page the same thing
|
|
49
72
|
// twice.
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { transact } from "../../kernel/db/tx.js";
|
|
2
|
+
import { derive, satisfied } from "../../kernel/runner/graph.js";
|
|
3
|
+
import { finishStage, projectById, resetStage, setProjectPaused, stagesOf, updateProjectConfig, } from "../admission/repo.js";
|
|
4
|
+
import { clearUnfinishedAudio } from "../reruns/index.js";
|
|
5
|
+
import { providers as providerCatalog } from "../settings/model.js";
|
|
6
|
+
import { hasKey, listVoices } from "../settings/repo.js";
|
|
7
|
+
import { withProjectControl } from "./lock.js";
|
|
8
|
+
import { validateLocalProviderChanges, validateProviderChanges } from "./providers.js";
|
|
9
|
+
export const providerCheckTimeoutMs = 15_000;
|
|
10
|
+
async function providerFields(deps, changes) {
|
|
11
|
+
let timeout;
|
|
12
|
+
try {
|
|
13
|
+
return await Promise.race([
|
|
14
|
+
(async () => validateProviderChanges(changes, {
|
|
15
|
+
providers: await deps.providers(),
|
|
16
|
+
voices: listVoices(deps.db),
|
|
17
|
+
modelsFor: deps.modelsFor,
|
|
18
|
+
}))(),
|
|
19
|
+
new Promise((_resolve, reject) => {
|
|
20
|
+
timeout = setTimeout(() => reject(new Error("Provider catalog timed out")), providerCheckTimeoutMs);
|
|
21
|
+
}),
|
|
22
|
+
]);
|
|
23
|
+
}
|
|
24
|
+
finally {
|
|
25
|
+
clearTimeout(timeout);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function changed(deps, id, wake = false) {
|
|
29
|
+
deps.emit(id, { type: "project.updated", projectId: id });
|
|
30
|
+
if (wake)
|
|
31
|
+
deps.runner.tick(id);
|
|
32
|
+
else
|
|
33
|
+
deps.emit(id, {
|
|
34
|
+
type: "project.state",
|
|
35
|
+
projectId: id,
|
|
36
|
+
state: derive(stagesOf(deps.db, id), projectById(deps.db, id)?.paused),
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
export function pauseProject(deps, id) {
|
|
40
|
+
return withProjectControl(deps.db, id, async () => {
|
|
41
|
+
const project = projectById(deps.db, id);
|
|
42
|
+
if (project === undefined)
|
|
43
|
+
return { ok: false, reason: "no-project" };
|
|
44
|
+
const before = stagesOf(deps.db, id);
|
|
45
|
+
if (before.every((stage) => satisfied(stage.state)))
|
|
46
|
+
return { ok: true };
|
|
47
|
+
const running = before.filter((stage) => stage.state === "running").map((stage) => stage.id);
|
|
48
|
+
if (project.paused === true && running.length === 0 && !deps.runner.hasInflight?.(id))
|
|
49
|
+
return { ok: true };
|
|
50
|
+
transact(deps.db, () => setProjectPaused(deps.db, id, true, deps.clock.now().toISOString()));
|
|
51
|
+
deps.emit(id, { type: "project.updated", projectId: id });
|
|
52
|
+
await deps.runner.abortProject(id, "pause");
|
|
53
|
+
// Covers a failed final row write and runners that report interruption as canceled.
|
|
54
|
+
for (const stage of stagesOf(deps.db, id)) {
|
|
55
|
+
if (running.includes(stage.id) && (stage.state === "running" || stage.state === "canceled")) {
|
|
56
|
+
finishStage(deps.db, stage.id, "pending", null, deps.clock.now().toISOString());
|
|
57
|
+
deps.emit(id, { type: "stage.state", projectId: id, stage: stage.kind, state: "pending" });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
changed(deps, id);
|
|
61
|
+
return { ok: true };
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
export function resumeProject(deps, id) {
|
|
65
|
+
return withProjectControl(deps.db, id, () => {
|
|
66
|
+
const project = projectById(deps.db, id);
|
|
67
|
+
if (project === undefined)
|
|
68
|
+
return { ok: false, reason: "no-project" };
|
|
69
|
+
const stages = stagesOf(deps.db, id);
|
|
70
|
+
if (stages.some((stage) => stage.state === "running") || deps.runner.hasInflight?.(id))
|
|
71
|
+
return { ok: true };
|
|
72
|
+
if (project.paused !== true && stages.every((stage) => satisfied(stage.state)))
|
|
73
|
+
return { ok: true };
|
|
74
|
+
transact(deps.db, () => {
|
|
75
|
+
for (const stage of stages) {
|
|
76
|
+
if (stage.state === "failed" || stage.state === "canceled")
|
|
77
|
+
resetStage(deps.db, stage.id);
|
|
78
|
+
}
|
|
79
|
+
setProjectPaused(deps.db, id, false, deps.clock.now().toISOString());
|
|
80
|
+
});
|
|
81
|
+
changed(deps, id, true);
|
|
82
|
+
return { ok: true };
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
function editable(db, id, runner) {
|
|
86
|
+
const project = projectById(db, id);
|
|
87
|
+
if (project === undefined)
|
|
88
|
+
return { ok: false, reason: "no-project" };
|
|
89
|
+
const stages = stagesOf(db, id);
|
|
90
|
+
if (stages.some((stage) => stage.state === "running") || runner.hasInflight?.(id))
|
|
91
|
+
return { ok: false, reason: "running" };
|
|
92
|
+
if (project.paused !== true && derive(stages) !== "failed")
|
|
93
|
+
return { ok: false, reason: "not-editable" };
|
|
94
|
+
return { ok: true };
|
|
95
|
+
}
|
|
96
|
+
export function changeProviders(deps, id, changes) {
|
|
97
|
+
return withProjectControl(deps.db, id, async () => {
|
|
98
|
+
const allowed = editable(deps.db, id, deps.runner);
|
|
99
|
+
if (!allowed.ok)
|
|
100
|
+
return allowed;
|
|
101
|
+
let fields;
|
|
102
|
+
try {
|
|
103
|
+
fields = await providerFields(deps, changes);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
deps.log.write("warn", "project.providers", {
|
|
107
|
+
projectId: id,
|
|
108
|
+
detail: "Provider readiness or model catalog could not be loaded.",
|
|
109
|
+
});
|
|
110
|
+
return { ok: false, reason: "catalog-unavailable" };
|
|
111
|
+
}
|
|
112
|
+
if (fields.length > 0)
|
|
113
|
+
return { ok: false, reason: "invalid-providers", fields };
|
|
114
|
+
// Catalog/CLI reads await. Recheck after them before touching a project another
|
|
115
|
+
// action may have deleted, and never rebuild its saved prompts or upload choices.
|
|
116
|
+
const stillAllowed = editable(deps.db, id, deps.runner);
|
|
117
|
+
if (!stillAllowed.ok)
|
|
118
|
+
return stillAllowed;
|
|
119
|
+
const project = projectById(deps.db, id);
|
|
120
|
+
if (project === undefined)
|
|
121
|
+
return { ok: false, reason: "no-project" };
|
|
122
|
+
// Settings can change while a remote catalog is loading. Revalidate its local
|
|
123
|
+
// key and voice rows synchronously so the saved choice is usable at commit.
|
|
124
|
+
const localFields = validateLocalProviderChanges(changes, {
|
|
125
|
+
providers: providerCatalog.map((provider) => ({
|
|
126
|
+
...provider,
|
|
127
|
+
readiness: provider.auth === "cli"
|
|
128
|
+
? { kind: "cli", installed: true }
|
|
129
|
+
: { kind: "keyed", hasKey: hasKey(deps.db, provider.id) },
|
|
130
|
+
})),
|
|
131
|
+
voices: listVoices(deps.db),
|
|
132
|
+
});
|
|
133
|
+
if (localFields.length > 0)
|
|
134
|
+
return { ok: false, reason: "invalid-providers", fields: localFields };
|
|
135
|
+
const audioChanged = changes.audio !== undefined &&
|
|
136
|
+
(changes.audio.provider !== project.config.audio?.provider ||
|
|
137
|
+
changes.audio.model !== project.config.audio?.model ||
|
|
138
|
+
changes.audio.voice !== project.config.audio?.voice);
|
|
139
|
+
const orphaned = transact(deps.db, () => {
|
|
140
|
+
const files = audioChanged ? clearUnfinishedAudio(deps, id) : [];
|
|
141
|
+
updateProjectConfig(deps.db, id, {
|
|
142
|
+
...project.config,
|
|
143
|
+
...(changes.llm === undefined ? {} : { llm: changes.llm }),
|
|
144
|
+
...(changes.audio === undefined ? {} : { audio: changes.audio }),
|
|
145
|
+
...(changes.images === undefined ? {} : { images: changes.images }),
|
|
146
|
+
}, deps.clock.now().toISOString());
|
|
147
|
+
return files;
|
|
148
|
+
});
|
|
149
|
+
// The clear helper owns the row changes; files are unlinked only after commit.
|
|
150
|
+
for (const remove of orphaned)
|
|
151
|
+
remove();
|
|
152
|
+
deps.emit(id, { type: "project.updated", projectId: id });
|
|
153
|
+
return { ok: true };
|
|
154
|
+
});
|
|
155
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
// Every project mutation uses the same queue. A pause owns it until provider calls
|
|
2
|
+
// have unwound, so a second tab cannot resume or replace configuration mid-abort.
|
|
3
|
+
const queues = new WeakMap();
|
|
4
|
+
export function withProjectControl(db, projectId, work) {
|
|
5
|
+
let projects = queues.get(db);
|
|
6
|
+
if (projects === undefined) {
|
|
7
|
+
projects = new Map();
|
|
8
|
+
queues.set(db, projects);
|
|
9
|
+
}
|
|
10
|
+
const previous = projects.get(projectId);
|
|
11
|
+
const result = previous === undefined ? Promise.resolve().then(work) : previous.then(work, work);
|
|
12
|
+
projects.set(projectId, result);
|
|
13
|
+
const release = () => {
|
|
14
|
+
if (projects.get(projectId) === result)
|
|
15
|
+
projects.delete(projectId);
|
|
16
|
+
};
|
|
17
|
+
void result.then(release, release);
|
|
18
|
+
return result;
|
|
19
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
const choice = z
|
|
3
|
+
.object({
|
|
4
|
+
provider: z.string().trim().min(1).max(200),
|
|
5
|
+
model: z.string().trim().min(1).max(200),
|
|
6
|
+
})
|
|
7
|
+
.strict();
|
|
8
|
+
export const providerChangesSchema = z
|
|
9
|
+
.object({
|
|
10
|
+
llm: choice.optional(),
|
|
11
|
+
audio: choice.extend({ voice: z.string().trim().min(1).max(200) }).optional(),
|
|
12
|
+
images: choice.optional(),
|
|
13
|
+
})
|
|
14
|
+
.strict()
|
|
15
|
+
.refine((value) => Object.keys(value).length > 0, "Pick at least one provider to update.");
|
|
16
|
+
export function validateLocalProviderChanges(changes, current) {
|
|
17
|
+
const fields = [];
|
|
18
|
+
const families = { llm: "llm", audio: "tts", images: "image" };
|
|
19
|
+
for (const key of ["llm", "audio", "images"]) {
|
|
20
|
+
const picked = changes[key];
|
|
21
|
+
if (picked === undefined)
|
|
22
|
+
continue;
|
|
23
|
+
const provider = current.providers.find((one) => one.id === picked.provider && one.family === families[key]);
|
|
24
|
+
if (provider === undefined) {
|
|
25
|
+
fields.push({ field: key, message: "Choose a provider for this kind of generation." });
|
|
26
|
+
continue;
|
|
27
|
+
}
|
|
28
|
+
const usable = provider.readiness.kind === "cli" ? provider.readiness.installed : provider.readiness.hasKey;
|
|
29
|
+
if (!usable) {
|
|
30
|
+
fields.push({
|
|
31
|
+
field: key,
|
|
32
|
+
message: provider.readiness.kind === "cli"
|
|
33
|
+
? "Install and sign in to this CLI first."
|
|
34
|
+
: "Save this provider's API key in Settings first.",
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (changes.audio !== undefined &&
|
|
39
|
+
!current.voices.some((voice) => voice.provider === changes.audio?.provider && voice.voiceId === changes.audio.voice)) {
|
|
40
|
+
fields.push({ field: "audio.voice", message: "Choose a saved voice for this provider." });
|
|
41
|
+
}
|
|
42
|
+
return fields;
|
|
43
|
+
}
|
|
44
|
+
export async function validateProviderChanges(changes, current) {
|
|
45
|
+
const fields = [...validateLocalProviderChanges(changes, current)];
|
|
46
|
+
if (current.modelsFor === undefined)
|
|
47
|
+
return fields;
|
|
48
|
+
const families = { llm: "llm", audio: "tts", images: "image" };
|
|
49
|
+
for (const key of ["llm", "audio", "images"]) {
|
|
50
|
+
const picked = changes[key];
|
|
51
|
+
if (picked === undefined || fields.some((field) => field.field === key))
|
|
52
|
+
continue;
|
|
53
|
+
const models = await current.modelsFor(picked.provider, families[key]);
|
|
54
|
+
if (!models.some((model) => model.id === picked.model)) {
|
|
55
|
+
fields.push({ field: `${key}.model`, message: "Choose a model supported by this provider." });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return fields;
|
|
59
|
+
}
|
|
@@ -2,9 +2,11 @@
|
|
|
2
2
|
// the picked templates, the slot names they ask for, and the rendered text stored on the
|
|
3
3
|
// project. The bodies are read at the click, not at selection, so an edit made in between
|
|
4
4
|
// is the one that runs.
|
|
5
|
+
import { normaliseDraft } from "../admission/rules.js";
|
|
5
6
|
import { collectFields, render } from "../admission/substitute.js";
|
|
6
7
|
import { entryByName, promptByName } from "./repo.js";
|
|
7
|
-
export function pickTemplates(db,
|
|
8
|
+
export function pickTemplates(db, input) {
|
|
9
|
+
const draft = normaliseDraft(input);
|
|
8
10
|
const { sources } = draft;
|
|
9
11
|
const missing = [];
|
|
10
12
|
// Only the prompts of stages set to Generate, plus the picked
|