@gentbajko/slopify 1.5.1 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -8
- package/dist/adapters/alignment/cache.js +67 -7
- package/dist/edge/cli.js +10 -2
- package/dist/edge/docker-launch.js +77 -0
- package/dist/edge/docker-projects/activation.js +35 -0
- package/dist/edge/docker-projects/claims.js +85 -0
- package/dist/edge/docker-projects/committed.js +40 -0
- package/dist/edge/docker-projects/engine.js +384 -0
- package/dist/edge/docker-projects/install.js +246 -0
- package/dist/edge/docker-projects/recover.js +100 -0
- package/dist/edge/docker-projects/state.js +221 -0
- package/dist/edge/docker-projects/tree.js +170 -0
- package/dist/edge/docker-projects/volume.js +95 -0
- package/dist/edge/docker.js +6 -0
- package/dist/edge/http/actions.js +64 -15
- package/dist/edge/http/app.js +6 -0
- package/dist/edge/http/folder-location-schema.js +11 -0
- package/dist/edge/http/folder-location.js +65 -0
- package/dist/edge/http/open-folder.js +2 -14
- package/dist/edge/http/projects.js +2 -0
- package/dist/edge/http/revision-files.js +2 -18
- package/dist/kernel/db/migrations/0012-project-recovery.sql +17 -0
- package/dist/kernel/lock.js +20 -6
- package/dist/main.js +11 -1
- package/dist/slices/admission/rules.js +6 -0
- package/dist/slices/admission/schema.js +3 -1
- package/dist/slices/article/plain.js +16 -9
- package/dist/slices/control/revision-control.js +2 -2
- package/dist/slices/narration/pronunciation-chunks.js +105 -0
- package/dist/slices/narration/pronunciation.js +126 -0
- package/dist/slices/narration/steering.js +82 -24
- package/dist/slices/play-drafts/convert.js +3 -1
- package/dist/slices/play-drafts/schema.js +3 -1
- package/dist/slices/rebuild/admission-repo.js +7 -4
- package/dist/slices/rebuild/recipe-audio-parts.js +70 -14
- package/dist/slices/rebuild/recipe-audio.js +61 -40
- package/dist/slices/rebuild/recipe-model.js +5 -3
- package/dist/slices/rebuild/recipe-preparation.js +6 -4
- package/dist/slices/rebuild/recipe-text.js +11 -3
- package/dist/slices/rebuild/recovery-conflict.js +30 -0
- package/dist/slices/rebuild/recovery-model.js +44 -0
- package/dist/slices/rebuild/recovery-repo.js +84 -0
- package/dist/slices/rebuild/recovery-selection.js +68 -0
- package/dist/slices/rebuild/recovery.js +257 -0
- package/dist/slices/rebuild/runtime-export-inputs.js +3 -2
- package/dist/slices/rebuild/runtime-narration-publication.js +30 -0
- package/dist/slices/rebuild/runtime-narration-reuse.js +20 -2
- package/dist/slices/rebuild/runtime-plan.js +8 -2
- package/dist/slices/rebuild/runtime-publication.js +9 -4
- package/dist/slices/rebuild/service.js +44 -22
- package/dist/slices/rebuild/transition-repo.js +1 -1
- package/dist/slices/revisions/mutation-assets.js +6 -1
- package/dist/slices/revisions/mutation-request.js +4 -0
- package/dist/slices/revisions/mutations.js +10 -5
- package/dist/slices/revisions/projection.js +9 -1
- package/dist/slices/revisions/rules.js +43 -0
- package/dist/slices/revisions/schema.js +12 -0
- package/dist/slices/settings/repo.js +4 -2
- package/dist/web/assets/index-CZNiHm8P.css +1 -0
- package/dist/web/assets/index-DPgGLX5m.js +137 -0
- package/dist/web/index.html +2 -2
- package/package.json +3 -2
- package/scripts/docker-run.sh +1 -74
- package/dist/web/assets/index-CbGJtCio.js +0 -137
- package/dist/web/assets/index-DX1_chKs.css +0 -1
|
@@ -8,6 +8,9 @@ import { cancelProject } from "../../slices/cancel/index.js";
|
|
|
8
8
|
import { settleReleasedCheckpoints } from "../../slices/checkpoints/recovery.js";
|
|
9
9
|
import { pauseProject } from "../../slices/control/index.js";
|
|
10
10
|
import { revisionControlSchema, } from "../../slices/control/revision-control-schema.js";
|
|
11
|
+
import { recoverProject } from "../../slices/rebuild/recovery.js";
|
|
12
|
+
import { recoveryResultSchema } from "../../slices/rebuild/recovery-model.js";
|
|
13
|
+
import { resumable } from "../../slices/rebuild/recovery-repo.js";
|
|
11
14
|
import { adoptBaseline } from "../../slices/revisions/adopt.js";
|
|
12
15
|
import { currentRevisionId } from "../../slices/revisions/repo.js";
|
|
13
16
|
import { allowsCustomModel } from "../../slices/settings/models.js";
|
|
@@ -64,6 +67,7 @@ export function actionRoutes(deps) {
|
|
|
64
67
|
const stages = stagesOf(deps.db, projectId);
|
|
65
68
|
return {
|
|
66
69
|
revisionId: currentRevisionId(deps.db, projectId) ?? null,
|
|
70
|
+
resumable: resumable(deps.db, projectId),
|
|
67
71
|
project: {
|
|
68
72
|
...projectById(deps.db, projectId),
|
|
69
73
|
status: derive(stages, projectPaused(deps.db, projectId)),
|
|
@@ -108,6 +112,57 @@ export function actionRoutes(deps) {
|
|
|
108
112
|
},
|
|
109
113
|
});
|
|
110
114
|
};
|
|
115
|
+
const recovery = async (c, id, input, action) => {
|
|
116
|
+
if (deps.rebuild === undefined)
|
|
117
|
+
return problem(c, {
|
|
118
|
+
status: 503,
|
|
119
|
+
title: titleOf(503),
|
|
120
|
+
detail: "Recovery is unavailable. Try again after restart.",
|
|
121
|
+
});
|
|
122
|
+
const result = recoveryResultSchema.parse(await recoverProject(deps.rebuild, id, { ...input, action }));
|
|
123
|
+
if (result.ok)
|
|
124
|
+
return c.json(result, 202);
|
|
125
|
+
const status = result.reason === "no-project" || result.reason === "no-revision"
|
|
126
|
+
? 404
|
|
127
|
+
: result.reason === "invalid-selection" || result.reason === "invalid-edit"
|
|
128
|
+
? 400
|
|
129
|
+
: 409;
|
|
130
|
+
const messages = {
|
|
131
|
+
"no-project": "This project no longer exists.",
|
|
132
|
+
"no-revision": "This revision no longer exists. Reload the project.",
|
|
133
|
+
conflict: "The saved project changed. Reload it and try Resume.",
|
|
134
|
+
"idempotency-conflict": "This request ID belongs to another action. Reload the project.",
|
|
135
|
+
"invalid-edit": "This section needs attention in Edit project before rerunning.",
|
|
136
|
+
"stale-preview": "The required work changed. Try Resume to check it again.",
|
|
137
|
+
"invalid-selection": "This section has no generated work to rerun. Use Edit project to change its source.",
|
|
138
|
+
"review-required": "Changed supplied content or manual captions need Edit project or optional Advanced rebuild review.",
|
|
139
|
+
"cost-ack-required": "Use optional Advanced rebuild review for this request.",
|
|
140
|
+
readiness: "Check the provider, model, voice or source files, then try Resume.",
|
|
141
|
+
running: "Wait for this section to finish, or Pause the project before rerunning it.",
|
|
142
|
+
"accepted-job": "This section cannot be rerun yet.",
|
|
143
|
+
"control-changed": "A newer control action took precedence. Check the project before using Resume.",
|
|
144
|
+
};
|
|
145
|
+
const detail = [
|
|
146
|
+
messages[result.reason],
|
|
147
|
+
...(result.fields ?? []).map((row) => row.message),
|
|
148
|
+
...(result.intentRevisionId === undefined
|
|
149
|
+
? []
|
|
150
|
+
: ["The rerun revision was saved. Use Resume to continue it; do not rerun again."]),
|
|
151
|
+
].join(" ");
|
|
152
|
+
return problem(c, {
|
|
153
|
+
status,
|
|
154
|
+
title: titleOf(status),
|
|
155
|
+
detail,
|
|
156
|
+
extensions: {
|
|
157
|
+
reason: result.reason,
|
|
158
|
+
fields: result.fields ?? [],
|
|
159
|
+
currentRevisionId: currentRevisionId(deps.db, id) ?? null,
|
|
160
|
+
...(result.intentRevisionId === undefined
|
|
161
|
+
? {}
|
|
162
|
+
: { intentRevisionId: result.intentRevisionId }),
|
|
163
|
+
},
|
|
164
|
+
});
|
|
165
|
+
};
|
|
111
166
|
const controlInput = async (c, id) => {
|
|
112
167
|
if (deps.catalogue !== undefined)
|
|
113
168
|
adoptBaseline(deps, id);
|
|
@@ -135,13 +190,7 @@ export function actionRoutes(deps) {
|
|
|
135
190
|
? input
|
|
136
191
|
: controlled(c, id, await pauseProject(control, id, input));
|
|
137
192
|
})
|
|
138
|
-
.post("/:id/resume", zValidator("param", idParam, onInvalid), (c) => {
|
|
139
|
-
const { id } = c.req.valid("param");
|
|
140
|
-
return controlled(c, id, {
|
|
141
|
-
ok: false,
|
|
142
|
-
reason: projectById(deps.db, id) === undefined ? "no-project" : "rebuild-required",
|
|
143
|
-
});
|
|
144
|
-
})
|
|
193
|
+
.post("/:id/resume", zValidator("param", idParam, onInvalid), zValidator("json", revisionControlSchema, onInvalid), (c) => recovery(c, c.req.valid("param").id, c.req.valid("json"), { kind: "resume" }))
|
|
145
194
|
.patch("/:id/providers", zValidator("param", idParam, onInvalid), revisionRequired)
|
|
146
195
|
.post("/:id/cancel", zValidator("param", idParam, onInvalid), async (c) => {
|
|
147
196
|
const { id } = c.req.valid("param");
|
|
@@ -153,14 +202,14 @@ export function actionRoutes(deps) {
|
|
|
153
202
|
return controlled(c, id, result);
|
|
154
203
|
return c.json({ ...view(id), canceled: result.canceled });
|
|
155
204
|
})
|
|
156
|
-
.post("/:id/stages/:kind/retry", zValidator("param", stageParam, onInvalid), (c) => {
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
205
|
+
.post("/:id/stages/:kind/retry", zValidator("param", stageParam, onInvalid), zValidator("json", revisionControlSchema, onInvalid), (c) => recovery(c, c.req.valid("param").id, c.req.valid("json"), {
|
|
206
|
+
kind: "retry",
|
|
207
|
+
stage: c.req.valid("param").kind,
|
|
208
|
+
}))
|
|
209
|
+
.post("/:id/stages/:kind/rerun", zValidator("param", stageParam, onInvalid), zValidator("json", revisionControlSchema, onInvalid), (c) => recovery(c, c.req.valid("param").id, c.req.valid("json"), {
|
|
210
|
+
kind: "rerun",
|
|
211
|
+
stage: c.req.valid("param").kind,
|
|
212
|
+
}))
|
|
164
213
|
.put("/:id/article", zValidator("param", idParam, onInvalid), revisionRequired)
|
|
165
214
|
.delete("/:id/images/:outputId", zValidator("param", imageParam, onInvalid), revisionRequired)
|
|
166
215
|
.post("/:id/images/:outputId/regenerate", zValidator("param", imageParam, onInvalid), revisionRequired);
|
package/dist/edge/http/app.js
CHANGED
|
@@ -76,6 +76,12 @@ export function createApp(deps) {
|
|
|
76
76
|
.use("/api/*", async (c, next) => {
|
|
77
77
|
if (["GET", "HEAD", "OPTIONS"].includes(c.req.method))
|
|
78
78
|
return next();
|
|
79
|
+
if (deps.installationPending?.() && new URL(c.req.url).pathname !== "/api/update/activate")
|
|
80
|
+
return problem(c, {
|
|
81
|
+
status: 503,
|
|
82
|
+
title: titleOf(503),
|
|
83
|
+
detail: "Docker installation is being verified. Wait for the launcher to finish.",
|
|
84
|
+
});
|
|
79
85
|
const releaseRequest = deps.mutations?.begin();
|
|
80
86
|
if (deps.mutations !== undefined && releaseRequest === undefined)
|
|
81
87
|
return problem(c, {
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export const folderReplySchema = z.discriminatedUnion("opened", [
|
|
3
|
+
z.object({ opened: z.literal(true) }).strict(),
|
|
4
|
+
z
|
|
5
|
+
.object({
|
|
6
|
+
opened: z.literal(false),
|
|
7
|
+
location: z.literal("docker-host"),
|
|
8
|
+
path: z.string().min(1),
|
|
9
|
+
})
|
|
10
|
+
.strict(),
|
|
11
|
+
]);
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { lstat } from "node:fs/promises";
|
|
2
|
+
import { dirname, isAbsolute, join, relative, sep } from "node:path";
|
|
3
|
+
import { folderReplySchema } from "./folder-location-schema.js";
|
|
4
|
+
import { problem, titleOf } from "./problem.js";
|
|
5
|
+
export async function replyForFolder(c, deps, projectId, file) {
|
|
6
|
+
c.header("Cache-Control", "no-store");
|
|
7
|
+
if (deps.folderLocation?.container) {
|
|
8
|
+
const host = deps.folderLocation.hostProjects;
|
|
9
|
+
if (!host)
|
|
10
|
+
return problem(c, {
|
|
11
|
+
status: 503,
|
|
12
|
+
title: titleOf(503),
|
|
13
|
+
detail: "This Docker installation has no verified host project folder. Downloads remain available. Rerun the managed Docker launcher to set up host files.",
|
|
14
|
+
});
|
|
15
|
+
try {
|
|
16
|
+
const r = relative(deps.paths.projects, file);
|
|
17
|
+
if (!r || r === ".." || r.startsWith(`..${sep}`) || isAbsolute(r))
|
|
18
|
+
throw new Error("Outside projects");
|
|
19
|
+
let path = deps.paths.projects;
|
|
20
|
+
const root = await lstat(path);
|
|
21
|
+
if (!root.isDirectory() || root.isSymbolicLink())
|
|
22
|
+
throw new Error("Invalid project root");
|
|
23
|
+
const parts = r.split(sep);
|
|
24
|
+
for (const [i, part] of parts.entries()) {
|
|
25
|
+
path = join(path, part);
|
|
26
|
+
const s = await lstat(path);
|
|
27
|
+
if (s.isSymbolicLink() || (i === parts.length - 1 ? !s.isFile() : !s.isDirectory()))
|
|
28
|
+
throw new Error("Unavailable output");
|
|
29
|
+
}
|
|
30
|
+
return c.json(folderReplySchema.parse({
|
|
31
|
+
opened: false,
|
|
32
|
+
location: "docker-host",
|
|
33
|
+
path: join(host, dirname(r)),
|
|
34
|
+
}));
|
|
35
|
+
}
|
|
36
|
+
catch {
|
|
37
|
+
deps.log.write("warn", "project.locate-folder", {
|
|
38
|
+
projectId,
|
|
39
|
+
detail: "The verified output folder is no longer available.",
|
|
40
|
+
});
|
|
41
|
+
return problem(c, {
|
|
42
|
+
status: 404,
|
|
43
|
+
title: titleOf(404),
|
|
44
|
+
detail: "This saved output folder is no longer available.",
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
try {
|
|
49
|
+
if (!deps.openFolder)
|
|
50
|
+
throw new Error("No folder opener configured");
|
|
51
|
+
await deps.openFolder(dirname(file));
|
|
52
|
+
return c.json(folderReplySchema.parse({ opened: true }));
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
deps.log.write("warn", "project.open-folder", {
|
|
56
|
+
projectId,
|
|
57
|
+
detail: "The file manager could not be opened for a saved file.",
|
|
58
|
+
});
|
|
59
|
+
return problem(c, {
|
|
60
|
+
status: 503,
|
|
61
|
+
title: titleOf(503),
|
|
62
|
+
detail: "Could not open the file manager on the machine running Slopify. Make sure a desktop session is available.",
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
|
-
import { dirname } from "node:path";
|
|
2
1
|
import { zValidator } from "@hono/zod-validator";
|
|
3
2
|
import { Hono } from "hono";
|
|
4
3
|
import { z } from "zod";
|
|
5
4
|
import { assetOf, findDownload } from "../../slices/storage/downloads.js";
|
|
6
5
|
import { outputsOf } from "../../slices/storage/repo.js";
|
|
6
|
+
import { replyForFolder } from "./folder-location.js";
|
|
7
7
|
import { onInvalid, problem, titleOf } from "./problem.js";
|
|
8
8
|
export function openFolderRoutes(deps) {
|
|
9
9
|
return new Hono().post("/:id/open-folder", zValidator("param", z.object({
|
|
@@ -44,18 +44,6 @@ export function openFolderRoutes(deps) {
|
|
|
44
44
|
detail: "No saved file was found for this output. Re-run the stage if the file was removed.",
|
|
45
45
|
});
|
|
46
46
|
}
|
|
47
|
-
|
|
48
|
-
if (deps.openFolder === undefined)
|
|
49
|
-
throw new Error("No folder opener configured");
|
|
50
|
-
await deps.openFolder(dirname(download.download.path));
|
|
51
|
-
return c.json({ opened: true });
|
|
52
|
-
}
|
|
53
|
-
catch {
|
|
54
|
-
return problem(c, {
|
|
55
|
-
status: 503,
|
|
56
|
-
title: titleOf(503),
|
|
57
|
-
detail: "Could not open the file manager on the machine running Slopify. Make sure a desktop session is available.",
|
|
58
|
-
});
|
|
59
|
-
}
|
|
47
|
+
return replyForFolder(c, deps, id, download.download.path);
|
|
60
48
|
});
|
|
61
49
|
}
|
|
@@ -9,6 +9,7 @@ import { startRun } from "../../slices/admission/start.js";
|
|
|
9
9
|
import { withProjectControl } from "../../slices/control/lock.js";
|
|
10
10
|
import { resolveFont } from "../../slices/fonts/index.js";
|
|
11
11
|
import { pickTemplates, renderPicked } from "../../slices/library/slots.js";
|
|
12
|
+
import { resumable } from "../../slices/rebuild/recovery-repo.js";
|
|
12
13
|
import { adoptBaseline } from "../../slices/revisions/adopt.js";
|
|
13
14
|
import { currentRevisionId } from "../../slices/revisions/repo.js";
|
|
14
15
|
import { deleteProject } from "../../slices/storage/delete-project.js";
|
|
@@ -142,6 +143,7 @@ export function projectRoutes(deps) {
|
|
|
142
143
|
adoptBaseline(deps, project.id);
|
|
143
144
|
return c.json({
|
|
144
145
|
revisionId: currentRevisionId(deps.db, project.id) ?? null,
|
|
146
|
+
resumable: resumable(deps.db, project.id),
|
|
145
147
|
project: summarise(project),
|
|
146
148
|
stages: stagesOf(deps.db, project.id),
|
|
147
149
|
outputs: outputsOf(deps.db, project.id),
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { createReadStream } from "node:fs";
|
|
2
|
-
import { dirname } from "node:path";
|
|
3
2
|
import { Readable } from "node:stream";
|
|
4
3
|
import { zValidator } from "@hono/zod-validator";
|
|
5
4
|
import { Hono } from "hono";
|
|
6
5
|
import { z } from "zod";
|
|
7
6
|
import { findRevisionDownload, revisionImagesZip } from "../../slices/revisions/downloads.js";
|
|
7
|
+
import { replyForFolder } from "./folder-location.js";
|
|
8
8
|
import { onInvalid, problem, titleOf } from "./problem.js";
|
|
9
9
|
const id = z
|
|
10
10
|
.string()
|
|
@@ -53,23 +53,7 @@ export function revisionFolderRoutes(deps) {
|
|
|
53
53
|
const result = findRevisionDownload(deps, projectId, revisionId, recordId);
|
|
54
54
|
if (!result.ok)
|
|
55
55
|
return unavailable(c, result.reason);
|
|
56
|
-
|
|
57
|
-
if (deps.openFolder === undefined)
|
|
58
|
-
throw new Error("No folder opener configured");
|
|
59
|
-
await deps.openFolder(dirname(result.download.path));
|
|
60
|
-
return c.json({ opened: true });
|
|
61
|
-
}
|
|
62
|
-
catch {
|
|
63
|
-
deps.log.write("warn", "project.open-folder", {
|
|
64
|
-
projectId,
|
|
65
|
-
detail: "The file manager could not be opened for a retained file.",
|
|
66
|
-
});
|
|
67
|
-
return problem(c, {
|
|
68
|
-
status: 503,
|
|
69
|
-
title: titleOf(503),
|
|
70
|
-
detail: "Could not open the file manager on the machine running Slopify. Make sure a desktop session is available.",
|
|
71
|
-
});
|
|
72
|
-
}
|
|
56
|
+
return replyForFolder(c, deps, projectId, result.download.path);
|
|
73
57
|
});
|
|
74
58
|
}
|
|
75
59
|
function unavailable(c, reason) {
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
ALTER TABLE provider_keys ADD COLUMN credential_generation TEXT NOT NULL DEFAULT 'legacy';
|
|
2
|
+
|
|
3
|
+
CREATE TABLE project_recovery_requests (
|
|
4
|
+
project_id TEXT NOT NULL,
|
|
5
|
+
idempotency_key TEXT NOT NULL,
|
|
6
|
+
request_hash TEXT NOT NULL,
|
|
7
|
+
base_revision_id TEXT NOT NULL,
|
|
8
|
+
authority_stamp TEXT NOT NULL,
|
|
9
|
+
edit_json TEXT CHECK(edit_json IS NULL OR json_valid(edit_json)),
|
|
10
|
+
intent_revision_id TEXT,
|
|
11
|
+
response_json TEXT CHECK(response_json IS NULL OR json_valid(response_json)),
|
|
12
|
+
PRIMARY KEY(project_id,idempotency_key),
|
|
13
|
+
FOREIGN KEY(project_id,base_revision_id)
|
|
14
|
+
REFERENCES project_revisions(project_id,id) ON DELETE CASCADE,
|
|
15
|
+
FOREIGN KEY(project_id,intent_revision_id)
|
|
16
|
+
REFERENCES project_revisions(project_id,id) ON DELETE CASCADE
|
|
17
|
+
);
|
package/dist/kernel/lock.js
CHANGED
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { readFileSync, unlinkSync, writeFileSync } from "node:fs";
|
|
2
|
+
// Seconds since the epoch at which this process started, stable for its whole life.
|
|
3
|
+
const startedAt = Math.round(Date.now() / 1000 - process.uptime());
|
|
2
4
|
export function acquireInstanceLock(lockPath) {
|
|
3
5
|
const pid = process.pid;
|
|
4
6
|
if (!claim(lockPath, pid)) {
|
|
5
7
|
const holder = holderOf(lockPath);
|
|
6
|
-
if (holder !== undefined &&
|
|
7
|
-
throw new Error(`another Slopify instance is already running on this data directory (pid ${holder}). ` +
|
|
8
|
+
if (holder !== undefined && holds(holder, pid)) {
|
|
9
|
+
throw new Error(`another Slopify instance is already running on this data directory (pid ${holder.pid}). ` +
|
|
8
10
|
`Stop it, or delete ${lockPath} if it is stale.`);
|
|
9
11
|
}
|
|
10
12
|
// The recorded process is gone, or the file says nothing usable. Reclaiming it here
|
|
@@ -16,7 +18,8 @@ export function acquireInstanceLock(lockPath) {
|
|
|
16
18
|
}
|
|
17
19
|
return {
|
|
18
20
|
release: () => {
|
|
19
|
-
|
|
21
|
+
const holder = holderOf(lockPath);
|
|
22
|
+
if (holder?.pid === pid && holder.startedAt === startedAt) {
|
|
20
23
|
discard(lockPath);
|
|
21
24
|
}
|
|
22
25
|
},
|
|
@@ -24,7 +27,7 @@ export function acquireInstanceLock(lockPath) {
|
|
|
24
27
|
}
|
|
25
28
|
function claim(lockPath, pid) {
|
|
26
29
|
try {
|
|
27
|
-
writeFileSync(lockPath, `${pid}\n`, { flag: "wx", mode: 0o600 });
|
|
30
|
+
writeFileSync(lockPath, `${pid} ${startedAt}\n`, { flag: "wx", mode: 0o600 });
|
|
28
31
|
return true;
|
|
29
32
|
}
|
|
30
33
|
catch (error) {
|
|
@@ -42,8 +45,19 @@ function holderOf(lockPath) {
|
|
|
42
45
|
catch {
|
|
43
46
|
return undefined;
|
|
44
47
|
}
|
|
45
|
-
const
|
|
46
|
-
|
|
48
|
+
const [pidText = "", startText] = contents.trim().split(/\s+/);
|
|
49
|
+
const pid = Number(pidText);
|
|
50
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
51
|
+
return undefined;
|
|
52
|
+
const started = startText === undefined ? undefined : Number(startText);
|
|
53
|
+
return { pid, startedAt: Number.isInteger(started) ? started : undefined };
|
|
54
|
+
}
|
|
55
|
+
// A lock naming this very pid is live only if this process wrote it; one from an earlier
|
|
56
|
+
// process that reused the pid (a restarted container) is stale.
|
|
57
|
+
function holds(holder, pid) {
|
|
58
|
+
if (holder.pid === pid)
|
|
59
|
+
return holder.startedAt === startedAt;
|
|
60
|
+
return isAlive(holder.pid);
|
|
47
61
|
}
|
|
48
62
|
function discard(lockPath) {
|
|
49
63
|
try {
|
package/dist/main.js
CHANGED
|
@@ -10,6 +10,7 @@ import { createHostCliClient } from "./adapters/host-cli/index.js";
|
|
|
10
10
|
import { nodeRunCli } from "./adapters/llm/run-cli.js";
|
|
11
11
|
import { curateRegistry } from "./catalog/registry.js";
|
|
12
12
|
import { createCatalogueStore } from "./catalog/store.js";
|
|
13
|
+
import { dockerActivationCommitted, dockerFolderConfiguration, } from "./edge/docker-projects/activation.js";
|
|
13
14
|
import { createHub } from "./edge/events/hub.js";
|
|
14
15
|
import { currentProjectEvent } from "./edge/events/visibility.js";
|
|
15
16
|
import { createApp } from "./edge/http/app.js";
|
|
@@ -112,6 +113,11 @@ export async function boot(config) {
|
|
|
112
113
|
const clock = systemClock;
|
|
113
114
|
const ids = ulidIds;
|
|
114
115
|
const paths = layout(config.dataDir);
|
|
116
|
+
const dockerState = process.env.SLOPIFY_DOCKER_INSTALL_STATE;
|
|
117
|
+
if (dockerState !== undefined &&
|
|
118
|
+
(process.env.SLOPIFY_CONTAINER !== "1" ||
|
|
119
|
+
dockerState !== "/opt/slopify-install/activation.json"))
|
|
120
|
+
throw new Error("Invalid managed Docker activation configuration.");
|
|
115
121
|
const candidateToken = process.env.SLOPIFY_UPDATE_TOKEN ?? "";
|
|
116
122
|
const pendingActivation = isUpdateToken(candidateToken) && process.env.SLOPIFY_UPDATE_PENDING === "1";
|
|
117
123
|
ensureDirs(paths, { mode: 0o700 });
|
|
@@ -203,7 +209,9 @@ export async function boot(config) {
|
|
|
203
209
|
candidate: {
|
|
204
210
|
token: candidateToken,
|
|
205
211
|
pending: pendingActivation,
|
|
206
|
-
committed: () =>
|
|
212
|
+
committed: () => dockerState === undefined
|
|
213
|
+
? updateCommitted(paths.dataDir, version, candidateToken)
|
|
214
|
+
: dockerActivationCommitted(dockerState, candidateToken),
|
|
207
215
|
settle: () => {
|
|
208
216
|
try {
|
|
209
217
|
const settled = reconcileStorage(updateDb, paths);
|
|
@@ -313,6 +321,8 @@ export async function boot(config) {
|
|
|
313
321
|
schedules: scheduleDeps,
|
|
314
322
|
...(rebuild.measureAudio === undefined ? {} : { measureAudio: rebuild.measureAudio }),
|
|
315
323
|
openFolder,
|
|
324
|
+
folderLocation: await dockerFolderConfiguration(process.env, paths.projects),
|
|
325
|
+
...(dockerState === undefined ? {} : { installationPending: () => updater.locked() }),
|
|
316
326
|
db,
|
|
317
327
|
paths,
|
|
318
328
|
hub,
|
|
@@ -232,3 +232,9 @@ export function narrationPreparationFields(draft) {
|
|
|
232
232
|
function chosen(choice) {
|
|
233
233
|
return choice !== undefined && choice.provider.trim() !== "" && choice.model.trim() !== "";
|
|
234
234
|
}
|
|
235
|
+
export function usesPronunciationGlossary(draft) {
|
|
236
|
+
return (draft.sources.audio === "generate" &&
|
|
237
|
+
draft.audio?.usePronunciationGlossary === true &&
|
|
238
|
+
draft.audio.provider === "inworld" &&
|
|
239
|
+
(draft.audio.model === "inworld-tts-2" || draft.audio.model === "inworld-tts-2-flash"));
|
|
240
|
+
}
|
|
@@ -32,7 +32,9 @@ export const runDraftSchema = z.object({
|
|
|
32
32
|
video: z.enum(stageSources),
|
|
33
33
|
}),
|
|
34
34
|
llm: providerChoice.optional(),
|
|
35
|
-
audio: providerChoice
|
|
35
|
+
audio: providerChoice
|
|
36
|
+
.extend({ voice: z.string(), usePronunciationGlossary: z.boolean().optional() })
|
|
37
|
+
.optional(),
|
|
36
38
|
images: providerChoice.optional(),
|
|
37
39
|
articlePrompt: z.string().optional(),
|
|
38
40
|
narrationPrompt: z.string().optional(),
|
|
@@ -1,14 +1,21 @@
|
|
|
1
1
|
import { remark } from "remark";
|
|
2
2
|
import remarkGfm from "remark-gfm";
|
|
3
3
|
import stripMarkdown from "strip-markdown";
|
|
4
|
-
// The article is stored twice, as the markdown the model wrote and as the plain text the
|
|
5
|
-
// narration is read from, so no TTS voice ever says a hash or a bracket. remark parses and
|
|
6
|
-
// strip-markdown drops the formatting; the GFM extension is what makes a table a table rather
|
|
7
|
-
// than a paragraph full of pipes, and what takes a footnote marker out of the middle of a
|
|
8
|
-
// sentence.
|
|
9
4
|
export function plainText(markdown) {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const
|
|
13
|
-
|
|
5
|
+
const processor = remark().use(remarkGfm).use(stripMarkdown);
|
|
6
|
+
const tree = processor.runSync(processor.parse(markdown));
|
|
7
|
+
const paragraphs = tree.children
|
|
8
|
+
.map((node) => {
|
|
9
|
+
if (node.type !== "paragraph")
|
|
10
|
+
throw new Error("Expected stripped prose paragraphs.");
|
|
11
|
+
return node.children
|
|
12
|
+
.map((child) => {
|
|
13
|
+
if (child.type !== "text")
|
|
14
|
+
throw new Error("Expected stripped prose text.");
|
|
15
|
+
return child.value;
|
|
16
|
+
})
|
|
17
|
+
.join("");
|
|
18
|
+
})
|
|
19
|
+
.filter((text) => text.trim() !== "");
|
|
20
|
+
return paragraphs.length === 0 ? "" : paragraphs.join("\n\n") + "\n";
|
|
14
21
|
}
|
|
@@ -39,8 +39,8 @@ export function checkRevisionControl(deps, projectId, operation, input, response
|
|
|
39
39
|
return { ok: true, identity: null, response: responseSchema.parse(response) };
|
|
40
40
|
}
|
|
41
41
|
if (db
|
|
42
|
-
.prepare("SELECT 1 FROM revision_mutations WHERE project_id=? AND idempotency_key=? UNION ALL SELECT 1 FROM rebuild_admissions WHERE project_id=? AND idempotency_key=?")
|
|
43
|
-
.get(projectId, identity.idempotencyKey, projectId, identity.idempotencyKey) !== undefined)
|
|
42
|
+
.prepare("SELECT 1 FROM revision_mutations WHERE project_id=? AND idempotency_key=? UNION ALL SELECT 1 FROM rebuild_admissions WHERE project_id=? AND idempotency_key=? UNION ALL SELECT 1 FROM project_recovery_requests WHERE project_id=? AND idempotency_key=?")
|
|
43
|
+
.get(projectId, identity.idempotencyKey, projectId, identity.idempotencyKey, projectId, identity.idempotencyKey) !== undefined)
|
|
44
44
|
return { ok: false, reason: "idempotency-conflict" };
|
|
45
45
|
if (head !== identity.baseRevisionId)
|
|
46
46
|
return { ok: false, reason: "conflict" };
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { fingerprint } from "../../kernel/runner/work.js";
|
|
2
|
+
import { chunkNarration, defaultChunkCharacters, defaultChunkWords, } from "./chunk.js";
|
|
3
|
+
import { pronunciationMatches } from "./pronunciation.js";
|
|
4
|
+
export function pronunciationChunks(source, chunking, entries, overriddenKeys = new Set(), sources = {}) {
|
|
5
|
+
const occurrences = new Map();
|
|
6
|
+
const usedKeys = new Set();
|
|
7
|
+
const nextKey = (text) => {
|
|
8
|
+
const hash = fingerprint(text).slice(0, 20);
|
|
9
|
+
let occurrence = (occurrences.get(hash) ?? 0) + 1;
|
|
10
|
+
while (usedKeys.has(`audio:body:${hash}-${occurrence}`))
|
|
11
|
+
occurrence++;
|
|
12
|
+
occurrences.set(hash, occurrence);
|
|
13
|
+
const key = `audio:body:${hash}-${occurrence}`;
|
|
14
|
+
usedKeys.add(key);
|
|
15
|
+
return key;
|
|
16
|
+
};
|
|
17
|
+
let offset = 0;
|
|
18
|
+
const chunks = chunkNarration(source, chunking).map((text) => {
|
|
19
|
+
const start = source.indexOf(text, offset);
|
|
20
|
+
if (start < 0)
|
|
21
|
+
throw new Error("Narration chunk is missing from its source.");
|
|
22
|
+
offset = start + text.length;
|
|
23
|
+
return { key: nextKey(text), text, start, end: offset };
|
|
24
|
+
});
|
|
25
|
+
const bodyFingerprint = fingerprint(source);
|
|
26
|
+
const chunkingFingerprint = fingerprint([
|
|
27
|
+
chunking.mode,
|
|
28
|
+
chunking.mode === "words"
|
|
29
|
+
? (chunking.words ?? defaultChunkWords)
|
|
30
|
+
: chunking.mode === "characters"
|
|
31
|
+
? (chunking.characters ?? defaultChunkCharacters)
|
|
32
|
+
: null,
|
|
33
|
+
]);
|
|
34
|
+
const starts = new Map(chunks.map((chunk, index) => [chunk.start, index]));
|
|
35
|
+
const ends = new Map(chunks.map((chunk, index) => [chunk.end, index]));
|
|
36
|
+
const pinned = new Map();
|
|
37
|
+
let lastPinned = -1;
|
|
38
|
+
for (const [key, saved] of Object.entries(sources).sort((a, b) => a[1].start - b[1].start)) {
|
|
39
|
+
const first = starts.get(saved.start);
|
|
40
|
+
const last = ends.get(saved.start + saved.text.length);
|
|
41
|
+
if (saved.bodyFingerprint !== bodyFingerprint ||
|
|
42
|
+
saved.chunkingFingerprint !== chunkingFingerprint ||
|
|
43
|
+
source.slice(saved.start, saved.start + saved.text.length) !== saved.text ||
|
|
44
|
+
!new RegExp(`^audio:body:${fingerprint(saved.text).slice(0, 20)}-[1-9][0-9]*$`, "u").test(key) ||
|
|
45
|
+
first === undefined ||
|
|
46
|
+
last === undefined ||
|
|
47
|
+
first <= lastPinned ||
|
|
48
|
+
last <= first ||
|
|
49
|
+
chunks.slice(first, last + 1).some((chunk) => overriddenKeys.has(chunk.key)))
|
|
50
|
+
continue;
|
|
51
|
+
pinned.set(first, { key, source: saved, last });
|
|
52
|
+
usedKeys.add(key);
|
|
53
|
+
lastPinned = last;
|
|
54
|
+
}
|
|
55
|
+
const blocked = [
|
|
56
|
+
...chunks.filter((chunk) => overriddenKeys.has(chunk.key)),
|
|
57
|
+
...[...pinned.values()].map((row) => ({
|
|
58
|
+
start: row.source.start,
|
|
59
|
+
end: row.source.start + row.source.text.length,
|
|
60
|
+
})),
|
|
61
|
+
].sort((a, b) => a.start - b.start);
|
|
62
|
+
let blockedIndex = 0;
|
|
63
|
+
const matches = pronunciationMatches(source, entries).filter((match) => {
|
|
64
|
+
while ((blocked[blockedIndex]?.end ?? Infinity) <= match.start)
|
|
65
|
+
blockedIndex++;
|
|
66
|
+
return (blocked[blockedIndex]?.start ?? Infinity) >= match.end;
|
|
67
|
+
});
|
|
68
|
+
const result = [];
|
|
69
|
+
let from = 0;
|
|
70
|
+
let matchIndex = 0;
|
|
71
|
+
for (let index = 0; index < chunks.length; index++) {
|
|
72
|
+
const sticky = pinned.get(index);
|
|
73
|
+
if (sticky !== undefined) {
|
|
74
|
+
result.push({ key: sticky.key, text: sticky.source.text, source: sticky.source });
|
|
75
|
+
index = sticky.last;
|
|
76
|
+
from = index + 1;
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
79
|
+
const chunk = chunks[index];
|
|
80
|
+
if (chunk === undefined)
|
|
81
|
+
throw new Error("Narration chunk disappeared.");
|
|
82
|
+
while ((matches[matchIndex]?.end ?? Infinity) <= chunk.end)
|
|
83
|
+
matchIndex++;
|
|
84
|
+
const match = matches[matchIndex];
|
|
85
|
+
const next = chunks[index + 1];
|
|
86
|
+
if (next !== undefined &&
|
|
87
|
+
match !== undefined &&
|
|
88
|
+
match.start < chunk.end &&
|
|
89
|
+
next.start < match.end)
|
|
90
|
+
continue;
|
|
91
|
+
const first = chunks[from];
|
|
92
|
+
if (first === undefined)
|
|
93
|
+
throw new Error("Narration group has no first chunk.");
|
|
94
|
+
const text = source.slice(first.start, chunk.end);
|
|
95
|
+
result.push(from === index
|
|
96
|
+
? { key: chunk.key, text }
|
|
97
|
+
: {
|
|
98
|
+
key: nextKey(text),
|
|
99
|
+
text,
|
|
100
|
+
source: { text, start: first.start, bodyFingerprint, chunkingFingerprint },
|
|
101
|
+
});
|
|
102
|
+
from = index + 1;
|
|
103
|
+
}
|
|
104
|
+
return result;
|
|
105
|
+
}
|