@gentbajko/slopify 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapter-registry.js +11 -12
- package/dist/adapters/image/bytes.js +1 -1
- package/dist/adapters/image/fal.js +28 -41
- package/dist/adapters/image/openai.js +19 -24
- package/dist/adapters/image/replicate.js +33 -43
- package/dist/adapters/llm/claude-code.js +16 -19
- package/dist/adapters/llm/codex.js +14 -15
- package/dist/adapters/llm/openrouter.js +13 -16
- package/dist/adapters/llm/run-cli.js +9 -11
- package/dist/adapters/llm/sse-lines.js +1 -2
- package/dist/adapters/retry-after.js +6 -6
- package/dist/adapters/tts/cartesia.js +5 -5
- package/dist/adapters/tts/elevenlabs.js +14 -19
- package/dist/adapters/tts/openai.js +9 -10
- package/dist/edge/cli.js +0 -0
- package/dist/edge/http/actions.js +8 -9
- package/dist/edge/http/app.js +3 -3
- package/dist/edge/http/entries.js +1 -1
- package/dist/edge/http/projects.js +7 -7
- package/dist/edge/http/prompts.js +6 -6
- package/dist/edge/http/providers.js +4 -4
- package/dist/edge/http/settings.js +2 -2
- package/dist/edge/http/staging.js +1 -1
- package/dist/edge/http/telemetry.js +6 -7
- package/dist/edge/http/usage.js +5 -5
- package/dist/kernel/config/index.js +1 -1
- package/dist/kernel/db/tx.js +9 -12
- package/dist/kernel/log.js +5 -5
- package/dist/kernel/pipeline.js +7 -9
- package/dist/kernel/ports/model.js +3 -4
- package/dist/kernel/runner/attempt-repo.js +5 -6
- package/dist/kernel/runner/attempt.js +16 -23
- package/dist/kernel/runner/graph.js +18 -29
- package/dist/kernel/runner/index.js +27 -33
- package/dist/kernel/runner/piece-repo.js +9 -11
- package/dist/kernel/runner/providers.js +5 -7
- package/dist/kernel/version.js +3 -3
- package/dist/main.js +10 -11
- package/dist/slices/admission/model.js +2 -2
- package/dist/slices/admission/repo.js +6 -7
- package/dist/slices/admission/rules.js +12 -15
- package/dist/slices/admission/start.js +7 -8
- package/dist/slices/admission/substitute.js +11 -11
- package/dist/slices/article/continuation.js +9 -9
- package/dist/slices/article/plain.js +5 -5
- package/dist/slices/article/run.js +25 -25
- package/dist/slices/article/split.js +1 -1
- package/dist/slices/article/store.js +5 -5
- package/dist/slices/cancel/index.js +8 -8
- package/dist/slices/images/run.js +32 -34
- package/dist/slices/library/lint.js +3 -3
- package/dist/slices/library/model.js +7 -7
- package/dist/slices/library/repo.js +3 -3
- package/dist/slices/library/save.js +7 -7
- package/dist/slices/library/slots.js +5 -5
- package/dist/slices/narration/chunk.js +12 -12
- package/dist/slices/narration/concat.js +12 -13
- package/dist/slices/narration/run.js +46 -49
- package/dist/slices/reruns/cascade.js +10 -11
- package/dist/slices/reruns/index.js +36 -37
- package/dist/slices/research/planner.js +4 -5
- package/dist/slices/research/run.js +22 -23
- package/dist/slices/research/synthesis.js +5 -5
- package/dist/slices/settings/cli-status.js +9 -10
- package/dist/slices/settings/keys.js +8 -9
- package/dist/slices/settings/model.js +5 -5
- package/dist/slices/settings/readiness.js +3 -4
- package/dist/slices/settings/repo.js +1 -1
- package/dist/slices/settings/voices.js +3 -4
- package/dist/slices/storage/asset-name.js +2 -2
- package/dist/slices/storage/delete-project.js +3 -5
- package/dist/slices/storage/downloads.js +4 -4
- package/dist/slices/storage/layout.js +5 -6
- package/dist/slices/storage/model.js +1 -1
- package/dist/slices/storage/reconcile.js +6 -7
- package/dist/slices/storage/repo.js +1 -1
- package/dist/slices/storage/staging.js +12 -15
- package/dist/slices/telemetry/collector-client.js +2 -2
- package/dist/slices/telemetry/flush.js +8 -8
- package/dist/slices/telemetry/machine.js +6 -7
- package/dist/slices/telemetry/model.js +11 -13
- package/dist/slices/telemetry/record.js +7 -7
- package/dist/slices/telemetry/repo.js +4 -4
- package/dist/slices/telemetry/usage.js +5 -5
- package/dist/slices/thumbnail/by-llm.js +2 -2
- package/dist/slices/thumbnail/run.js +30 -32
- package/dist/slices/video/ffmpeg.js +6 -6
- package/dist/slices/video/plan.js +9 -9
- package/dist/slices/video/run.js +19 -20
- package/dist/web/assets/index-77hb2Mr0.js +81 -0
- package/dist/web/index.html +1 -1
- package/package.json +1 -1
- package/dist/web/assets/index-D_sWbKQi.js +0 -81
|
@@ -11,14 +11,13 @@ export function telemetryRoutes(deps) {
|
|
|
11
11
|
appVersion: deps.version,
|
|
12
12
|
};
|
|
13
13
|
return (new Hono()
|
|
14
|
-
// What the SPA asks before it shows the first-run notice
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
// with the number it is true of.
|
|
14
|
+
// What the SPA asks before it shows the first-run notice. The app version comes with it
|
|
15
|
+
// because the notice names the version that goes out in every report, and the promise has
|
|
16
|
+
// to be made with the number it is true of.
|
|
18
17
|
.get("/notice", (c) => c.json({ seen: noticeSeen(deps.db), appVersion: deps.version }))
|
|
19
|
-
// "Got it" is the only control on that modal, and pressing it is what creates the
|
|
20
|
-
//
|
|
21
|
-
//
|
|
18
|
+
// "Got it" is the only control on that modal, and pressing it is what creates the machine
|
|
19
|
+
// id. Idempotent: a second press, a reload or a second tab finds the machine already
|
|
20
|
+
// there and mints nothing.
|
|
22
21
|
.post("/notice", (c) => {
|
|
23
22
|
dismissNotice(telemetry);
|
|
24
23
|
deps.flushSoon();
|
package/dist/edge/http/usage.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { Hono } from "hono";
|
|
2
2
|
import { allTelemetryEvents, machineOf } from "../../slices/telemetry/repo.js";
|
|
3
3
|
import { usageOf } from "../../slices/telemetry/usage.js";
|
|
4
|
-
// The data behind
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
//
|
|
4
|
+
// The data behind the Usage screen: the five counters, the "Tokens by stage" table, and the
|
|
5
|
+
// machine id with the app version. All of it is computed from the local event log,
|
|
6
|
+
// independent of delivery, so this route never touches the collector and answers just as
|
|
7
|
+
// well offline.
|
|
8
8
|
// The return type is inferred so Hono keeps the route types the SPA's client is
|
|
9
9
|
// generated from; see stagingRoutes.
|
|
10
10
|
export function usageRoutes(deps) {
|
|
@@ -12,7 +12,7 @@ export function usageRoutes(deps) {
|
|
|
12
12
|
events: allTelemetryEvents(deps.db),
|
|
13
13
|
// The machine id is shown on this page and goes no further: the server is bound to
|
|
14
14
|
// loopback and the SPA reading it is this machine's own. Nothing puts it in a
|
|
15
|
-
// payload - the collector is told it once, in the envelope
|
|
15
|
+
// payload - the collector is told it once, in the envelope.
|
|
16
16
|
machineId: machineOf(deps.db)?.machineId ?? null,
|
|
17
17
|
appVersion: deps.version,
|
|
18
18
|
})));
|
package/dist/kernel/db/tx.js
CHANGED
|
@@ -1,25 +1,22 @@
|
|
|
1
1
|
// SQLite starts a transaction for a SAVEPOINT issued outside one and nests it inside one
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// exactly the nesting we want, and nothing a caller passes reaches the statement.
|
|
2
|
+
// already open, so a slice can wrap its own writes without knowing whether its caller did
|
|
3
|
+
// too. The name is a constant, not a parameter: SQLite matches RELEASE and ROLLBACK TO
|
|
4
|
+
// against the most recent savepoint of that name, and nothing a caller passes reaches SQL.
|
|
6
5
|
const savepoint = "slopify";
|
|
7
6
|
export function transact(db, run) {
|
|
8
7
|
db.exec(`SAVEPOINT ${savepoint}`);
|
|
9
8
|
try {
|
|
10
9
|
const result = run();
|
|
11
10
|
// node:sqlite is synchronous, so an async block would release this frame while its
|
|
12
|
-
// awaited writes were still to come
|
|
13
|
-
// transaction. The frame is unwound below, which is the only safe answer.
|
|
11
|
+
// awaited writes were still to come, and they would land outside any transaction.
|
|
14
12
|
if (isThenable(result)) {
|
|
15
|
-
// Refusing the block orphans the promise it already started
|
|
16
|
-
// unhandled rejection as fatal
|
|
17
|
-
// error thrown here, which is the one that names the mistake.
|
|
13
|
+
// Refusing the block orphans the promise it already started, and Node treats an
|
|
14
|
+
// unhandled rejection as fatal - it would bury the error thrown here.
|
|
18
15
|
void Promise.resolve(result).catch(() => { });
|
|
19
16
|
throw new Error("transact runs synchronous work only: an awaited write would land after the savepoint is released");
|
|
20
17
|
}
|
|
21
18
|
// Inside the try: a RELEASE that fails leaves the frame open with the block's writes
|
|
22
|
-
// uncommitted, which is a rollback, not a
|
|
19
|
+
// uncommitted, which is a rollback, not a value to return.
|
|
23
20
|
db.exec(`RELEASE ${savepoint}`);
|
|
24
21
|
return result;
|
|
25
22
|
}
|
|
@@ -29,8 +26,8 @@ export function transact(db, run) {
|
|
|
29
26
|
}
|
|
30
27
|
}
|
|
31
28
|
// An unguarded ROLLBACK TO would leave the frame open on failure, and the enclosing
|
|
32
|
-
// transact -
|
|
33
|
-
//
|
|
29
|
+
// transact - same savepoint name - would aim its rollback at the leaked frame and commit
|
|
30
|
+
// part of what it meant to undo.
|
|
34
31
|
function unwind(db, error) {
|
|
35
32
|
try {
|
|
36
33
|
db.exec(`ROLLBACK TO ${savepoint}`);
|
package/dist/kernel/log.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { appendFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
// ceiling: prefix matching, not a secret scanner. The prefixes cover the providers in
|
|
4
|
-
// `slices/settings/model.ts`; fal.ai's `<uuid>:<hex>` form has no prefix
|
|
5
|
-
//
|
|
6
|
-
//
|
|
4
|
+
// `slices/settings/model.ts`; fal.ai's `<uuid>:<hex>` form has no prefix and is the known
|
|
5
|
+
// gap. Hence LogFields is a closed set of three strings - no caller can hand this module
|
|
6
|
+
// an arbitrary payload to serialise.
|
|
7
7
|
const keyLike = /\b(?:sk|rk|pk|hf|gsk|xai|nvapi|r8)[-_][A-Za-z0-9_-]{16,}|\bAIza[A-Za-z0-9_-]{16,}/g;
|
|
8
8
|
const bearerLike = /\bBearer\s+[A-Za-z0-9._~+/=-]{8,}/gi;
|
|
9
9
|
export function openLog(logsDir, clock) {
|
|
@@ -27,8 +27,8 @@ export function openLog(logsDir, clock) {
|
|
|
27
27
|
},
|
|
28
28
|
};
|
|
29
29
|
}
|
|
30
|
-
// Exported for the attempt wrapper: a provider's verbatim error
|
|
31
|
-
//
|
|
30
|
+
// Exported for the attempt wrapper: a provider's verbatim error reaches the project page
|
|
31
|
+
// and the attempt row, so it goes through the same filter as a log line.
|
|
32
32
|
export function redact(text) {
|
|
33
33
|
return text.replace(bearerLike, "Bearer [redacted]").replace(keyLike, "[redacted]");
|
|
34
34
|
}
|
package/dist/kernel/pipeline.js
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
// The pipeline's vocabulary. It sits in the kernel because the runner, the slices that
|
|
2
|
-
// implement a stage
|
|
3
|
-
//
|
|
4
|
-
// The frame a run is made in, and the aspect an image is asked for: one set, named once
|
|
5
|
-
// (`logic/04`, `logic/09` step 1).
|
|
2
|
+
// implement a stage and the edge that reports one all name the same finite sets, and the
|
|
3
|
+
// runner may not import a slice to learn them.
|
|
4
|
+
// The frame a run is made in, and the aspect an image is asked for: one set, named once.
|
|
6
5
|
export const formats = ["16:9", "9:16"];
|
|
7
6
|
export const stageKinds = ["research", "article", "audio", "images", "thumbnail", "video"];
|
|
8
7
|
export const stageStates = [
|
|
@@ -14,9 +13,8 @@ export const stageStates = [
|
|
|
14
13
|
"provided",
|
|
15
14
|
"skipped",
|
|
16
15
|
];
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
// the fallback the derivation returns. Nothing stores it: status is always derived.
|
|
16
|
+
// Four project states, derived in order: running if any stage is running, else canceled,
|
|
17
|
+
// else failed, else done when video is done. A project whose stages are all pending matches
|
|
18
|
+
// none of them - the real window between creating it and the runner starting the first
|
|
19
|
+
// stage - so `pending` is the fallback. Nothing stores it.
|
|
22
20
|
export const projectStates = ["running", "canceled", "failed", "done", "pending"];
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
// The vocabulary that crosses the provider seam. Domain types only: a vendor's payload
|
|
2
|
-
// shape never leaves its adapter
|
|
2
|
+
// shape never leaves its adapter.
|
|
3
3
|
export const providerFamilies = ["llm", "tts", "image"];
|
|
4
4
|
export const providerErrorKinds = [
|
|
5
5
|
"auth",
|
|
6
|
-
// Distinct from `auth`,
|
|
7
|
-
//
|
|
8
|
-
// immediately, without retries", so the wrapper needs two kinds to tell apart.
|
|
6
|
+
// Distinct from `auth`, a key the provider rejected: a bad key runs the whole retry
|
|
7
|
+
// policy, an absent one fails immediately.
|
|
9
8
|
"missing_key",
|
|
10
9
|
"rate_limit",
|
|
11
10
|
"refusal",
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { providerErrorKinds } from "../ports/model.js";
|
|
3
|
-
// One row per provider call attempt
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// the evidence behind (`logic/01` §Q7).
|
|
3
|
+
// One row per provider call attempt. The project page reads the last error text off these rows
|
|
4
|
+
// and Usage counts them, so an attempt is written when it opens rather than when it ends: a
|
|
5
|
+
// process that dies mid-call leaves the evidence behind.
|
|
7
6
|
export const attemptOutcomes = ["ok", "canceled", ...providerErrorKinds];
|
|
8
7
|
const attemptRow = z.object({
|
|
9
8
|
id: z.string(),
|
|
@@ -20,8 +19,8 @@ export function sqliteAttempts(db, ids) {
|
|
|
20
19
|
start: (attempt) => {
|
|
21
20
|
const id = ids.next();
|
|
22
21
|
db.prepare("INSERT INTO attempts (id, stage_id, piece_id, n, started_at) VALUES (?, ?, ?, ?, ?)").run(id, attempt.stageId, attempt.pieceId, attempt.n, attempt.startedAt);
|
|
23
|
-
//
|
|
24
|
-
//
|
|
22
|
+
// The stage shows the attempt count beside the error. A stage with pieces (twenty images)
|
|
23
|
+
// shows the count of the call being attempted, not a sum.
|
|
25
24
|
db.prepare("UPDATE stages SET attempt_count = ? WHERE id = ?").run(attempt.n, attempt.stageId);
|
|
26
25
|
return id;
|
|
27
26
|
},
|
|
@@ -1,24 +1,20 @@
|
|
|
1
1
|
import { redact } from "../log.js";
|
|
2
2
|
import { isProviderError, providerError } from "../ports/model.js";
|
|
3
|
-
// The retry policy
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// "attempted up to 4 times: the first attempt plus 3 retries, waiting 2 s, 8 s, 30 s
|
|
7
|
-
// between attempts" (`logic/01` step 6, §Q4).
|
|
3
|
+
// The retry policy, in one place. A stage slice is handed the wrapped calls of
|
|
4
|
+
// `providers.ts` and never an adapter, so there is no way round it.
|
|
5
|
+
// Four attempts - the first plus 3 retries - waiting 2 s, 8 s, 30 s.
|
|
8
6
|
export const attemptLimit = 4;
|
|
9
7
|
export const backoffMs = [2000, 8000, 30_000];
|
|
10
|
-
//
|
|
11
|
-
// 300 s" (`logic/01` step 6, §Q62; `logic/09` §Q77).
|
|
8
|
+
// Each attempt times out at 120 s, image calls at 300 s.
|
|
12
9
|
export const timeoutMs = {
|
|
13
10
|
llm: 120_000,
|
|
14
11
|
tts: 120_000,
|
|
15
12
|
image: 300_000,
|
|
16
13
|
};
|
|
17
|
-
//
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
// provider rejected, which is an `auth` failure and is retried like any other (§Q11).
|
|
14
|
+
// A refusal and an unsupported capability are the provider's final answer; retrying only
|
|
15
|
+
// spends the user's money again. The third is a key removed mid-run, which leaves nothing
|
|
16
|
+
// to call with. A key the provider rejected is different: that is an `auth` failure,
|
|
17
|
+
// retried like any other.
|
|
22
18
|
const terminalKinds = ["refusal", "unsupported", "missing_key"];
|
|
23
19
|
export async function attempt(ctx, call, opts) {
|
|
24
20
|
ctx.signal.throwIfAborted();
|
|
@@ -49,9 +45,8 @@ export async function attempt(ctx, call, opts) {
|
|
|
49
45
|
return result.value;
|
|
50
46
|
}
|
|
51
47
|
if (ctx.signal.aborted) {
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
// user stopped it, the provider did not fail.
|
|
48
|
+
// An aborted call counts nothing. The row is closed so the page is
|
|
49
|
+
// not left with an attempt that never ended, and carries no error text.
|
|
55
50
|
ctx.attempts.end(id, {
|
|
56
51
|
outcome: "canceled",
|
|
57
52
|
endedAt: ctx.clock.now().toISOString(),
|
|
@@ -73,12 +68,11 @@ export async function attempt(ctx, call, opts) {
|
|
|
73
68
|
if (terminalKinds.includes(failure.fault.kind) || n >= attemptLimit) {
|
|
74
69
|
throw failure;
|
|
75
70
|
}
|
|
76
|
-
//
|
|
71
|
+
// A 429 that names a Retry-After replaces the fixed wait with the provider's.
|
|
77
72
|
await ctx.clock.sleep(failure.fault.retryAfterMs ?? backoffMs[n - 1] ?? 0, ctx.signal);
|
|
78
73
|
}
|
|
79
74
|
}
|
|
80
|
-
// The wrapper is the only place a provider failure is named
|
|
81
|
-
// downstream reads `fault.kind` and shows `message`.
|
|
75
|
+
// The wrapper is the only place a provider failure is named.
|
|
82
76
|
function classify(error, expired, opts, limit) {
|
|
83
77
|
if (expired) {
|
|
84
78
|
const seconds = Math.round(limit / 1000);
|
|
@@ -90,8 +84,8 @@ function classify(error, expired, opts, limit) {
|
|
|
90
84
|
});
|
|
91
85
|
}
|
|
92
86
|
if (isProviderError(error)) {
|
|
93
|
-
// Rebuilt rather than rethrown so the text
|
|
94
|
-
//
|
|
87
|
+
// Rebuilt rather than rethrown so the text reaching the stage row, the page and the
|
|
88
|
+
// log has been through the redactor: a provider is free to quote the key back.
|
|
95
89
|
return providerError({
|
|
96
90
|
kind: error.fault.kind,
|
|
97
91
|
message: redact(error.message),
|
|
@@ -103,9 +97,8 @@ function classify(error, expired, opts, limit) {
|
|
|
103
97
|
message: redact(error instanceof Error ? error.message : String(error)),
|
|
104
98
|
});
|
|
105
99
|
}
|
|
106
|
-
//
|
|
107
|
-
//
|
|
108
|
-
// stream that stalls is. Composed with AbortSignal.any so the stage's cancel still wins.
|
|
100
|
+
// The same milliseconds, measured two ways. `idle` restarts the count on every chunk, so a
|
|
101
|
+
// stalled stream is cut off and a long answer is not. Cancel still wins.
|
|
109
102
|
function deadline(clock, parent, ms, idle) {
|
|
110
103
|
const timer = new AbortController();
|
|
111
104
|
const signal = AbortSignal.any([parent, timer.signal]);
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
//
|
|
1
|
+
// Research → article → {audio ∥ images ∥ thumbnail} → video.
|
|
2
2
|
export const deps = {
|
|
3
3
|
research: [],
|
|
4
4
|
article: ["research"],
|
|
@@ -7,11 +7,11 @@ export const deps = {
|
|
|
7
7
|
thumbnail: ["article"],
|
|
8
8
|
video: ["audio", "images", "thumbnail"],
|
|
9
9
|
};
|
|
10
|
-
//
|
|
10
|
+
// `provided` and `skipped` release a dependency exactly as `done` does.
|
|
11
11
|
export function satisfied(state) {
|
|
12
12
|
return state === "done" || state === "provided" || state === "skipped";
|
|
13
13
|
}
|
|
14
|
-
//
|
|
14
|
+
// The four states, in their order. Derived on every read, never stored.
|
|
15
15
|
export function derive(stages) {
|
|
16
16
|
if (stages.some((stage) => stage.state === "running")) {
|
|
17
17
|
return "running";
|
|
@@ -25,39 +25,28 @@ export function derive(stages) {
|
|
|
25
25
|
if (stages.some((stage) => stage.kind === "video" && stage.state === "done")) {
|
|
26
26
|
return "done";
|
|
27
27
|
}
|
|
28
|
-
// Extended rule,
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
32
|
-
//
|
|
33
|
-
// about to start. `logic/13` step 3 is unconditional - "the project reads `canceled`" -
|
|
34
|
-
// so that is what a run which started and stopped reads here.
|
|
35
|
-
//
|
|
36
|
-
// A stage in a terminal state is what tells the two apart: at creation every stage is
|
|
37
|
-
// `pending`, `provided` or `skipped`, and only a stage the runner carried to the end is
|
|
38
|
-
// `done`. Nothing restarts such a project by itself (§Q111), so `pending` is the one
|
|
39
|
-
// answer that is certainly wrong.
|
|
28
|
+
// Extended rule, beyond the four above. `pending` covers the window between creating a
|
|
29
|
+
// project and the runner claiming its first stage. A second case is reachable: a stage
|
|
30
|
+
// that stores its output in the same instant as the cancel stays `done`, so nothing is
|
|
31
|
+
// `canceled` and a stopped run would read as about to start. A stage in a terminal state
|
|
32
|
+
// tells the two apart - at creation none is `done`.
|
|
40
33
|
//
|
|
41
34
|
// ceiling: a process killed between a stage finishing and its dependent being claimed
|
|
42
|
-
//
|
|
43
|
-
//
|
|
44
|
-
// the upgrade if it ever matters; §Q9 keeps status derived and stores nothing.
|
|
35
|
+
// reads `canceled` too. Separating those needs the cancel stored on the project, and
|
|
36
|
+
// nothing about the status is stored.
|
|
45
37
|
if (stages.some((stage) => stage.state === "done")) {
|
|
46
38
|
return "canceled";
|
|
47
39
|
}
|
|
48
40
|
return "pending";
|
|
49
41
|
}
|
|
50
|
-
// The thin meter under a running row on
|
|
51
|
-
//
|
|
52
|
-
//
|
|
53
|
-
//
|
|
54
|
-
//
|
|
55
|
-
// `provided` and `skipped` are left out of the average rather than counted as finished.
|
|
56
|
-
// Nothing was asked of them, and counting them would put a run with research off and a
|
|
57
|
-
// supplied article at a third of the way along before the first call was made.
|
|
42
|
+
// The thin meter under a running row on Projects, averaging stage progress. Every stage
|
|
43
|
+
// the run asks for counts once: a finished one whole, a running one by its own progress, a
|
|
44
|
+
// waiting, failed or canceled one not at all. `provided` and `skipped` are left out rather
|
|
45
|
+
// than counted as finished, which would put a run with a supplied article a third of the
|
|
46
|
+
// way along before the first call was made.
|
|
58
47
|
export function progressOf(stages) {
|
|
59
48
|
const asked = stages.filter((stage) => stage.state !== "provided" && stage.state !== "skipped");
|
|
60
|
-
// A run made entirely of supplied files has nothing outstanding
|
|
49
|
+
// A run made entirely of supplied files has nothing outstanding; dividing by zero
|
|
61
50
|
// would answer NaN.
|
|
62
51
|
if (asked.length === 0) {
|
|
63
52
|
return 1;
|
|
@@ -74,8 +63,8 @@ function shareOf(stage) {
|
|
|
74
63
|
}
|
|
75
64
|
const total = stage.progressTotal ?? 0;
|
|
76
65
|
if (total <= 0) {
|
|
77
|
-
//
|
|
78
|
-
//
|
|
66
|
+
// The stage has not said how many chapters or chunks there are yet. Guessing gives a
|
|
67
|
+
// meter that moves backwards when the count arrives.
|
|
79
68
|
return 0;
|
|
80
69
|
}
|
|
81
70
|
return Math.min(1, Math.max(0, (stage.progressCurrent ?? 0) / total));
|
|
@@ -1,16 +1,14 @@
|
|
|
1
1
|
import { derive, deps as graph, satisfied } from "./graph.js";
|
|
2
2
|
export function createRunner(deps) {
|
|
3
3
|
const inflight = new Map();
|
|
4
|
-
// ceiling: one entry per project ticked since boot, never evicted
|
|
5
|
-
//
|
|
6
|
-
//
|
|
7
|
-
// same project, so the upgrade is to drop the entry when the project is deleted.
|
|
4
|
+
// ceiling: one entry per project ticked since boot, never evicted - a few dozen bytes
|
|
5
|
+
// each, emptied by any restart. Evicting on a terminal state would re-announce it on the
|
|
6
|
+
// next tick, so the upgrade is to drop the entry when the project is deleted.
|
|
8
7
|
const announced = new Map();
|
|
9
|
-
// The projects a cancel is walking through.
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
// hand-over to the next stage is exactly what must not happen.
|
|
8
|
+
// The projects a cancel is walking through. A set, not a flag, because a cancel leaves
|
|
9
|
+
// every other project untouched. Held for the whole of abortProject: a stage that stores
|
|
10
|
+
// its output in the same instant as the cancel stays `done`, and its hand-over to the
|
|
11
|
+
// next stage is what must not happen.
|
|
14
12
|
const stopped = new Set();
|
|
15
13
|
let count = 0;
|
|
16
14
|
let shuttingDown = false;
|
|
@@ -23,8 +21,8 @@ export function createRunner(deps) {
|
|
|
23
21
|
...(reason === null ? {} : { failureReason: reason }),
|
|
24
22
|
});
|
|
25
23
|
};
|
|
26
|
-
// The global tally
|
|
27
|
-
//
|
|
24
|
+
// The global tally counts projects with a stage in flight, read off the in-flight set
|
|
25
|
+
// rather than the database.
|
|
28
26
|
const retally = () => {
|
|
29
27
|
const projects = new Set();
|
|
30
28
|
for (const entry of inflight.values()) {
|
|
@@ -48,8 +46,8 @@ export function createRunner(deps) {
|
|
|
48
46
|
deps.stages.finish(stage.id, state, reason);
|
|
49
47
|
}
|
|
50
48
|
catch (error) {
|
|
51
|
-
// The
|
|
52
|
-
//
|
|
49
|
+
// The row stays `running` when the write fails. The event still goes out, and the
|
|
50
|
+
// next boot marks the row interrupted.
|
|
53
51
|
deps.log.write("error", "stage.finish", {
|
|
54
52
|
projectId: stage.projectId,
|
|
55
53
|
stage: stage.kind,
|
|
@@ -75,8 +73,8 @@ export function createRunner(deps) {
|
|
|
75
73
|
}
|
|
76
74
|
catch (error) {
|
|
77
75
|
if (controller.signal.aborted) {
|
|
78
|
-
//
|
|
79
|
-
//
|
|
76
|
+
// A late rejection from the aborted call is how a stage learns it was canceled, so
|
|
77
|
+
// it is not logged as a fault.
|
|
80
78
|
conclude(stage, "canceled", "canceled by user");
|
|
81
79
|
return;
|
|
82
80
|
}
|
|
@@ -98,8 +96,7 @@ export function createRunner(deps) {
|
|
|
98
96
|
release = resolve;
|
|
99
97
|
}),
|
|
100
98
|
};
|
|
101
|
-
// Registered before the
|
|
102
|
-
// starting has to see it in flight.
|
|
99
|
+
// Registered before the body runs: a tick arriving mid-start has to see it in flight.
|
|
103
100
|
inflight.set(stage.id, entry);
|
|
104
101
|
retally();
|
|
105
102
|
emitStage(stage, "running", null);
|
|
@@ -108,13 +105,13 @@ export function createRunner(deps) {
|
|
|
108
105
|
inflight.delete(stage.id);
|
|
109
106
|
release();
|
|
110
107
|
try {
|
|
111
|
-
//
|
|
112
|
-
//
|
|
108
|
+
// Claim the next stage before reading the tally, so a hand-over never reports
|
|
109
|
+
// the project as briefly not running.
|
|
113
110
|
tick(stage.projectId);
|
|
114
111
|
}
|
|
115
112
|
finally {
|
|
116
|
-
// stagesOf
|
|
117
|
-
//
|
|
113
|
+
// stagesOf can throw. The tally is replayed to every page that opens later, so
|
|
114
|
+
// it is read whether or not the tick got that far.
|
|
118
115
|
retally();
|
|
119
116
|
}
|
|
120
117
|
})
|
|
@@ -127,10 +124,9 @@ export function createRunner(deps) {
|
|
|
127
124
|
});
|
|
128
125
|
}
|
|
129
126
|
function tick(projectId) {
|
|
130
|
-
// A stage
|
|
131
|
-
//
|
|
132
|
-
// nobody
|
|
133
|
-
// aborted. The project's state still goes out, so an open page sees the run stop.
|
|
127
|
+
// A stage finishing in the same instant as a shutdown or cancel would otherwise release
|
|
128
|
+
// its dependent, leaving the abort waiting on a render nobody asked for - or one running
|
|
129
|
+
// under a controller nobody aborted. The state still goes out, so an open page sees it.
|
|
134
130
|
if (!shuttingDown && !stopped.has(projectId)) {
|
|
135
131
|
startEligible(projectId);
|
|
136
132
|
}
|
|
@@ -146,8 +142,7 @@ export function createRunner(deps) {
|
|
|
146
142
|
if (!graph[stage.kind].every((kind) => satisfied(stateOf(kind)))) {
|
|
147
143
|
continue;
|
|
148
144
|
}
|
|
149
|
-
// Nothing is awaited
|
|
150
|
-
// images and thumbnail together rather than one after another.
|
|
145
|
+
// Nothing is awaited here: the fan-out starts audio, images and thumbnail together.
|
|
151
146
|
if (deps.stages.claim(stage.id)) {
|
|
152
147
|
start({ ...stage, state: "running" });
|
|
153
148
|
}
|
|
@@ -171,8 +166,8 @@ export function createRunner(deps) {
|
|
|
171
166
|
tick,
|
|
172
167
|
settled,
|
|
173
168
|
abortProject: async (projectId) => {
|
|
174
|
-
// Up before the first abort
|
|
175
|
-
//
|
|
169
|
+
// Up before the first abort, down only once every stage has stopped. In between, the
|
|
170
|
+
// tick each finishing stage fires starts nothing.
|
|
176
171
|
stopped.add(projectId);
|
|
177
172
|
try {
|
|
178
173
|
for (const entry of inflight.values()) {
|
|
@@ -180,15 +175,14 @@ export function createRunner(deps) {
|
|
|
180
175
|
entry.controller.abort();
|
|
181
176
|
}
|
|
182
177
|
}
|
|
183
|
-
//
|
|
184
|
-
// returns". Waiting here is what makes that true rather than hoped for.
|
|
178
|
+
// No provider call continues after cancel returns. The wait makes it true.
|
|
185
179
|
await settledOf(projectId);
|
|
186
180
|
}
|
|
187
181
|
finally {
|
|
188
182
|
stopped.delete(projectId);
|
|
189
183
|
}
|
|
190
|
-
// The last stage's
|
|
191
|
-
//
|
|
184
|
+
// The last stage's tick announced the project while the barrier was up, before every
|
|
185
|
+
// row was written. This is the state the user asked for.
|
|
192
186
|
announce(projectId);
|
|
193
187
|
},
|
|
194
188
|
abortAll: async () => {
|
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
-
// One row per resumable sub-unit of a stage
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
//
|
|
6
|
-
// as: its shape belongs to whichever stage owns the kind, and nothing here reads inside.
|
|
2
|
+
// One `stage_pieces` row per resumable sub-unit of a stage: a research chapter, an audio
|
|
3
|
+
// chunk, an image index. It sits beside the attempt store because the wrapper stamps
|
|
4
|
+
// `piece_id` on every row it writes. The payload stays the JSON text it was stored as - its
|
|
5
|
+
// shape belongs to whichever stage owns the kind, and nothing here reads inside it.
|
|
7
6
|
export const pieceKinds = ["chapter", "chunk", "segment", "image", "prompt_written"];
|
|
8
7
|
export const pieceStates = ["pending", "running", "done", "failed"];
|
|
9
8
|
const pieceRow = z.object({
|
|
@@ -17,9 +16,8 @@ const pieceRow = z.object({
|
|
|
17
16
|
export function insertPiece(db, piece) {
|
|
18
17
|
db.prepare("INSERT INTO stage_pieces (id, stage_id, kind, idx, state, payload) VALUES (?, ?, ?, ?, ?, ?)").run(piece.id, piece.stageId, piece.kind, piece.idx, piece.state, piece.payload);
|
|
19
18
|
}
|
|
20
|
-
// In `idx` order:
|
|
21
|
-
//
|
|
22
|
-
// stage's output is assembled in.
|
|
19
|
+
// In `idx` order: a resume re-runs only the unfinished pieces, and planning order is
|
|
20
|
+
// assembly order.
|
|
23
21
|
export function piecesOf(db, stageId, kind) {
|
|
24
22
|
return db
|
|
25
23
|
.prepare("SELECT * FROM stage_pieces WHERE stage_id = ? AND kind = ? ORDER BY idx")
|
|
@@ -28,7 +26,7 @@ export function piecesOf(db, stageId, kind) {
|
|
|
28
26
|
}
|
|
29
27
|
// Every piece of a stage whatever its kind, for the one caller that does not care: a
|
|
30
28
|
// re-run throws the whole plan away, because the text a chunk was cut from and the prompt
|
|
31
|
-
// an image was sent with are what changed
|
|
29
|
+
// an image was sent with are what changed.
|
|
32
30
|
export function allPiecesOf(db, stageId) {
|
|
33
31
|
return db
|
|
34
32
|
.prepare("SELECT * FROM stage_pieces WHERE stage_id = ? ORDER BY kind, idx")
|
|
@@ -38,8 +36,8 @@ export function allPiecesOf(db, stageId) {
|
|
|
38
36
|
export function deletePieces(db, stageId) {
|
|
39
37
|
db.prepare("DELETE FROM stage_pieces WHERE stage_id = ?").run(stageId);
|
|
40
38
|
}
|
|
41
|
-
// One piece
|
|
42
|
-
//
|
|
39
|
+
// One piece: an image the user deleted leaves no row behind, or the next run would look at
|
|
40
|
+
// the plan and make it again.
|
|
43
41
|
export function deletePiece(db, id) {
|
|
44
42
|
db.prepare("DELETE FROM stage_pieces WHERE id = ?").run(id);
|
|
45
43
|
}
|
|
@@ -12,8 +12,8 @@ export function stageProviders(deps, context, pieceId) {
|
|
|
12
12
|
};
|
|
13
13
|
return {
|
|
14
14
|
llm: (call, onEvent) => {
|
|
15
|
-
// Resolved once
|
|
16
|
-
//
|
|
15
|
+
// Resolved once; the adapter reads the stored key per request, so a key replaced
|
|
16
|
+
// mid-run still reaches the next attempt.
|
|
17
17
|
const port = deps.registry.llm(call.provider);
|
|
18
18
|
return attempt(ctx, async (signal, progress) => {
|
|
19
19
|
let text = "";
|
|
@@ -25,8 +25,7 @@ export function stageProviders(deps, context, pieceId) {
|
|
|
25
25
|
...(call.webSearch === undefined ? {} : { webSearch: call.webSearch }),
|
|
26
26
|
signal,
|
|
27
27
|
})) {
|
|
28
|
-
// Every event is a sign of life, so the idle clock restarts here
|
|
29
|
-
// in each slice that consumes one.
|
|
28
|
+
// Every event is a sign of life, so the idle clock restarts here.
|
|
30
29
|
progress();
|
|
31
30
|
if (event.type === "delta") {
|
|
32
31
|
text += event.text;
|
|
@@ -40,8 +39,7 @@ export function stageProviders(deps, context, pieceId) {
|
|
|
40
39
|
const answer = { text, usage, finishReason };
|
|
41
40
|
const unusable = call.check?.(answer);
|
|
42
41
|
if (unusable !== undefined) {
|
|
43
|
-
// Thrown bare: the wrapper
|
|
44
|
-
// this `other`, which is retried like any other bad answer.
|
|
42
|
+
// Thrown bare: the wrapper names it `other` and retries it like any bad answer.
|
|
45
43
|
throw new Error(unusable);
|
|
46
44
|
}
|
|
47
45
|
return answer;
|
|
@@ -84,7 +82,7 @@ export function stageProviders(deps, context, pieceId) {
|
|
|
84
82
|
aspect: call.aspect,
|
|
85
83
|
signal,
|
|
86
84
|
}),
|
|
87
|
-
// One request, one answer: the 300 s runs over the whole call
|
|
85
|
+
// One request, one answer: the 300 s runs over the whole call.
|
|
88
86
|
{ kind: "image" });
|
|
89
87
|
},
|
|
90
88
|
forPiece: (piece) => stageProviders(deps, context, piece),
|
package/dist/kernel/version.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { readFileSync } from "node:fs";
|
|
2
|
-
// package.json sits two directories above this module both
|
|
3
|
-
//
|
|
4
|
-
//
|
|
2
|
+
// package.json sits two directories above this module in both src/ and dist/, because the
|
|
3
|
+
// build mirrors the source tree. Reading it through import.meta.url survives `npx` from
|
|
4
|
+
// any working directory.
|
|
5
5
|
export function readVersion() {
|
|
6
6
|
const parsed = JSON.parse(readFileSync(new URL("../../package.json", import.meta.url), "utf8"));
|
|
7
7
|
if (typeof parsed !== "object" ||
|