@gentbajko/slopify 3.12.1 → 3.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,23 @@
1
+ // The planner rebuilds a project's whole plan several times a runner tick, on the server's one
2
+ // thread, and each rebuild ran the same pure text work again over the full narration: matching
3
+ // the pronunciation glossary, cutting chunks, parsing the glossary. On a 2-hour narration that
4
+ // held the thread for seconds at a time, so the app stopped answering while a project ran.
5
+ // A pure function's last few results, by its exact input. Not a service: nothing is injected or
6
+ // shared but answers the function would give anyway. Results are frozen data, never mutated.
7
+ export function remembered(size, keyOf, work) {
8
+ const results = new Map();
9
+ return (...args) => {
10
+ const key = keyOf(...args);
11
+ const known = results.get(key);
12
+ if (known !== undefined || results.has(key)) {
13
+ results.delete(key);
14
+ results.set(key, known);
15
+ return known;
16
+ }
17
+ const value = work(...args);
18
+ results.set(key, value);
19
+ if (results.size > size)
20
+ results.delete(results.keys().next().value);
21
+ return value;
22
+ };
23
+ }
package/dist/main.js CHANGED
@@ -62,7 +62,7 @@ import { waitToRetry, wakeRetries } from "./slices/rebuild/retry.js";
62
62
  import { createReviewRedos, reviewHold } from "./slices/rebuild/review-redo.js";
63
63
  import { materializeAdmittedWork } from "./slices/rebuild/runtime-materialize.js";
64
64
  import { runRevisionInvocation } from "./slices/rebuild/runtime-run.js";
65
- import { executionStages, executionStandings, invocationReady, projectStandings, recordWorkProgress, } from "./slices/rebuild/runtime-store.js";
65
+ import { executionStandings, invocationReady, projectStandings, recordWorkProgress, } from "./slices/rebuild/runtime-store.js";
66
66
  import { currentRevisionId } from "./slices/revisions/repo.js";
67
67
  import { createLimitGate, resumeAfterRestart } from "./slices/run-cost/limits.js";
68
68
  import { createStandaloneMeter, createUsageMeter } from "./slices/run-cost/meter.js";
@@ -759,8 +759,7 @@ export function wireRunner({ onFinished, audioPreviews, db, paths, clock, ids, l
759
759
  stages: {
760
760
  stagesOf: (projectId) => {
761
761
  materializeAdmittedWork(execution, projectId);
762
- projectStandings(execution, projectId);
763
- return executionStages(execution, projectId);
762
+ return projectStandings(execution, projectId);
764
763
  },
765
764
  standingsOf: (projectId) => executionStandings(execution, projectId),
766
765
  ready: (work) => invocationReady(execution, work),
@@ -0,0 +1,5 @@
1
+ # Slopify 3.13.0
2
+
3
+ Released 8 October 2026.
4
+
5
+ - **Editing the article re-voices only what you changed.** Slopify splits the narration into chunks by length, filling each one sentence by sentence from the top. Removing or adding a sentence near the start moved every chunk boundary after it, so every chunk looked new and the whole narration was voiced again. A save that changes the article or narration text now keeps the chunks it had wherever their text is unchanged, and only cuts the edited stretch again. Dropping one line from the top of a 29-chunk narration now re-voices 1 chunk instead of 29. Changing the chunk size in settings still splits the whole narration again.
@@ -0,0 +1,5 @@
1
+ # Slopify 3.13.1
2
+
3
+ Released 8 October 2026.
4
+
5
+ - **Slopify keeps answering while a long project runs.** Each time a step started or finished, Slopify worked out the project's whole plan again several times over, on the same thread that answers the app. On a two-hour narration that took about 7 seconds per step, so with steps finishing back to back the app stopped responding for minutes. The repeated text work (matching the pronunciation glossary, cutting chunks, reading the glossary) is now done once and remembered, and the plan is worked out once per step instead of several times. The same project now takes about 1 second per step.
@@ -1,4 +1,16 @@
1
1
  [
2
+ {
3
+ "id": "3.13.1",
4
+ "title": "Slopify 3.13.1",
5
+ "version": "3.13.1",
6
+ "date": "2026-10-08"
7
+ },
8
+ {
9
+ "id": "3.13.0",
10
+ "title": "Slopify 3.13.0",
11
+ "version": "3.13.0",
12
+ "date": "2026-10-08"
13
+ },
2
14
  {
3
15
  "id": "3.12.1",
4
16
  "title": "Slopify 3.12.1",
@@ -1,3 +1,4 @@
1
+ import { remembered } from "../../kernel/remembered.js";
1
2
  export const chunkModes = ["whole", "paragraph", "words", "characters"];
2
3
  export const defaultChunkWords = 500;
3
4
  export const defaultChunkCharacters = 3000;
@@ -7,7 +8,9 @@ export const defaultChunking = { mode: "whole" };
7
8
  // A run of blank lines is one paragraph break, not several, and a line of nothing but
8
9
  // spaces between two paragraphs is still a break: strip-markdown output is full of both.
9
10
  const paragraphBreak = /\r?\n[ \t\r]*(?:\r?\n[ \t\r]*)+/;
10
- export function chunkNarration(text, chunking) {
11
+ // Remembered (`kernel/remembered.ts`): every plan rebuild cuts the whole narration again.
12
+ export const chunkNarration = remembered(16, (text, chunking) => `${JSON.stringify(chunking)}\u0000${text}`, cutNarration);
13
+ function cutNarration(text, chunking) {
11
14
  switch (chunking.mode) {
12
15
  case "whole":
13
16
  return nonEmpty([text]);
@@ -19,6 +22,28 @@ export function chunkNarration(text, chunking) {
19
22
  return sentenceRuns(text, chunking.characters ?? defaultChunkCharacters, (part) => Array.from(part.trim()).length);
20
23
  }
21
24
  }
25
+ // The chunks of an edited text, keeping the previous version's chunks wherever their text is
26
+ // still there unchanged: only the stretches between them are cut again. Cutting the whole text
27
+ // afresh packs sentences greedily from the start, so one sentence removed near the top moved
28
+ // every boundary after it and every chunk was voiced again (a 3,800-character edit re-voiced
29
+ // 113,000). `anchors` are the previous chunks in order; without them this is `chunkNarration`.
30
+ export function anchoredChunks(text, chunking, anchors) {
31
+ if (anchors === undefined || anchors.length === 0)
32
+ return chunkNarration(text, chunking);
33
+ const chunks = [];
34
+ let offset = 0;
35
+ for (const anchor of anchors) {
36
+ if (anchor === "")
37
+ continue;
38
+ const at = text.indexOf(anchor, offset);
39
+ if (at < 0)
40
+ continue;
41
+ chunks.push(...chunkNarration(text.slice(offset, at), chunking), anchor);
42
+ offset = at + anchor.length;
43
+ }
44
+ chunks.push(...chunkNarration(text.slice(offset), chunking));
45
+ return chunks;
46
+ }
22
47
  // What "N words" counts: runs of non-space. Exported because it is half of the rule -
23
48
  // a test that asserts a chunk fits in N has to count the same way the cut did. Chinese,
24
49
  // Japanese, Thai and the other scripts written without spaces are counted by the platform's
@@ -1,7 +1,21 @@
1
1
  import { fingerprint } from "../../kernel/runner/work.js";
2
- import { chunkNarration, defaultChunkCharacters, defaultChunkWords, } from "./chunk.js";
2
+ import { anchoredChunks, defaultChunkCharacters, defaultChunkWords, } from "./chunk.js";
3
3
  import { pronunciationMatches } from "./pronunciation.js";
4
- export function pronunciationChunks(source, chunking, entries, overriddenKeys = new Set(), sources = {}) {
4
+ export function chunkingFingerprintOf(chunking) {
5
+ return fingerprint([
6
+ chunking.mode,
7
+ chunking.mode === "words"
8
+ ? (chunking.words ?? defaultChunkWords)
9
+ : chunking.mode === "characters"
10
+ ? (chunking.characters ?? defaultChunkCharacters)
11
+ : null,
12
+ ]);
13
+ }
14
+ // The plain chunks of a narration text, before pronunciation merges: what anchors record.
15
+ export function narrationChunkTexts(source, chunking, anchors) {
16
+ return anchoredChunks(source, chunking, anchors?.chunkingFingerprint === chunkingFingerprintOf(chunking) ? anchors.texts : undefined);
17
+ }
18
+ export function pronunciationChunks(source, chunking, entries, overriddenKeys = new Set(), sources = {}, anchors) {
5
19
  const occurrences = new Map();
6
20
  const usedKeys = new Set();
7
21
  const nextKey = (text) => {
@@ -15,7 +29,7 @@ export function pronunciationChunks(source, chunking, entries, overriddenKeys =
15
29
  return key;
16
30
  };
17
31
  let offset = 0;
18
- const chunks = chunkNarration(source, chunking).map((text) => {
32
+ const chunks = narrationChunkTexts(source, chunking, anchors).map((text) => {
19
33
  const start = source.indexOf(text, offset);
20
34
  if (start < 0)
21
35
  throw new Error("Slopify hit an internal error (a narration chunk isn't in the article text). Try again; if it happens again, use Download diagnostics in Settings and report it.");
@@ -23,14 +37,7 @@ export function pronunciationChunks(source, chunking, entries, overriddenKeys =
23
37
  return { key: nextKey(text), text, start, end: offset };
24
38
  });
25
39
  const bodyFingerprint = fingerprint(source);
26
- const chunkingFingerprint = fingerprint([
27
- chunking.mode,
28
- chunking.mode === "words"
29
- ? (chunking.words ?? defaultChunkWords)
30
- : chunking.mode === "characters"
31
- ? (chunking.characters ?? defaultChunkCharacters)
32
- : null,
33
- ]);
40
+ const chunkingFingerprint = chunkingFingerprintOf(chunking);
34
41
  const starts = new Map(chunks.map((chunk, index) => [chunk.start, index]));
35
42
  const ends = new Map(chunks.map((chunk, index) => [chunk.end, index]));
36
43
  const pinned = new Map();
@@ -1,5 +1,6 @@
1
1
  import { remark } from "remark";
2
2
  import remarkGfm from "remark-gfm";
3
+ import { remembered } from "../../kernel/remembered.js";
3
4
  // Inworld requires English IPA, not every symbol in Unicode's IPA Extensions block.
4
5
  // https://docs.inworld.ai/tts/capabilities/custom-pronunciation
5
6
  const ipaAtom = "[ˈˌ]?[abdefghijklmnoprstuvwxzæðŋθɑɒɔəɚɛɜɝɡɪɹʃʊʌʒʔɫɾ][\\u0303\\u031a\\u0325\\u0329\\u032a\\u032c\\u032f\\u035c\\u0361ʰʲʷ]*[ːˑ]?";
@@ -49,7 +50,8 @@ function termIdentity(term) {
49
50
  }
50
51
  // `language` is the project's (`kernel/ports/languages.ts`): absent or English accepts the
51
52
  // standard-English IPA Inworld asks for, as it always did; any other accepts full IPA.
52
- export function parsePronunciationGlossary(markdown, language) {
53
+ export const parsePronunciationGlossary = remembered(8, (markdown, language) => `${language ?? ""}\u0000${markdown}`, parseGlossary);
54
+ function parseGlossary(markdown, language) {
53
55
  const english = language === undefined || language === "en";
54
56
  const symbols = english ? ipaSymbols : worldSymbols;
55
57
  const kind = english ? "standard-English IPA" : "IPA";
@@ -127,7 +129,9 @@ export function skippedSpeakerPronunciationsNotice(rows) {
127
129
  return (`Speaker pronunciations: some entries are skipped and read as ordinary text (${parts.join(". ")}). ` +
128
130
  "The rest are used. To use them, fix those entries under Speakers → Pronunciations for that speaker (Play → Narration, or Edit project → Providers).");
129
131
  }
130
- export function pronunciationMatches(source, entries) {
132
+ // Remembered (`kernel/remembered.ts`): every plan rebuild matches the whole narration again.
133
+ export const pronunciationMatches = remembered(1024, (source, entries) => `${source}\u0000${JSON.stringify(entries)}`, matchPronunciations);
134
+ function matchPronunciations(source, entries) {
131
135
  if (entries.length === 0)
132
136
  return [];
133
137
  const ordered = [...entries].sort((a, b) => b.term.length - a.term.length);
@@ -14,7 +14,7 @@ import { voiceBodyRecipes } from "./recipe-voices.js";
14
14
  export function bodyNarrationGroups(context, text) {
15
15
  if (text.narrationText === null)
16
16
  return [];
17
- const groups = pronunciationChunks(normalizeNarrationText(text.narrationText), context.config.chunking ?? defaultChunking, usesPronunciationGlossary(context.config) && text.glossary?.ok ? text.glossary.entries : [], new Set(Object.keys(context.content.narrationOverrides)), context.content.narrationSources);
17
+ const groups = pronunciationChunks(normalizeNarrationText(text.narrationText), context.config.chunking ?? defaultChunking, usesPronunciationGlossary(context.config) && text.glossary?.ok ? text.glossary.entries : [], new Set(Object.keys(context.content.narrationOverrides)), context.content.narrationSources, context.content.narrationAnchors);
18
18
  // A chunk's key is its text's, so an article edit that makes a chunk say what its narration
19
19
  // edit already said (a line dropped from both) would make a new chunk to voice again. It
20
20
  // keeps the edited chunk's key instead, and with it the narration already made.
@@ -28,6 +28,13 @@ export function materializeAdmittedWork(deps, projectId) {
28
28
  .all(projectId, head)
29
29
  .map((row) => reservation.parse(row));
30
30
  const origins = new Map(reservations.map((row) => [row.origin_revision, row]));
31
+ // The head's view, read once: each narration part used to read the whole version again.
32
+ let headView = null;
33
+ const headOnce = () => {
34
+ if (headView === null)
35
+ headView = getRevisionView(deps, projectId, head);
36
+ return headView;
37
+ };
31
38
  for (const origin of origins.values()) {
32
39
  const view = executionView(deps, projectId, origin.origin_revision);
33
40
  if (view === undefined)
@@ -40,7 +47,7 @@ export function materializeAdmittedWork(deps, projectId) {
40
47
  if (recipe.deferred || recipe.unresolved)
41
48
  continue;
42
49
  if (recipe.input.kind === "tts") {
43
- const current = getRevisionView(deps, projectId, head);
50
+ const current = headOnce();
44
51
  const desired = current?.revision.fingerprints[`${recipe.input.logicalKey}:1`];
45
52
  if (desired !== undefined && desired !== recipe.logicalFingerprint)
46
53
  continue;
@@ -46,7 +46,9 @@ export function executionStages(deps, projectId) {
46
46
  !reviewsNarration(view.revision.config))
47
47
  return false;
48
48
  if (piece.input.kind === "deferred" && piece.key.endsWith(":future")) {
49
- const cacheKey = String(row.id);
49
+ // One plan per version and saved catalogue: every placeholder of the same version
50
+ // asked for the same plan again, a whole-project rebuild each time.
51
+ const cacheKey = `${String(row.revision_id)}\u0000${String(row.recipe_context)}`;
50
52
  let deferred = futureKeys.get(cacheKey);
51
53
  if (deferred === undefined) {
52
54
  const origin = executionView(deps, projectId, String(row.revision_id));
@@ -197,6 +199,8 @@ function latestRetry(deps, group) {
197
199
  }
198
200
  return latest;
199
201
  }
202
+ // The stage rows brought up to date, and the steps they were derived from, so the runner's
203
+ // tick uses them rather than deriving the whole plan again.
200
204
  export function projectStandings(deps, projectId) {
201
205
  // One derivation serves every stage: each builds the whole execution plan.
202
206
  const work = executionStages(deps, projectId);
@@ -227,6 +231,7 @@ export function projectStandings(deps, projectId) {
227
231
  .run(stage.state, textOf(failure?.failure_reason), textOf(failure?.failure_kind), stage.retryAt ?? null, invocations.length === 0 ? null : current, invocations.length === 0 ? null : invocations.length, stage.state, deps.clock.now().toISOString(), projectId, stage.kind);
228
232
  }
229
233
  settleScheduleRunsForProject(deps.db, projectId, deps.clock.now().toISOString());
234
+ return work;
230
235
  }
231
236
  // Progress moves one number and nothing else. The stage row already counts this invocation's
232
237
  // last fraction, from the claim or the previous event, so it takes the difference instead
@@ -1,11 +1,45 @@
1
1
  import { isDeepStrictEqual } from "node:util";
2
- import { narrationRegenerationKey } from "../narration/plan.js";
2
+ import { defaultChunking } from "../narration/chunk.js";
3
+ import { narrationRegenerationKey, normalizeNarrationText } from "../narration/plan.js";
4
+ import { chunkingFingerprintOf, narrationChunkTexts, } from "../narration/pronunciation-chunks.js";
3
5
  import { bodyNarrationGroups } from "../rebuild/recipe-audio.js";
4
6
  import { buildRecipes } from "../rebuild/recipe-build.js";
5
7
  import { normalizeArticleIntent, validateRevisionEdit } from "../rebuild/recipe-save.js";
6
8
  import { textRecipes } from "../rebuild/recipe-text.js";
9
+ // The chunks to keep through this save: when the narration text changes, the chunks it had
10
+ // before (themselves kept from earlier edits), so only the edited stretches are cut and voiced
11
+ // again; when it doesn't, the ones already kept. None once the chunk size changes.
12
+ function narrationAnchorsFor(base, edit, content) {
13
+ const before = base.revision.config.chunking ?? defaultChunking;
14
+ const after = edit.config.chunking ?? defaultChunking;
15
+ const chunkingFingerprint = chunkingFingerprintOf(after);
16
+ if (chunkingFingerprint !== chunkingFingerprintOf(before))
17
+ return undefined;
18
+ const previous = {
19
+ config: base.revision.config,
20
+ content: base.revision.content,
21
+ manifest: base,
22
+ resolved: {
23
+ articleMarkdown: base.articleMarkdown,
24
+ researchNotes: base.revision.config.provided.research ?? null,
25
+ },
26
+ };
27
+ const old = textRecipes(previous).narrationText;
28
+ const next = textRecipes({ ...previous, config: edit.config, content }).narrationText;
29
+ if (old === null || next === null)
30
+ return undefined;
31
+ const kept = base.revision.content.narrationAnchors;
32
+ if (normalizeNarrationText(old) === normalizeNarrationText(next))
33
+ return kept?.chunkingFingerprint === chunkingFingerprint ? kept : undefined;
34
+ return {
35
+ chunkingFingerprint,
36
+ texts: narrationChunkTexts(normalizeNarrationText(old), before, kept),
37
+ };
38
+ }
7
39
  export function bindNarrationSources(base, edit) {
8
- const { narrationSources: _submitted, ...content } = normalizeArticleIntent(base, edit);
40
+ const { narrationSources: _submitted, narrationAnchors: _sent, ...intent } = normalizeArticleIntent(base, edit);
41
+ const anchors = narrationAnchorsFor(base, edit, intent);
42
+ const content = anchors === undefined ? intent : { ...intent, narrationAnchors: anchors };
9
43
  const keys = new Set([
10
44
  ...Object.keys(content.narrationOverrides),
11
45
  ...Object.keys(base.revision.content.regenerationTokens),
@@ -91,6 +91,14 @@ export const revisionContentSchema = z
91
91
  .refine((rows) => Object.keys(rows).length <= 10000 &&
92
92
  Object.values(rows).reduce((length, row) => length + row.text.length, 0) <= 500000, "Narration source bindings exceed the article limit.")
93
93
  .optional(),
94
+ narrationAnchors: z
95
+ .object({
96
+ chunkingFingerprint: z.string().regex(/^[0-9a-f]{64}$/u),
97
+ texts: z.array(z.string().min(1).max(500000)).max(10000),
98
+ })
99
+ .strict()
100
+ .refine((row) => row.texts.reduce((length, text) => length + text.length, 0) <= 500000, "Narration anchors exceed the article limit.")
101
+ .optional(),
94
102
  subtitleCues: z
95
103
  .object({
96
104
  audioFingerprint: z.string().min(1),