@gobing-ai/knowledge-kit 0.0.15 → 0.0.17

package/plugins/kk/scripts/kk-workflow-stages.ts

@@ -50,6 +50,10 @@ const VALIDATE_SIDECAR = 'validate-voicescript.ts';
 const WRAP_SIDECAR = 'wrap-voicescript-doc.ts';
 const RENDER_SIDECAR = 'render-md.ts';
 const REVISE_COUNT = '.revise-count';
+// Post-translation title-dedup threshold (task 0138 R3). ponytail: the 2026-09-16 Gemini 3.8 Live
+// pair measures 0.417 zh title-only Jaccard, the nearest distinct pair 0.1 (4x margin). Character
+// bigrams need same-language input — pre-translate dedup over the mixed-language blend cannot work.
+const DEDUP_TITLE_THRESHOLD = 0.4;
 
 function fail(message: string): never {
     console.error(message);
@@ -643,6 +647,72 @@ function dailyQcContent(args: string[]): void {
     });
 }
 
+/**
+ * Character-bigram Jaccard over whitespace-normalized, lowercased titles — the twin of
+ * episode-plan-gen's `titleSimilarity`. Duplicated on purpose: this file is node-builtins-only
+ * (no workspace imports), so it cannot reach that module.
+ */
+function titleSimilarity(a: string, b: string): number {
+    const bigrams = (text: string): Set<string> => {
+        const normalized = text.replace(/\s+/g, ' ').trim().toLowerCase();
+        const out = new Set<string>();
+        for (let i = 0; i < normalized.length - 1; i++) out.add(normalized.slice(i, i + 2));
+        return out;
+    };
+    const aGrams = bigrams(a);
+    const bGrams = bigrams(b);
+    if (aGrams.size === 0 || bGrams.size === 0) return 0;
+    let intersection = 0;
+    for (const gram of aGrams) if (bGrams.has(gram)) intersection++;
+    return intersection / (aGrams.size + bGrams.size - intersection);
+}
+
+/**
+ * Post-translation near-duplicate dedup (task 0138 R3): drop any plan item whose title is a
+ * >=DEDUP_TITLE_THRESHOLD bigram match for an EARLIER item, so curated plan order decides the
+ * survivor. It runs on the translated plan because cross-language near-dups are invisible before
+ * plan-translate (the 2026-09-16 Gemini pair measures 0.208 mixed-language, 0.417 all-zh).
+ */
+export function dedupPlanByTitle<T extends { id?: unknown; title?: unknown }>(
+    docs: T[],
+): { kept: T[]; dropped: string[] } {
+    const kept: T[] = [];
+    const dropped: string[] = [];
+    for (const doc of docs) {
+        const title = typeof doc.title === 'string' ? doc.title : '';
+        const duplicate =
+            title !== '' &&
+            kept.some((earlier) => {
+                const earlierTitle = typeof earlier.title === 'string' ? earlier.title : '';
+                return titleSimilarity(earlierTitle, title) >= DEDUP_TITLE_THRESHOLD;
+            });
+        if (duplicate) {
+            dropped.push(typeof doc.id === 'string' && doc.id !== '' ? doc.id : title);
+            continue;
+        }
+        kept.push(doc);
+    }
+    return { kept, dropped };
+}
+
+/**
+ * Run the VoiceScript content validator over a written script and fail the stage when it rejects
+ * (task 0139 R3). `spawnSync` rather than `delegate` on purpose: `delegate` exits the process
+ * directly, which would skip `timed`'s `finally` and lose the step-timing entry the run report
+ * reads. The sidecar's own stderr carries `content check failed: segment <idx> …`; this adds the
+ * stage-level naming so the operator sees which artifact was rejected.
+ */
+function assertScriptContent(scriptPath: string): void {
+    // `spawnSync` + `failWith`, deliberately NOT `delegate`: delegate is `never`-returning (it calls
+    // process.exit), so using it mid-function would abort the calling stage on SUCCESS — dailyWrapDocs
+    // would exit before packaging its docs. A clean exit here falls through to the rest of the stage.
+    const sidecar = requireSidecar('', VALIDATE_SIDECAR, 'validate_script');
+    const runner = spawnSync(process.execPath, [sidecar, '--in', scriptPath], { stdio: 'inherit' });
+    if (runner.status !== 0) {
+        failWith(runner.status ?? 1, `script content check failed: ${scriptPath} was rejected (see above)`);
+    }
+}
+
 function dailyArticle(args: string[]): void {
     const [workDir = '', runDate = '', pluginsPath = ''] = args;
     const articleMd = join(workDir, '2-article', `${runDate}_06_article_article.md`);
@@ -650,10 +720,6 @@ function dailyArticle(args: string[]): void {
     const planIn = join(workDir, '2-plan', `${runDate}_03_plan_plan.json`);
     const candidates = join(workDir, '2-plan', `${runDate}_04_quality-control-content_candidates.json`);
     timed(workDir, 'article', () => {
-        if (isFile(articleMd)) {
-            console.log(`article cached: ${articleMd}`);
-            return;
-        }
         let planUsable = true;
         const raw = isFile(planIn) ? readFileSync(planIn, 'utf-8') : '';
         try {
@@ -671,6 +737,21 @@ function dailyArticle(args: string[]): void {
             }
         }
         if (!planUsable) failWith(1, 'article: no usable plan input');
+        // R3 (task 0138): dedup runs BEFORE the cache early-return — the script stage reads this same
+        // plan.json and must see one survivor even when the article is already cached. Idempotent:
+        // a no-op once the plan is deduped.
+        const planParsed: unknown = readJson(planIn);
+        if (Array.isArray(planParsed)) {
+            const { kept, dropped } = dedupPlanByTitle(planParsed as Record<string, unknown>[]);
+            if (dropped.length > 0) {
+                sted('write', planIn, () => writeJsonText(planIn, kept));
+                console.error(`article: dropped ${dropped.length} near-duplicate plan item(s): ${dropped.join(', ')}`);
+            }
+        }
+        if (isFile(articleMd)) {
+            console.log(`article cached: ${articleMd}`);
+            return;
+        }
         kk(
             [
                 'executor',
@@ -753,6 +834,9 @@ function dailyScript(args: string[]): void {
         const body = readJson(contentOut)?.body;
         if (typeof body !== 'string') failWith(1, `script: ${contentOut} has no string body`);
         sted('write', yamlOut, () => writeRaw(yamlOut, body));
+        // R3 (task 0139): fresh writes are validated here; the script cache path is covered by
+        // wrap-docs, which runs on every path that reaches audio.
+        assertScriptContent(yamlOut);
     });
 }
 
@@ -761,6 +845,9 @@ function dailyWrapDocs(args: string[]): void {
     const scriptPath = join(workDir, '2-script', `${runDate}_08_script_voicescript.yaml`);
     const docsPath = join(workDir, '3-audio', `${runDate}_10_wrap-docs_docs.json`);
     timed(workDir, 'wrap-docs', () => {
+        // R3 (task 0139): the second validator placement — covers a HITL-edited YAML and the
+        // script-stage cache path, neither of which re-enters dailyScript's fresh-write branch.
+        assertScriptContent(scriptPath);
         const url = new URL(scriptPath, `file://${process.cwd()}/`);
         const body = readFileSync(decodeURIComponent(url.pathname), 'utf-8');
         const profile = voiceProfile || process.env.VOICEBOX_DEFAULT_PROFILE || 'robin-news';
@@ -1033,6 +1120,8 @@ function dailyRunReport(args: string[]): void {
                     `${runDate}_04_quality-control-content_candidates.rejected.json`,
                 ),
                 NEWS_REPORT_TIMING_FILE: join(workDir, 'step-timing.json'),
+                // 0139 R5b: the generate stage's QC audit, read best-effort (absent -> section omitted).
+                NEWS_REPORT_QC_FILE: join(workDir, '3-audio', `${runDate}_11_generate_content.json`),
                 NEWS_REPORT_DATE: runDate,
             },
         });
@@ -1118,109 +1207,121 @@ function dailyTranslateSync(args: string[]): void {
 
 const [stage, ...rest] = process.argv.slice(2);
 
-switch (stage) {
-    case 'prepare-itc':
-        prepareItc(rest);
-        break;
-    case 'prepare-solo':
-        prepareSolo(rest);
-        break;
-    case 'prepare-storm':
-        prepareStorm(rest);
-        break;
-    case 'duration':
-        duration(rest);
-        break;
-    case 'validate': {
-        const { override, values } = parseStage(rest, { in: { type: 'string' } });
-        if (!values.in) fail('usage: kk-workflow-stages.ts validate [<override>] --in <voicescript.yaml>');
-        delegate(requireSidecar(override, VALIDATE_SIDECAR, 'validate_script'), ['--in', values.in]);
-        break;
-    }
-    case 'wrap': {
-        const { override, values } = parseStage(rest, {
-            in: { type: 'string' },
-            out: { type: 'string' },
-            profile: { type: 'string' },
-        });
-        if (!values.in || !values.out) {
-            fail(
-                'usage: kk-workflow-stages.ts wrap [<override>] --in <voicescript.yaml> --out <docs.json> [--profile <name>]',
-            );
+// Imported as a module (unit tests reach `dedupPlanByTitle`) → exports only, never a stage run;
+// executed as the entrypoint → dispatch.
+if (import.meta.main)
+    switch (stage) {
+        case 'prepare-itc':
+            prepareItc(rest);
+            break;
+        case 'prepare-solo':
+            prepareSolo(rest);
+            break;
+        case 'prepare-storm':
+            prepareStorm(rest);
+            break;
+        case 'duration':
+            duration(rest);
+            break;
+        case 'validate': {
+            const { override, values } = parseStage(rest, { in: { type: 'string' } });
+            if (!values.in) fail('usage: kk-workflow-stages.ts validate [<override>] --in <voicescript.yaml>');
+            delegate(requireSidecar(override, VALIDATE_SIDECAR, 'validate_script'), ['--in', values.in]);
+            break;
         }
-        delegate(requireSidecar(override, WRAP_SIDECAR, 'wrap_script'), [values.in, values.out, values.profile ?? '']);
-        break;
-    }
-    case 'render-storm': {
-        const { override, values } = parseStage(rest, { in: { type: 'string' }, out: { type: 'string' } });
-        if (!values.in || !values.out) {
-            fail('usage: kk-workflow-stages.ts render-storm [<override>] --in <content.json> --out <content.md>');
+        case 'wrap': {
+            const { override, values } = parseStage(rest, {
+                in: { type: 'string' },
+                out: { type: 'string' },
+                profile: { type: 'string' },
+            });
+            if (!values.in || !values.out) {
+                fail(
+                    'usage: kk-workflow-stages.ts wrap [<override>] --in <voicescript.yaml> --out <docs.json> [--profile <name>]',
+                );
+            }
+            delegate(requireSidecar(override, WRAP_SIDECAR, 'wrap_script'), [
+                values.in,
+                values.out,
+                values.profile ?? '',
+            ]);
+            break;
+        }
+        case 'render-storm': {
+            const { override, values } = parseStage(rest, { in: { type: 'string' }, out: { type: 'string' } });
+            if (!values.in || !values.out) {
+                fail('usage: kk-workflow-stages.ts render-storm [<override>] --in <content.json> --out <content.md>');
+            }
+            delegate(requireSidecar(override, RENDER_SIDECAR, 'render_script'), [
+                '--in',
+                values.in,
+                '--out',
+                values.out,
+            ]);
+            break;
         }
-        delegate(requireSidecar(override, RENDER_SIDECAR, 'render_script'), ['--in', values.in, '--out', values.out]);
-        break;
+        case 'daily-prepare':
+            dailyPrepare(rest);
+            break;
+        case 'daily-collect-facts':
+            dailyCollectFacts(rest);
+            break;
+        case 'daily-plan':
+            dailyPlan(rest);
+            break;
+        case 'daily-qc-content':
+            dailyQcContent(rest);
+            break;
+        case 'daily-article':
+            dailyArticle(rest);
+            break;
+        case 'daily-cover-normalize':
+            dailyCoverNormalize(rest);
+            break;
+        case 'daily-script':
+            dailyScript(rest);
+            break;
+        case 'daily-wrap-docs':
+            dailyWrapDocs(rest);
+            break;
+        case 'daily-generate':
+            dailyGenerate(rest);
+            break;
+        case 'daily-quality-report':
+            dailyQualityReport(rest);
+            break;
+        case 'daily-publish-prep':
+            dailyPublishPrep(rest);
+            break;
+        case 'daily-publish-surfdash':
+            dailyPublishSurfdash(rest);
+            break;
+        case 'daily-show-notes':
+            dailyShowNotes(rest);
+            break;
+        case 'daily-publish-podcast':
+            dailyPublishPodcast(rest);
+            break;
+        case 'daily-publish-classify':
+            dailyPublishClassify(rest);
+            break;
+        case 'daily-publish-partial':
+            dailyPublishPartial(rest);
+            break;
+        case 'daily-run-report':
+            dailyRunReport(rest);
+            break;
+        case 'daily-translate-en':
+            dailyTranslateEn(rest);
+            break;
+        case 'daily-translate-ja':
+            dailyTranslateJa(rest);
+            break;
+        case 'daily-translate-sync':
+            dailyTranslateSync(rest);
+            break;
+        default:
+            fail(
+                `unknown stage: ${stage ?? '(none)'} (expected prepare-itc|prepare-solo|prepare-storm|validate|wrap|duration|render-storm|daily-*)`,
+            );
     }
-    case 'daily-prepare':
-        dailyPrepare(rest);
-        break;
-    case 'daily-collect-facts':
-        dailyCollectFacts(rest);
-        break;
-    case 'daily-plan':
-        dailyPlan(rest);
-        break;
-    case 'daily-qc-content':
-        dailyQcContent(rest);
-        break;
-    case 'daily-article':
-        dailyArticle(rest);
-        break;
-    case 'daily-cover-normalize':
-        dailyCoverNormalize(rest);
-        break;
-    case 'daily-script':
-        dailyScript(rest);
-        break;
-    case 'daily-wrap-docs':
-        dailyWrapDocs(rest);
-        break;
-    case 'daily-generate':
-        dailyGenerate(rest);
-        break;
-    case 'daily-quality-report':
-        dailyQualityReport(rest);
-        break;
-    case 'daily-publish-prep':
-        dailyPublishPrep(rest);
-        break;
-    case 'daily-publish-surfdash':
-        dailyPublishSurfdash(rest);
-        break;
-    case 'daily-show-notes':
-        dailyShowNotes(rest);
-        break;
-    case 'daily-publish-podcast':
-        dailyPublishPodcast(rest);
-        break;
-    case 'daily-publish-classify':
-        dailyPublishClassify(rest);
-        break;
-    case 'daily-publish-partial':
-        dailyPublishPartial(rest);
-        break;
-    case 'daily-run-report':
-        dailyRunReport(rest);
-        break;
-    case 'daily-translate-en':
-        dailyTranslateEn(rest);
-        break;
-    case 'daily-translate-ja':
-        dailyTranslateJa(rest);
-        break;
-    case 'daily-translate-sync':
-        dailyTranslateSync(rest);
-        break;
-    default:
-        fail(
-            `unknown stage: ${stage ?? '(none)'} (expected prepare-itc|prepare-solo|prepare-storm|validate|wrap|duration|render-storm|daily-*)`,
-        );
-}

package/plugins/kk/scripts/validate-voicescript.ts

@@ -118,6 +118,59 @@ function assertCrossfade(value: number | undefined): void {
     }
 }
 
+// ─── Content checks (task 0139 R3) ───────────────────────────────────────────
+// Deterministic spoken-copy gates. The workflow wires this validator at the end of the script
+// stage's fresh write and again in wrap-docs (which covers a HITL-edited YAML and the script cache
+// path), so a voicescript that reaches audio carries no title echo, no duplicated segment, no
+// research scaffolding and no colon-mangled URL. Every failure names the segment index and reason.
+
+/** Punctuation/space the translator drifts between a title and its body restatement. */
+const CONTENT_ECHO_NOISE_RE = /[，。、：:；;!！?？…\s「」『』【】]/g;
+/** Research scaffold labels the plan-translate stage must not emit as spoken copy. */
+const CONTENT_SCAFFOLD_LABEL_RE =
+    /^(?:\*\*)?[「『]?(?:背景|潜在影响|可做角度|为什么现在值得注意|社区讨论)[」』]?(?:\*\*)?(?:[：:，,]|\s|$)/;
+/** `标签，` metadata tails — the plan item's tag line spoken aloud. */
+const CONTENT_TAG_TAIL_RE = /标签[：:，,]/;
+/** `wss，//` / `https。//` — the translator replacing a URL colon with a Chinese comma. */
+const CONTENT_URL_MANGLE_RE = /[，。]\/\//;
+
+function assertSegmentContent(script: VoiceScript): void {
+    const seen = new Map<string, number>();
+    for (let idx = 0; idx < script.segments.length; idx++) {
+        const text = (script.segments[idx]?.text ?? '').trim();
+        if (text === '') continue;
+
+        const previous = seen.get(text);
+        if (previous !== undefined) {
+            throw new Error(`content check failed: segment ${idx} duplicates segment ${previous} verbatim`);
+        }
+        seen.set(text, idx);
+
+        // Title echo: the 【lead】 content reappearing immediately after the lead, punctuation-drift
+        // tolerant — the same comparison `stripLeadingTitleEcho` uses on the producer side.
+        const lead = text.match(/^[^【]*【([^】]{4,80})】[，。、\s]*/);
+        if (lead !== null) {
+            const bracketed = (lead[1] ?? '').replace(CONTENT_ECHO_NOISE_RE, '');
+            const rest = text.slice(lead[0].length).replace(CONTENT_ECHO_NOISE_RE, '');
+            if (bracketed.length >= 4 && rest.startsWith(bracketed)) {
+                throw new Error(
+                    `content check failed: segment ${idx} repeats its 【${lead[1]}】 title in the body text`,
+                );
+            }
+        }
+
+        if (CONTENT_SCAFFOLD_LABEL_RE.test(text)) {
+            throw new Error(`content check failed: segment ${idx} opens with a research scaffold label`);
+        }
+        if (CONTENT_TAG_TAIL_RE.test(text)) {
+            throw new Error(`content check failed: segment ${idx} carries a 标签 metadata tail`);
+        }
+        if (CONTENT_URL_MANGLE_RE.test(text)) {
+            throw new Error(`content check failed: segment ${idx} has a colon-mangled URL`);
+        }
+    }
+}
+
 function validateVoiceScript(script: VoiceScript): void {
     if (!script || typeof script !== 'object') {
         throw new Error('VoiceScript must be an object');
@@ -173,6 +226,8 @@ function validateVoiceScript(script: VoiceScript): void {
         assertEngine(segment.engine, `Segment at index ${idx}`);
         assertLanguage(segment.language, `Segment at index ${idx}`);
     }
+
+    assertSegmentContent(script);
 }
 
 function main(): void {

package/plugins/kk/workflows/kk-daily-ai-voice.yaml

@@ -121,12 +121,14 @@ states:
         options:
           role: scribe
           agent: ${vars.agent}
+          # ADR-115 error tier: keep this prompt <= 1000 chars (0133 R3; 0138 R4a adds the copy discipline).
           input: |
-            Read ${vars.work_dir}/2-plan/${vars.run_date}_04_quality-control-content_candidates.json — its metadata.docs (Doc[]) is the episode plan to translate. Do not edit candidates.json.
-            For every item with needsTranslation, translate "title" and "body" into natural, concise simplified Chinese.
-            Titles must read as a news event (subject + action + object), derived from the body — e.g. "OpenAI 发布 Sora 应用，集成 Sora 2 模型". Keep product/model/company names verbatim, preserve URLs inside the body, and use 「」 instead of raw ASCII double quotes inside Chinese text.
-            Keep every other field untouched (id, sourceUri, mediaType, metadata), keep technical terms in their original form where that is clearer, and produce no trailing commas or comments.
-            Output strictly valid JSON — a same-length, same-order array carrying the same ids — to ${vars.work_dir}/2-plan/${vars.run_date}_03_plan_plan.json.
+            Read ${vars.work_dir}/2-plan/${vars.run_date}_04_quality-control-content_candidates.json — metadata.docs (Doc[]) is the plan; don't edit it.
+            For items with needsTranslation, translate "title" and "body" into natural, concise simplified Chinese.
+            Titles read as a news event (subject + action + object) from the body — e.g. "OpenAI 发布 Sora 应用，集成 Sora 2 模型". Keep product/model/company names verbatim, preserve body URLs, use 「」 not ASCII double quotes.
+            Copy discipline: reader-facing news copy — no scaffold labels (「可做角度」「为什么现在值得注意」「社区讨论」「背景」「潜在影响」「影响」), no duplicated labels or hedging (「读者宜另行核实」「仍待验证」) — merge their substance into prose or drop.
+            Keep other fields (id, sourceUri, mediaType, metadata), technical terms as-is; no trailing commas or comments.
+            Output strictly valid JSON — a same-length, same-order array with the same ids — to ${vars.work_dir}/2-plan/${vars.run_date}_03_plan_plan.json.
           expectFile: ${vars.work_dir}/2-plan/${vars.run_date}_03_plan_plan.json
           allowAppend: false
 
@@ -518,27 +520,30 @@ transitions:
     guard:
       kind: always
 
+  # 0140: intentionally NO `qc.passed` conjunct on these two edges (operator decision, ratified
+  # 2026-09-17, see the 0140 Q&A). The gate that used to sit here read
+  # `jq -r '.metadata.qc.passed // true' <generate content>` — and jq's `//` treats `false` as empty,
+  # so it printed "true" for a failing audit and the conjunct passed for every input. It never gated
+  # (trace `dogfood-daily-031816` fired quality-control → publish-prep with `passed:false` on disk),
+  # so removing it is behaviour-neutral. Policy: degradation-class audits (loudness dips,
+  # transcription fidelity) publish warn-only and surface in the run report's ## 音频 QC block;
+  # the credibility-damaging class (`repetition_detected`) is hard-failed upstream by 0139 R5a.
+  # Do not restore a `qc.passed` guard here without also fixing the `// false`-swallowing expression.
   - from: quality-control
     to: publish-prep
-    description: "publish enabled AND QC passed -> merge article + audio for the publish seam"
+    description: "publish enabled -> merge article + audio for the publish seam"
     guard:
       kind: shell
       options:
-        command: 'test "${vars.publish_enabled}" = "true" && test "$(jq -r ''.metadata.qc.passed // true'' "${vars.work_dir}/3-audio/${vars.run_date}_11_generate_content.json")" = "true"'
+        command: 'test "${vars.publish_enabled}" = "true"'
 
   - from: quality-control
     to: done
-    description: "publish disabled AND QC passed -> audio episode complete (no publish)"
+    description: "publish disabled -> audio episode complete (no publish)"
     guard:
       kind: shell
       options:
-        command: 'test "${vars.publish_enabled}" != "true" && test "$(jq -r ''.metadata.qc.passed // true'' "${vars.work_dir}/3-audio/${vars.run_date}_11_generate_content.json")" = "true"'
-
-  - from: quality-control
-    to: failed
-    description: "Audio QC failed -> pipeline failed"
-    guard:
-      kind: always
+        command: 'test "${vars.publish_enabled}" != "true"'
 
   - from: publish-prep
     to: publish-surfdash