@gobing-ai/knowledge-kit 0.0.7 → 0.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gobing-ai/knowledge-kit",
-    "version": "0.0.7",
+    "version": "0.0.9",
     "description": "An ingest → create → publish content pipeline CLI (Bun).",
     "repository": {
         "type": "git",
@@ -28,6 +28,9 @@
         "typecheck": "tsc --noEmit"
     },
     "dependencies": {
+        "@gobing-ai/ts-infra": "^0.4.39",
+        "@gobing-ai/ts-runtime": "^0.4.39",
+        "@gobing-ai/ts-utils": "^0.4.39",
         "commander": "15.0.0"
     },
     "devDependencies": {

package/plugins/generations/content-gen/package.json

@@ -7,7 +7,8 @@
     },
     "dependencies": {
         "@gobing-ai/kk-core": "workspace:*",
-        "@gobing-ai/utils": "workspace:*",
+        "@gobing-ai/ts-runtime": "catalog:",
+        "@gobing-ai/ts-utils": "catalog:",
         "zod": "4.4.3"
     },
     "devDependencies": {

package/plugins/generations/content-gen/src/index.ts

@@ -1,7 +1,7 @@
-import { readFile, rm, writeFile } from 'node:fs/promises';
 import { parseArgs } from 'node:util';
 import { type Content, ContentSchema, type Doc, DocListSchema } from '@gobing-ai/kk-core';
-import { logger } from '@gobing-ai/utils';
+import { createNodeFileSystem } from '@gobing-ai/ts-runtime';
+import { echoError } from '@gobing-ai/ts-utils';
 import { generateWithStormPipeline } from './storm';
 
 type ContentGenerator = (docs: Doc[], inputPath: string) => Promise<Content>;
@@ -19,10 +19,11 @@ export async function processGeneratorIO(
     outputPath: string,
     generate: ContentGenerator = generateWithStormPipeline,
 ): Promise<void> {
-    await rm(outputPath, { force: true });
-    const docs = DocListSchema.parse(JSON.parse(await readFile(inputPath, 'utf-8')));
+    const fs = createNodeFileSystem();
+    await fs.deleteFile(outputPath);
+    const docs = DocListSchema.parse(JSON.parse(await fs.readFile(inputPath)));
     const content = docs.length === 0 ? EMPTY_CONTENT : ContentSchema.parse(await generate(docs, inputPath));
-    await writeFile(outputPath, JSON.stringify(content, null, 2), 'utf-8');
+    await fs.writeFile(outputPath, JSON.stringify(content, null, 2));
 }
 
 export async function main(): Promise<number> {
@@ -33,7 +34,7 @@ export async function main(): Promise<number> {
         },
     });
     if (!values.in || !values.out) {
-        logger.error('Missing required arguments: --in and --out');
+        echoError('Missing required arguments: --in and --out');
         return 1;
     }
 
@@ -41,8 +42,8 @@ export async function main(): Promise<number> {
         await processGeneratorIO(values.in, values.out);
         return 0;
     } catch (error: unknown) {
-        logger.error(`content-gen failed: ${error instanceof Error ? error.message : String(error)}`);
-        await rm(values.out, { force: true });
+        echoError(`content-gen failed: ${error instanceof Error ? error.message : String(error)}`);
+        await createNodeFileSystem().deleteFile(values.out);
         return 1;
     }
 }

package/plugins/generations/content-gen/src/storm.ts

@@ -113,8 +113,13 @@ function stageError(stage: string, detail: string): Error {
     return new Error(`${stage} stage failed: ${detail}`);
 }
 
+/** Strip markdown footnote markers agents prepend to citation ids (`^doc-1` → `doc-1`). */
+function bareCitationId(id: string): string {
+    return id.replace(/^\^+/, '');
+}
+
 function assertKnownEvidence(stage: string, ids: string[], allowed: Set<string>): void {
-    const unknown = ids.filter((id) => !allowed.has(id));
+    const unknown = ids.map(bareCitationId).filter((id) => !allowed.has(id));
     if (unknown.length > 0) {
         throw stageError(stage, `cites unknown document ID(s): ${[...new Set(unknown)].join(', ')}`);
     }
@@ -237,6 +242,7 @@ export async function articleStage(
             `The validated curation result for the article is: ${JSON.stringify(curation)}`,
             `The validated outline for the article is: ${JSON.stringify(outline)}`,
             'Write the complete article as markdown, one section per outline heading. Every citation must reference one of the allowed document IDs.',
+            'Synthesize: the body must draw findings from across the supplied documents, compare them, and state conclusions. Never list sources one per line or produce an index of references — references are evidence, not the deliverable.',
             'Return only a JSON object matching {"title":string,"body":string,"citations":[string]} where body is the full markdown draft and every citations entry is one of the allowed document IDs.',
         ].join('\n'),
     );
@@ -289,9 +295,11 @@ export async function polishStage(
     // Machine-verify the preservation mandate: every heading and citation in the draft must
     // survive, and polish must not introduce citations to unknown document IDs.
     const polishedHeadings = bodyHeadings(result.body);
-    const polishedCitations = bodyCitations(result.body);
+    const polishedCitations = bodyCitations(result.body).map(bareCitationId);
     const droppedHeadings = bodyHeadings(article.body).filter((heading) => !polishedHeadings.includes(heading));
-    const droppedCitations = bodyCitations(article.body).filter((citation) => !polishedCitations.includes(citation));
+    const droppedCitations = bodyCitations(article.body)
+        .map(bareCitationId)
+        .filter((citation) => !polishedCitations.includes(citation));
     if (droppedHeadings.length > 0 || droppedCitations.length > 0) {
         const dropped: string[] = [];
         if (droppedHeadings.length > 0) dropped.push(`dropped heading(s): ${droppedHeadings.join(', ')}`);
@@ -320,7 +328,7 @@ export async function generateWithStormPipeline(
     return ContentSchema.parse({
         title:
             polished.title ?? article.title ?? outline.title ?? docs[0]?.title ?? 'Generated Knowledge Kit Synthesis',
-        body: polished.body,
+        body: polished.body.replace(/\[\^([^\]()\s]+)\](?!\()/g, '[$1]'),
         format: 'markdown',
         outline: polished.outline ?? outline.outline,
         references: docs.map((doc) => ({ url: doc.sourceUri, title: doc.title, cite: doc.id })),

package/plugins/generations/dailynews-gen/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "@gobing-ai/dailynews-gen",
+    "type": "module",
+    "private": true,
+    "scripts": {
+        "typecheck": "tsc --noEmit"
+    },
+    "dependencies": {
+        "@gobing-ai/kk-core": "workspace:*",
+        "@gobing-ai/ts-runtime": "catalog:",
+        "@gobing-ai/ts-utils": "catalog:",
+        "zod": "4.4.3"
+    },
+    "devDependencies": {
+        "@types/bun": "1.3.14"
+    }
+}

package/plugins/generations/dailynews-gen/plugin.json

@@ -0,0 +1,7 @@
+{
+    "name": "dailynews-gen",
+    "kind": "generator",
+    "entry": "./src/index.ts",
+    "version": "1.0.0",
+    "description": "Generates structured spoken broadcast VoiceScript YAML Content from AI news Doc[]"
+}

package/plugins/generations/dailynews-gen/src/index.ts

@@ -0,0 +1,111 @@
+import { dirname } from 'node:path';
+import { parseArgs } from 'node:util';
+import { type Content, ContentSchema, type Doc, DocListSchema } from '@gobing-ai/kk-core';
+import { createNodeFileSystem } from '@gobing-ai/ts-runtime';
+import { echoError } from '@gobing-ai/ts-utils';
+import { buildNewsVoiceScript, type ScriptBuilderOptions } from './script-builder';
+
+export * from './script-builder';
+
+export interface GeneratorOptions extends ScriptBuilderOptions {
+    in: string;
+    out: string;
+}
+
+export async function processGeneratorIO(
+    inputPath: string,
+    outputPath: string,
+    options?: ScriptBuilderOptions,
+): Promise<Content> {
+    const fs = createNodeFileSystem();
+
+    let docs: Doc[];
+    try {
+        const rawInput = await fs.readFile(inputPath);
+        docs = DocListSchema.parse(JSON.parse(rawInput));
+    } catch (err: unknown) {
+        throw new Error(`Invalid DocList input: ${err instanceof Error ? err.message : String(err)}`);
+    }
+
+    const script = buildNewsVoiceScript(docs, options);
+    const yamlBody = Bun.YAML.stringify(script);
+
+    const content: Content = {
+        title: script.title,
+        body: yamlBody,
+        format: 'markdown',
+        references: docs
+            .filter((d) => d.sourceUri)
+            .map((d) => ({
+                title: d.title,
+                url: d.sourceUri,
+            })),
+        metadata: {
+            generator: 'kk:dailynews-gen',
+            language: script.language,
+            voiceProfile: script.default_profile,
+            storyCount: docs.length,
+            stories: docs.map((d) => ({
+                id: d.id,
+                title: d.title,
+                sourceUri: d.sourceUri,
+            })),
+        },
+    };
+
+    const validatedContent = ContentSchema.parse(content);
+
+    const outDir = dirname(outputPath);
+    if (outDir && outDir !== '.') {
+        await fs.ensureDir(outDir);
+    }
+    await fs.writeFile(outputPath, JSON.stringify(validatedContent, null, 2));
+
+    return validatedContent;
+}
+
+export async function main(): Promise<number> {
+    let values: {
+        in?: string;
+        out?: string;
+        language?: string;
+        title?: string;
+        'voice-profile'?: string;
+    };
+
+    try {
+        ({ values } = parseArgs({
+            options: {
+                in: { type: 'string' },
+                out: { type: 'string' },
+                language: { type: 'string' },
+                title: { type: 'string' },
+                'voice-profile': { type: 'string' },
+            },
+        }));
+    } catch (err: unknown) {
+        echoError(`dailynews-gen failed: ${err instanceof Error ? err.message : String(err)}`);
+        return 1;
+    }
+
+    if (!values.in || !values.out) {
+        echoError('dailynews-gen failed: Missing required arguments: --in and --out');
+        return 1;
+    }
+
+    try {
+        await processGeneratorIO(values.in, values.out, {
+            language: values.language || process.env.DAILYNEWS_LANGUAGE,
+            title: values.title || process.env.DAILYNEWS_TITLE,
+            voiceProfile: values['voice-profile'] || process.env.VOICEBOX_DEFAULT_PROFILE,
+        });
+        return 0;
+    } catch (err: unknown) {
+        echoError(`dailynews-gen failed: ${err instanceof Error ? err.message : String(err)}`);
+        return 1;
+    }
+}
+
+if (import.meta.main) {
+    process.exit(await main());
+}

package/plugins/generations/dailynews-gen/src/script-builder.ts

@@ -0,0 +1,121 @@
+import type { Doc } from '@gobing-ai/kk-core';
+import { z } from 'zod';
+
+export const VoiceScriptSchema = z.object({
+    title: z.string().min(1),
+    language: z.enum(['zh', 'en']),
+    default_profile: z.string().min(1).optional(),
+    segments: z
+        .array(
+            z.object({
+                text: z.string().min(1),
+                gap_ms: z.number().int().nonnegative().optional(),
+                instruct: z.string().min(1).optional(),
+            }),
+        )
+        .min(1),
+});
+
+export interface VoiceScriptSegment {
+    text: string;
+    gap_ms?: number;
+    instruct?: string;
+}
+
+export interface VoiceScript {
+    title: string;
+    language: string;
+    default_profile?: string;
+    segments: VoiceScriptSegment[];
+}
+
+export interface ScriptBuilderOptions {
+    language?: string;
+    voiceProfile?: string;
+    title?: string;
+}
+
+export function buildNewsVoiceScript(docs: Doc[], options?: ScriptBuilderOptions): VoiceScript {
+    const language = z.enum(['zh', 'en']).parse((options?.language || 'zh').toLowerCase());
+    const isZh = language === 'zh';
+    const voiceProfile = options?.voiceProfile || process.env.VOICEBOX_DEFAULT_PROFILE || 'Robin';
+
+    if (docs.length === 0) {
+        const title = options?.title || (isZh ? '今日AI快讯（暂无更新）' : 'AI Daily Briefing (No Updates)');
+        const segments: VoiceScriptSegment[] = isZh
+            ? [
+                  {
+                      text: '大家好，欢迎收听今日AI快讯。今天暂无最新的精选新闻动态。',
+                      gap_ms: 400,
+                  },
+                  {
+                      text: '感谢收听，我们下期再见。',
+                      gap_ms: 200,
+                  },
+              ]
+            : [
+                  {
+                      text: 'Hello and welcome to your AI Daily Briefing. There are no new featured stories today.',
+                      gap_ms: 400,
+                  },
+                  {
+                      text: 'Thank you for listening, and have a great day.',
+                      gap_ms: 200,
+                  },
+              ];
+
+        return VoiceScriptSchema.parse({
+            title,
+            language,
+            default_profile: voiceProfile,
+            segments,
+        });
+    }
+
+    const title =
+        options?.title || (isZh ? `今日AI快讯（共${docs.length}条）` : `AI Daily Briefing (${docs.length} stories)`);
+    const segments: VoiceScriptSegment[] = [];
+
+    // 1. Intro segment
+    const introText = isZh
+        ? `大家好，欢迎收听今日AI快讯。今天为您精选了 ${docs.length} 条重点动态，让我们快速浏览：`
+        : `Hello and welcome to your AI Daily Briefing. Here are ${docs.length} top stories for today:`;
+
+    segments.push({
+        text: introText,
+        gap_ms: 600,
+    });
+
+    // 2. Story segments
+    docs.forEach((doc, idx) => {
+        const storyIndex = idx + 1;
+        const cleanTitle = (doc.title ?? '').trim();
+        const cleanBody = (doc.body ?? '').trim();
+
+        const storyText = isZh
+            ? `第 ${storyIndex} 条动态：${cleanTitle}。${cleanBody}`
+            : `Story number ${storyIndex}: ${cleanTitle}. ${cleanBody}`;
+
+        segments.push({
+            text: storyText,
+            gap_ms: 500,
+        });
+    });
+
+    // 3. Outro segment
+    const outroText = isZh
+        ? '以上就是今天AI快讯的全部精选内容。感谢您的收听，祝您拥有高效的一天。'
+        : 'That concludes our AI Daily Briefing for today. Thanks for listening, and have a productive day.';
+
+    segments.push({
+        text: outroText,
+        gap_ms: 300,
+    });
+
+    return VoiceScriptSchema.parse({
+        title,
+        language,
+        default_profile: voiceProfile,
+        segments,
+    });
+}

package/plugins/generations/dailynews-gen/tsconfig.json

@@ -0,0 +1,4 @@
+{
+    "extends": "../../../tooling/typescript/base.json",
+    "include": ["src", "tests"]
+}

package/plugins/generations/voice-gen/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "@gobing-ai/voice-gen",
+    "type": "module",
+    "private": true,
+    "scripts": {
+        "typecheck": "tsc --noEmit"
+    },
+    "dependencies": {
+        "@gobing-ai/kk-core": "workspace:*",
+        "@gobing-ai/ts-runtime": "catalog:",
+        "@gobing-ai/ts-utils": "catalog:",
+        "zod": "4.4.3"
+    },
+    "devDependencies": {
+        "@types/bun": "1.3.14"
+    }
+}

package/plugins/generations/voice-gen/plugin.json

@@ -0,0 +1,6 @@
+{
+    "name": "voice-gen",
+    "kind": "generator",
+    "entry": "./src/index.ts",
+    "version": "1.0.0"
+}

package/plugins/generations/voice-gen/src/concat.ts

@@ -0,0 +1,218 @@
+export interface ParsedWav {
+    audioFormat: number;
+    numChannels: number;
+    sampleRate: number;
+    byteRate: number;
+    blockAlign: number;
+    bitsPerSample: number;
+    dataBytes: Uint8Array;
+}
+
+/**
+ * Parse a RIFF WAVE buffer into its format fields and raw audio data.
+ */
+export function parseWav(buffer: Uint8Array): ParsedWav {
+    if (buffer.length < 44) {
+        throw new Error('Invalid WAV buffer: buffer too short');
+    }
+
+    const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+
+    const b0 = buffer[0];
+    const b1 = buffer[1];
+    const b2 = buffer[2];
+    const b3 = buffer[3];
+    const b8 = buffer[8];
+    const b9 = buffer[9];
+    const b10 = buffer[10];
+    const b11 = buffer[11];
+
+    if (
+        b0 === undefined ||
+        b1 === undefined ||
+        b2 === undefined ||
+        b3 === undefined ||
+        b8 === undefined ||
+        b9 === undefined ||
+        b10 === undefined ||
+        b11 === undefined
+    ) {
+        throw new Error('Invalid WAV buffer: buffer too short');
+    }
+
+    const riffTag = String.fromCharCode(b0, b1, b2, b3);
+    const waveTag = String.fromCharCode(b8, b9, b10, b11);
+
+    if (riffTag !== 'RIFF' || waveTag !== 'WAVE') {
+        throw new Error(`Invalid WAV buffer: expected RIFF/WAVE header, got ${riffTag}/${waveTag}`);
+    }
+
+    let offset = 12;
+    let formatInfo:
+        | {
+              audioFormat: number;
+              numChannels: number;
+              sampleRate: number;
+              byteRate: number;
+              blockAlign: number;
+              bitsPerSample: number;
+          }
+        | undefined;
+    let dataBytes: Uint8Array | undefined;
+
+    while (offset + 8 <= buffer.length) {
+        const c0 = buffer[offset];
+        const c1 = buffer[offset + 1];
+        const c2 = buffer[offset + 2];
+        const c3 = buffer[offset + 3];
+        if (c0 === undefined || c1 === undefined || c2 === undefined || c3 === undefined) {
+            break;
+        }
+
+        const chunkId = String.fromCharCode(c0, c1, c2, c3);
+        const chunkSize = view.getUint32(offset + 4, true);
+        const chunkDataOffset = offset + 8;
+
+        if (chunkId === 'fmt ') {
+            if (chunkSize < 16) {
+                throw new Error('Invalid WAV fmt chunk size');
+            }
+            const audioFormat = view.getUint16(chunkDataOffset, true);
+            const numChannels = view.getUint16(chunkDataOffset + 2, true);
+            const sampleRate = view.getUint32(chunkDataOffset + 4, true);
+            const byteRate = view.getUint32(chunkDataOffset + 8, true);
+            const blockAlign = view.getUint16(chunkDataOffset + 12, true);
+            const bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
+
+            formatInfo = {
+                audioFormat,
+                numChannels,
+                sampleRate,
+                byteRate,
+                blockAlign,
+                bitsPerSample,
+            };
+        } else if (chunkId === 'data') {
+            const end = Math.min(chunkDataOffset + chunkSize, buffer.length);
+            dataBytes = buffer.subarray(chunkDataOffset, end);
+        }
+
+        // Advance to next chunk (aligned to 2 bytes)
+        offset = chunkDataOffset + chunkSize + (chunkSize % 2 === 1 ? 1 : 0);
+    }
+
+    if (!formatInfo) {
+        throw new Error('Invalid WAV: missing "fmt " chunk');
+    }
+    if (!dataBytes) {
+        throw new Error('Invalid WAV: missing "data" chunk');
+    }
+
+    return {
+        ...formatInfo,
+        dataBytes,
+    };
+}
+
+/**
+ * Concatenate multiple WAV audio buffers in order, inserting gap_ms of silence before clips.
+ */
+export function concatWavs(wavBuffers: Uint8Array[], gapsMs: number[] = []): Uint8Array {
+    if (wavBuffers.length === 0) {
+        return new Uint8Array(0);
+    }
+
+    const parsedWavs = wavBuffers.map((buf) => parseWav(buf));
+    const first = parsedWavs[0];
+    if (!first) {
+        return new Uint8Array(0);
+    }
+
+    // Verify format compatibility
+    for (let i = 1; i < parsedWavs.length; i++) {
+        const current = parsedWavs[i];
+        if (!current) {
+            continue;
+        }
+        if (current.sampleRate !== first.sampleRate) {
+            throw new Error(`WAV sample rate mismatch: expected ${first.sampleRate} Hz, got ${current.sampleRate} Hz`);
+        }
+        if (
+            current.audioFormat !== first.audioFormat ||
+            current.numChannels !== first.numChannels ||
+            current.bitsPerSample !== first.bitsPerSample
+        ) {
+            throw new Error(
+                `WAV format mismatch between segment 0 and segment ${i}: format ${first.audioFormat} vs ${current.audioFormat}, channels ${first.numChannels} vs ${current.numChannels}, bits ${first.bitsPerSample} vs ${current.bitsPerSample}`,
+            );
+        }
+    }
+
+    // Compute total combined data length
+    let totalDataLength = 0;
+    const pieces: Uint8Array[] = [];
+
+    for (let i = 0; i < parsedWavs.length; i++) {
+        const wav = parsedWavs[i];
+        if (!wav) {
+            continue;
+        }
+        const gap = gapsMs[i] ?? 0;
+        if (gap > 0) {
+            const numGapBytes = Math.floor(first.sampleRate * (gap / 1000)) * first.blockAlign;
+            if (numGapBytes > 0) {
+                const silence = new Uint8Array(numGapBytes);
+                pieces.push(silence);
+                totalDataLength += numGapBytes;
+            }
+        }
+        pieces.push(wav.dataBytes);
+        totalDataLength += wav.dataBytes.length;
+    }
+
+    // Build the consolidated WAV buffer
+    const headerSize = 44;
+    const totalFileSize = headerSize + totalDataLength;
+    const output = new Uint8Array(totalFileSize);
+    const view = new DataView(output.buffer, output.byteOffset, output.byteLength);
+
+    // RIFF chunk descriptor
+    output[0] = 'R'.charCodeAt(0);
+    output[1] = 'I'.charCodeAt(0);
+    output[2] = 'F'.charCodeAt(0);
+    output[3] = 'F'.charCodeAt(0);
+    view.setUint32(4, 36 + totalDataLength, true);
+    output[8] = 'W'.charCodeAt(0);
+    output[9] = 'A'.charCodeAt(0);
+    output[10] = 'V'.charCodeAt(0);
+    output[11] = 'E'.charCodeAt(0);
+
+    // fmt sub-chunk
+    output[12] = 'f'.charCodeAt(0);
+    output[13] = 'm'.charCodeAt(0);
+    output[14] = 't'.charCodeAt(0);
+    output[15] = ' '.charCodeAt(0);
+    view.setUint32(16, 16, true);
+    view.setUint16(20, first.audioFormat, true);
+    view.setUint16(22, first.numChannels, true);
+    view.setUint32(24, first.sampleRate, true);
+    view.setUint32(28, first.byteRate, true);
+    view.setUint16(32, first.blockAlign, true);
+    view.setUint16(34, first.bitsPerSample, true);
+
+    // data sub-chunk
+    output[36] = 'd'.charCodeAt(0);
+    output[37] = 'a'.charCodeAt(0);
+    output[38] = 't'.charCodeAt(0);
+    output[39] = 'a'.charCodeAt(0);
+    view.setUint32(40, totalDataLength, true);
+
+    // Copy audio data pieces
+    let currentOffset = headerSize;
+    for (const piece of pieces) {
+        output.set(piece, currentOffset);
+        currentOffset += piece.length;
+    }
+
+    return output;
+}