@gobing-ai/knowledge-kit 0.0.6 → 0.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@gobing-ai/knowledge-kit",
-    "version": "0.0.6",
+    "version": "0.0.8",
     "description": "An ingest → create → publish content pipeline CLI (Bun).",
     "repository": {
         "type": "git",
@@ -28,6 +28,9 @@
         "typecheck": "tsc --noEmit"
     },
     "dependencies": {
+        "@gobing-ai/ts-infra": "catalog:",
+        "@gobing-ai/ts-runtime": "catalog:",
+        "@gobing-ai/ts-utils": "catalog:",
         "commander": "15.0.0"
     },
     "devDependencies": {

package/plugins/generations/content-gen/package.json

@@ -7,7 +7,8 @@
     },
     "dependencies": {
         "@gobing-ai/kk-core": "workspace:*",
-        "@gobing-ai/utils": "workspace:*",
+        "@gobing-ai/ts-runtime": "catalog:",
+        "@gobing-ai/ts-utils": "catalog:",
         "zod": "4.4.3"
     },
     "devDependencies": {

package/plugins/generations/content-gen/src/index.ts

@@ -1,7 +1,7 @@
-import { readFile, rm, writeFile } from 'node:fs/promises';
 import { parseArgs } from 'node:util';
 import { type Content, ContentSchema, type Doc, DocListSchema } from '@gobing-ai/kk-core';
-import { logger } from '@gobing-ai/utils';
+import { createNodeFileSystem } from '@gobing-ai/ts-runtime';
+import { echoError } from '@gobing-ai/ts-utils';
 import { generateWithStormPipeline } from './storm';
 
 type ContentGenerator = (docs: Doc[], inputPath: string) => Promise<Content>;
@@ -19,10 +19,11 @@ export async function processGeneratorIO(
     outputPath: string,
     generate: ContentGenerator = generateWithStormPipeline,
 ): Promise<void> {
-    await rm(outputPath, { force: true });
-    const docs = DocListSchema.parse(JSON.parse(await readFile(inputPath, 'utf-8')));
+    const fs = createNodeFileSystem();
+    await fs.deleteFile(outputPath);
+    const docs = DocListSchema.parse(JSON.parse(await fs.readFile(inputPath)));
     const content = docs.length === 0 ? EMPTY_CONTENT : ContentSchema.parse(await generate(docs, inputPath));
-    await writeFile(outputPath, JSON.stringify(content, null, 2), 'utf-8');
+    await fs.writeFile(outputPath, JSON.stringify(content, null, 2));
 }
 
 export async function main(): Promise<number> {
@@ -33,7 +34,7 @@ export async function main(): Promise<number> {
         },
     });
     if (!values.in || !values.out) {
-        logger.error('Missing required arguments: --in and --out');
+        echoError('Missing required arguments: --in and --out');
         return 1;
     }
 
@@ -41,8 +42,8 @@ export async function main(): Promise<number> {
         await processGeneratorIO(values.in, values.out);
         return 0;
     } catch (error: unknown) {
-        logger.error(`content-gen failed: ${error instanceof Error ? error.message : String(error)}`);
-        await rm(values.out, { force: true });
+        echoError(`content-gen failed: ${error instanceof Error ? error.message : String(error)}`);
+        await createNodeFileSystem().deleteFile(values.out);
         return 1;
     }
 }

package/plugins/generations/content-gen/src/storm.ts

@@ -113,8 +113,13 @@ function stageError(stage: string, detail: string): Error {
     return new Error(`${stage} stage failed: ${detail}`);
 }
 
+/** Strip markdown footnote markers agents prepend to citation ids (`^doc-1` → `doc-1`). */
+function bareCitationId(id: string): string {
+    return id.replace(/^\^+/, '');
+}
+
 function assertKnownEvidence(stage: string, ids: string[], allowed: Set<string>): void {
-    const unknown = ids.filter((id) => !allowed.has(id));
+    const unknown = ids.map(bareCitationId).filter((id) => !allowed.has(id));
     if (unknown.length > 0) {
         throw stageError(stage, `cites unknown document ID(s): ${[...new Set(unknown)].join(', ')}`);
     }
@@ -237,6 +242,7 @@ export async function articleStage(
             `The validated curation result for the article is: ${JSON.stringify(curation)}`,
             `The validated outline for the article is: ${JSON.stringify(outline)}`,
             'Write the complete article as markdown, one section per outline heading. Every citation must reference one of the allowed document IDs.',
+            'Synthesize: the body must draw findings from across the supplied documents, compare them, and state conclusions. Never list sources one per line or produce an index of references — references are evidence, not the deliverable.',
             'Return only a JSON object matching {"title":string,"body":string,"citations":[string]} where body is the full markdown draft and every citations entry is one of the allowed document IDs.',
         ].join('\n'),
     );
@@ -289,9 +295,11 @@ export async function polishStage(
     // Machine-verify the preservation mandate: every heading and citation in the draft must
     // survive, and polish must not introduce citations to unknown document IDs.
     const polishedHeadings = bodyHeadings(result.body);
-    const polishedCitations = bodyCitations(result.body);
+    const polishedCitations = bodyCitations(result.body).map(bareCitationId);
     const droppedHeadings = bodyHeadings(article.body).filter((heading) => !polishedHeadings.includes(heading));
-    const droppedCitations = bodyCitations(article.body).filter((citation) => !polishedCitations.includes(citation));
+    const droppedCitations = bodyCitations(article.body)
+        .map(bareCitationId)
+        .filter((citation) => !polishedCitations.includes(citation));
     if (droppedHeadings.length > 0 || droppedCitations.length > 0) {
         const dropped: string[] = [];
         if (droppedHeadings.length > 0) dropped.push(`dropped heading(s): ${droppedHeadings.join(', ')}`);
@@ -320,7 +328,7 @@ export async function generateWithStormPipeline(
     return ContentSchema.parse({
         title:
             polished.title ?? article.title ?? outline.title ?? docs[0]?.title ?? 'Generated Knowledge Kit Synthesis',
-        body: polished.body,
+        body: polished.body.replace(/\[\^([^\]()\s]+)\](?!\()/g, '[$1]'),
         format: 'markdown',
         outline: polished.outline ?? outline.outline,
         references: docs.map((doc) => ({ url: doc.sourceUri, title: doc.title, cite: doc.id })),

package/plugins/generations/voice-gen/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "@gobing-ai/voice-gen",
+    "type": "module",
+    "private": true,
+    "scripts": {
+        "typecheck": "tsc --noEmit"
+    },
+    "dependencies": {
+        "@gobing-ai/kk-core": "workspace:*",
+        "@gobing-ai/ts-runtime": "catalog:",
+        "@gobing-ai/ts-utils": "catalog:",
+        "zod": "4.4.3"
+    },
+    "devDependencies": {
+        "@types/bun": "1.3.14"
+    }
+}

package/plugins/generations/voice-gen/plugin.json

@@ -0,0 +1,6 @@
+{
+    "name": "voice-gen",
+    "kind": "generator",
+    "entry": "./src/index.ts",
+    "version": "1.0.0"
+}

package/plugins/generations/voice-gen/src/concat.ts

@@ -0,0 +1,218 @@
+export interface ParsedWav {
+    audioFormat: number;
+    numChannels: number;
+    sampleRate: number;
+    byteRate: number;
+    blockAlign: number;
+    bitsPerSample: number;
+    dataBytes: Uint8Array;
+}
+
+/**
+ * Parse a RIFF WAVE buffer into its format fields and raw audio data.
+ */
+export function parseWav(buffer: Uint8Array): ParsedWav {
+    if (buffer.length < 44) {
+        throw new Error('Invalid WAV buffer: buffer too short');
+    }
+
+    const view = new DataView(buffer.buffer, buffer.byteOffset, buffer.byteLength);
+
+    const b0 = buffer[0];
+    const b1 = buffer[1];
+    const b2 = buffer[2];
+    const b3 = buffer[3];
+    const b8 = buffer[8];
+    const b9 = buffer[9];
+    const b10 = buffer[10];
+    const b11 = buffer[11];
+
+    if (
+        b0 === undefined ||
+        b1 === undefined ||
+        b2 === undefined ||
+        b3 === undefined ||
+        b8 === undefined ||
+        b9 === undefined ||
+        b10 === undefined ||
+        b11 === undefined
+    ) {
+        throw new Error('Invalid WAV buffer: buffer too short');
+    }
+
+    const riffTag = String.fromCharCode(b0, b1, b2, b3);
+    const waveTag = String.fromCharCode(b8, b9, b10, b11);
+
+    if (riffTag !== 'RIFF' || waveTag !== 'WAVE') {
+        throw new Error(`Invalid WAV buffer: expected RIFF/WAVE header, got ${riffTag}/${waveTag}`);
+    }
+
+    let offset = 12;
+    let formatInfo:
+        | {
+              audioFormat: number;
+              numChannels: number;
+              sampleRate: number;
+              byteRate: number;
+              blockAlign: number;
+              bitsPerSample: number;
+          }
+        | undefined;
+    let dataBytes: Uint8Array | undefined;
+
+    while (offset + 8 <= buffer.length) {
+        const c0 = buffer[offset];
+        const c1 = buffer[offset + 1];
+        const c2 = buffer[offset + 2];
+        const c3 = buffer[offset + 3];
+        if (c0 === undefined || c1 === undefined || c2 === undefined || c3 === undefined) {
+            break;
+        }
+
+        const chunkId = String.fromCharCode(c0, c1, c2, c3);
+        const chunkSize = view.getUint32(offset + 4, true);
+        const chunkDataOffset = offset + 8;
+
+        if (chunkId === 'fmt ') {
+            if (chunkSize < 16) {
+                throw new Error('Invalid WAV fmt chunk size');
+            }
+            const audioFormat = view.getUint16(chunkDataOffset, true);
+            const numChannels = view.getUint16(chunkDataOffset + 2, true);
+            const sampleRate = view.getUint32(chunkDataOffset + 4, true);
+            const byteRate = view.getUint32(chunkDataOffset + 8, true);
+            const blockAlign = view.getUint16(chunkDataOffset + 12, true);
+            const bitsPerSample = view.getUint16(chunkDataOffset + 14, true);
+
+            formatInfo = {
+                audioFormat,
+                numChannels,
+                sampleRate,
+                byteRate,
+                blockAlign,
+                bitsPerSample,
+            };
+        } else if (chunkId === 'data') {
+            const end = Math.min(chunkDataOffset + chunkSize, buffer.length);
+            dataBytes = buffer.subarray(chunkDataOffset, end);
+        }
+
+        // Advance to next chunk (aligned to 2 bytes)
+        offset = chunkDataOffset + chunkSize + (chunkSize % 2 === 1 ? 1 : 0);
+    }
+
+    if (!formatInfo) {
+        throw new Error('Invalid WAV: missing "fmt " chunk');
+    }
+    if (!dataBytes) {
+        throw new Error('Invalid WAV: missing "data" chunk');
+    }
+
+    return {
+        ...formatInfo,
+        dataBytes,
+    };
+}
+
+/**
+ * Concatenate multiple WAV audio buffers in order, inserting gap_ms of silence before clips.
+ */
+export function concatWavs(wavBuffers: Uint8Array[], gapsMs: number[] = []): Uint8Array {
+    if (wavBuffers.length === 0) {
+        return new Uint8Array(0);
+    }
+
+    const parsedWavs = wavBuffers.map((buf) => parseWav(buf));
+    const first = parsedWavs[0];
+    if (!first) {
+        return new Uint8Array(0);
+    }
+
+    // Verify format compatibility
+    for (let i = 1; i < parsedWavs.length; i++) {
+        const current = parsedWavs[i];
+        if (!current) {
+            continue;
+        }
+        if (current.sampleRate !== first.sampleRate) {
+            throw new Error(`WAV sample rate mismatch: expected ${first.sampleRate} Hz, got ${current.sampleRate} Hz`);
+        }
+        if (
+            current.audioFormat !== first.audioFormat ||
+            current.numChannels !== first.numChannels ||
+            current.bitsPerSample !== first.bitsPerSample
+        ) {
+            throw new Error(
+                `WAV format mismatch between segment 0 and segment ${i}: format ${first.audioFormat} vs ${current.audioFormat}, channels ${first.numChannels} vs ${current.numChannels}, bits ${first.bitsPerSample} vs ${current.bitsPerSample}`,
+            );
+        }
+    }
+
+    // Compute total combined data length
+    let totalDataLength = 0;
+    const pieces: Uint8Array[] = [];
+
+    for (let i = 0; i < parsedWavs.length; i++) {
+        const wav = parsedWavs[i];
+        if (!wav) {
+            continue;
+        }
+        const gap = gapsMs[i] ?? 0;
+        if (gap > 0) {
+            const numGapBytes = Math.floor(first.sampleRate * (gap / 1000)) * first.blockAlign;
+            if (numGapBytes > 0) {
+                const silence = new Uint8Array(numGapBytes);
+                pieces.push(silence);
+                totalDataLength += numGapBytes;
+            }
+        }
+        pieces.push(wav.dataBytes);
+        totalDataLength += wav.dataBytes.length;
+    }
+
+    // Build the consolidated WAV buffer
+    const headerSize = 44;
+    const totalFileSize = headerSize + totalDataLength;
+    const output = new Uint8Array(totalFileSize);
+    const view = new DataView(output.buffer, output.byteOffset, output.byteLength);
+
+    // RIFF chunk descriptor
+    output[0] = 'R'.charCodeAt(0);
+    output[1] = 'I'.charCodeAt(0);
+    output[2] = 'F'.charCodeAt(0);
+    output[3] = 'F'.charCodeAt(0);
+    view.setUint32(4, 36 + totalDataLength, true);
+    output[8] = 'W'.charCodeAt(0);
+    output[9] = 'A'.charCodeAt(0);
+    output[10] = 'V'.charCodeAt(0);
+    output[11] = 'E'.charCodeAt(0);
+
+    // fmt sub-chunk
+    output[12] = 'f'.charCodeAt(0);
+    output[13] = 'm'.charCodeAt(0);
+    output[14] = 't'.charCodeAt(0);
+    output[15] = ' '.charCodeAt(0);
+    view.setUint32(16, 16, true);
+    view.setUint16(20, first.audioFormat, true);
+    view.setUint16(22, first.numChannels, true);
+    view.setUint32(24, first.sampleRate, true);
+    view.setUint32(28, first.byteRate, true);
+    view.setUint16(32, first.blockAlign, true);
+    view.setUint16(34, first.bitsPerSample, true);
+
+    // data sub-chunk
+    output[36] = 'd'.charCodeAt(0);
+    output[37] = 'a'.charCodeAt(0);
+    output[38] = 't'.charCodeAt(0);
+    output[39] = 'a'.charCodeAt(0);
+    view.setUint32(40, totalDataLength, true);
+
+    // Copy audio data pieces
+    let currentOffset = headerSize;
+    for (const piece of pieces) {
+        output.set(piece, currentOffset);
+        currentOffset += piece.length;
+    }
+
+    return output;
+}

package/plugins/generations/voice-gen/src/index.ts

@@ -0,0 +1,213 @@
+import { basename, dirname, extname, join, resolve } from 'node:path';
+import { parseArgs } from 'node:util';
+import { type Content, ContentSchema, type Doc, DocListSchema } from '@gobing-ai/kk-core';
+import { createNodeFileSystem } from '@gobing-ai/ts-runtime';
+import { echoError } from '@gobing-ai/ts-utils';
+import { concatWavs } from './concat';
+import { createVoiceboxClient, type VoiceboxClient, type VoiceboxGenerateBody } from './voicebox-client';
+import {
+    mergeVoiceScripts,
+    parseDocToVoiceScript,
+    VOICEBOX_CHUNK_CHARS_DEFAULT,
+    VOICEBOX_CHUNK_CHARS_MAX,
+    VOICEBOX_CHUNK_CHARS_MIN,
+    VOICEBOX_CROSSFADE_MS_DEFAULT,
+    VOICEBOX_CROSSFADE_MS_MAX,
+    VOICEBOX_CROSSFADE_MS_MIN,
+    validateVoiceScript,
+} from './voicescript';
+
+export * from './concat';
+export * from './voicebox-client';
+export * from './voicescript';
+
+const EMPTY_CONTENT: Content = {
+    title: 'Notice',
+    body: '# Notice\nNo documents provided for generation.',
+    format: 'markdown',
+    references: [],
+};
+
+/**
+ * Read validated Doc[] input, generate validated audio Content via Voicebox, and write output files.
+ */
+export async function processGeneratorIO(
+    inputPath: string,
+    outputPath: string,
+    clientOverride?: VoiceboxClient,
+): Promise<void> {
+    const fs = createNodeFileSystem();
+    const outDir = dirname(outputPath);
+    const outStem = basename(outputPath, extname(outputPath));
+    const audioPath = resolve(join(outDir, `${outStem}.wav`));
+
+    // Delete existing output and sibling WAV at start
+    await fs.deleteFile(outputPath);
+    await fs.deleteFile(audioPath);
+
+    let docs: Doc[];
+    try {
+        const raw = await fs.readFile(inputPath);
+        docs = DocListSchema.parse(JSON.parse(raw));
+    } catch (err: unknown) {
+        throw new Error(`Invalid DocList input: ${err instanceof Error ? err.message : String(err)}`);
+    }
+
+    if (docs.length === 0) {
+        await fs.ensureDir(outDir);
+        await fs.writeFile(outputPath, JSON.stringify(ContentSchema.parse(EMPTY_CONTENT), null, 2));
+        return;
+    }
+
+    const client = clientOverride ?? createVoiceboxClient();
+
+    try {
+        // 1. Health check
+        await client.health();
+
+        // 2. Parse docs to VoiceScript
+        const envDefaultProfile = process.env.VOICEBOX_DEFAULT_PROFILE;
+        const scripts = docs.map((doc) => parseDocToVoiceScript(doc, envDefaultProfile));
+        const combinedScript = mergeVoiceScripts(scripts, docs, envDefaultProfile);
+
+        // 3. Validate VoiceScript
+        validateVoiceScript(combinedScript);
+
+        // 4. Generate each segment
+        const segmentWavs: Uint8Array[] = [];
+        const segmentGaps: number[] = [];
+        const segmentMetadata: Array<{ generationId: string; profile: string; duration: number }> = [];
+        let totalDuration = 0;
+
+        for (const segment of combinedScript.segments) {
+            const speakerConfig = segment.speaker ? combinedScript.speakers?.[segment.speaker] : undefined;
+            const profileTarget =
+                segment.profile ??
+                speakerConfig?.profile ??
+                combinedScript.default_profile ??
+                envDefaultProfile ??
+                (typeof docs[0]?.metadata?.voiceProfile === 'string' ? docs[0].metadata.voiceProfile : undefined);
+
+            if (!profileTarget) {
+                throw new Error(
+                    'No voice profile specified for segment (set profile, speaker profile, default_profile, or VOICEBOX_DEFAULT_PROFILE)',
+                );
+            }
+
+            const resolvedProfile = await client.resolveProfile(profileTarget);
+
+            const engine = segment.engine ?? speakerConfig?.engine ?? combinedScript.default_engine;
+            const language = segment.language ?? speakerConfig?.language ?? combinedScript.language ?? 'en';
+            const instruct = segment.instruct ?? speakerConfig?.instruct;
+
+            const envChunk = process.env.VOICEBOX_MAX_CHUNK_CHARS
+                ? parseInt(process.env.VOICEBOX_MAX_CHUNK_CHARS, 10)
+                : undefined;
+            const envCrossfade = process.env.VOICEBOX_CROSSFADE_MS
+                ? parseInt(process.env.VOICEBOX_CROSSFADE_MS, 10)
+                : undefined;
+
+            const max_chunk_chars =
+                segment.max_chunk_chars ?? combinedScript.max_chunk_chars ?? envChunk ?? VOICEBOX_CHUNK_CHARS_DEFAULT;
+            const crossfade_ms =
+                segment.crossfade_ms ?? combinedScript.crossfade_ms ?? envCrossfade ?? VOICEBOX_CROSSFADE_MS_DEFAULT;
+
+            if (max_chunk_chars < VOICEBOX_CHUNK_CHARS_MIN || max_chunk_chars > VOICEBOX_CHUNK_CHARS_MAX) {
+                throw new Error(
+                    `max_chunk_chars must be between ${VOICEBOX_CHUNK_CHARS_MIN} and ${VOICEBOX_CHUNK_CHARS_MAX}`,
+                );
+            }
+            if (crossfade_ms < VOICEBOX_CROSSFADE_MS_MIN || crossfade_ms > VOICEBOX_CROSSFADE_MS_MAX) {
+                throw new Error(
+                    `crossfade_ms must be between ${VOICEBOX_CROSSFADE_MS_MIN} and ${VOICEBOX_CROSSFADE_MS_MAX}`,
+                );
+            }
+
+            const generateBody: VoiceboxGenerateBody = {
+                profile_id: resolvedProfile.id,
+                text: segment.text,
+                language,
+                engine,
+                instruct,
+                max_chunk_chars,
+                crossfade_ms,
+                personality: false,
+                normalize: true,
+            };
+
+            const { id } = await client.generate(generateBody);
+            const history = await client.waitUntilDone(id);
+            if (history.status === 'failed') {
+                throw new Error(
+                    `Voicebox generation failed for profile "${resolvedProfile.name}": ${history.error || 'status failed'}`,
+                );
+            }
+
+            const wavBytes = await client.downloadAudio(id);
+            const gap = segment.gap_ms ?? 0;
+
+            segmentWavs.push(wavBytes);
+            segmentGaps.push(gap);
+
+            const segDuration = history.duration ?? 0;
+            totalDuration += segDuration + gap / 1000;
+            segmentMetadata.push({
+                generationId: id,
+                profile: resolvedProfile.name,
+                duration: segDuration,
+            });
+        }
+
+        // 5. Concatenate audio
+        const concatenatedWav = concatWavs(segmentWavs, segmentGaps);
+        await fs.ensureDir(outDir);
+        await Bun.write(audioPath, concatenatedWav);
+
+        // 6. Build Content
+        const content: Content = {
+            title: combinedScript.title || docs[0]?.title || 'Generated voice',
+            body: Bun.YAML.stringify(combinedScript),
+            format: 'audio',
+            references: [],
+            metadata: {
+                generator: 'kk:voice-gen',
+                audioPath,
+                duration: totalDuration,
+                segments: segmentMetadata,
+            },
+        };
+
+        const validatedContent = ContentSchema.parse(content);
+        await fs.writeFile(outputPath, JSON.stringify(validatedContent, null, 2));
+    } catch (err: unknown) {
+        await fs.deleteFile(outputPath);
+        await fs.deleteFile(audioPath);
+        throw err;
+    }
+}
+
+export async function main(): Promise<number> {
+    const { values } = parseArgs({
+        options: {
+            in: { type: 'string' },
+            out: { type: 'string' },
+        },
+    });
+
+    if (!values.in || !values.out) {
+        echoError('voice-gen failed: Missing required arguments: --in and --out');
+        return 1;
+    }
+
+    try {
+        await processGeneratorIO(values.in, values.out);
+        return 0;
+    } catch (err: unknown) {
+        echoError(`voice-gen failed: ${err instanceof Error ? err.message : String(err)}`);
+        return 1;
+    }
+}
+
+if (import.meta.main) {
+    process.exit(await main());
+}