@gobing-ai/knowledge-kit 0.0.11 → 0.0.12

package/plugins/generations/episode-plan-gen/tsconfig.json

@@ -0,0 +1,4 @@
+{
+    "extends": "../../../tooling/typescript/base.json",
+    "include": ["src", "tests"]
+}

package/plugins/generations/voice-gen/src/index.ts

@@ -5,7 +5,7 @@ import { createNodeFileSystem } from '@gobing-ai/ts-runtime';
 import { echoError } from '@gobing-ai/ts-utils';
 import { concatWavs } from './concat';
 import { isMp3Requested, type Mp3Transcoder, transcodeWavToMp3 } from './mp3';
-import { auditVoiceSegments } from './qc';
+import { auditVoiceSegments, detectLoudnessDip, TRANSCRIPTION_FIDELITY_FLOOR, transcriptionFidelity } from './qc';
 import { createVoiceboxClient, type VoiceboxClient, type VoiceboxGenerateBody } from './voicebox-client';
 import {
     mergeVoiceScripts,
@@ -92,6 +92,9 @@ export async function processGeneratorIO(
             speaker?: string;
             emotion?: string;
             duration: number;
+            verifyRetries?: number;
+            transcription?: string;
+            loudnessDip?: boolean;
         }> = [];
         let totalDuration = 0;
 
@@ -115,7 +118,8 @@ export async function processGeneratorIO(
 
             const resolvedProfile = await client.resolveProfile(profileTarget);
 
-            const engine = segment.engine ?? speakerConfig?.engine ?? combinedScript.default_engine;
+            const engine =
+                segment.engine ?? speakerConfig?.engine ?? combinedScript.default_engine ?? process.env.VOICEBOX_ENGINE;
             const language = segment.language ?? speakerConfig?.language ?? combinedScript.language ?? 'en';
             const rawInstruct = segment.instruct ?? speakerConfig?.instruct;
             const rawEmotion = segment.emotion ?? speakerConfig?.emotion ?? combinedScript.default_emotion;
@@ -170,28 +174,115 @@ export async function processGeneratorIO(
                 effects_chain,
             };
 
-            const { id } = await client.generate(generateBody);
-            const history = await client.waitUntilDone(id);
-            if (history.status === 'failed') {
-                throw new Error(
-                    `Voicebox generation failed for profile "${resolvedProfile.name}": ${history.error || 'status failed'}`,
-                );
+            // Timeout retry: a hung/timeout voicebox generation must not kill a multi-hour
+            // run (dogfood daily-ai-news-20260903: segment 108/132 timed out at 600s and
+            // failed the whole generate state after 140 minutes).
+            //
+            // Verify retry: qwen with cloned profiles occasionally emits contiguous loop
+            // garbage (fidelity 0.00 vs >=0.23 zh ASR floor on real audio — dogfood
+            // 2026-09-03). The server's runaway detector only catches gap-shaped failures,
+            // so verify per segment via transcription fidelity and regenerate failures with
+            // a varied seed and halved chunk size (mirrors the server's own retry policy).
+            let wavBytes: Uint8Array | undefined;
+            let segDuration = 0;
+            let generationId = '';
+            let transcription: string | undefined;
+            let verifyRetries = 0;
+            let loudnessDip = false;
+            let lastError: unknown;
+            let attemptBody = generateBody;
+            for (let verifyAttempt = 0; verifyAttempt <= 2 && wavBytes === undefined; verifyAttempt++) {
+                if (verifyAttempt > 0) {
+                    verifyRetries = verifyAttempt;
+                    const chunkChars = Math.max(VOICEBOX_CHUNK_CHARS_MIN, Math.floor(attemptBody.max_chunk_chars / 2));
+                    attemptBody = {
+                        ...generateBody,
+                        seed: Math.floor(Math.random() * 2 ** 31),
+                        max_chunk_chars: chunkChars,
+                    };
+                    echoError(
+                        `voice-gen: regenerating segment with seed ${attemptBody.seed} and max_chunk_chars ${chunkChars}`,
+                    );
+                }
+                let attemptBytes: Uint8Array | undefined;
+                for (let attempt = 1; attempt <= 2 && attemptBytes === undefined; attempt++) {
+                    try {
+                        const { id } = await client.generate(attemptBody);
+                        generationId = id;
+                        // Long qwen segments synthesize slower than the fixed 600s deadline
+                        // (live 2026-09-04: 171s-audio segment took >600s twice, both completed
+                        // server-side). Scale the poll deadline with text length: 600s base
+                        // (or VOICEBOX_TIMEOUT_MS) + 60s per 100 chars, ~2x observed worst case.
+                        const envTimeout = process.env.VOICEBOX_TIMEOUT_MS
+                            ? parseInt(process.env.VOICEBOX_TIMEOUT_MS, 10)
+                            : 600000;
+                        const waitTimeout = envTimeout + Math.ceil(segment.text.length / 100) * 60_000;
+                        const history = await client.waitUntilDone(id, waitTimeout);
+                        if (history.status === 'failed') {
+                            throw new Error(
+                                `Voicebox generation failed for profile "${resolvedProfile.name}": ${history.error || 'status failed'}`,
+                            );
+                        }
+                        attemptBytes = await client.downloadAudio(id);
+                        segDuration = history.duration ?? 0;
+                    } catch (err: unknown) {
+                        lastError = err;
+                        if (attempt === 1) {
+                            echoError(
+                                `voice-gen: segment failed (${err instanceof Error ? err.message : String(err)}); retrying once`,
+                            );
+                        }
+                    }
+                }
+                if (attemptBytes === undefined) {
+                    throw lastError instanceof Error ? lastError : new Error(String(lastError));
+                }
+                if (typeof client.transcribe === 'function') {
+                    try {
+                        const trans = await client.transcribe(attemptBytes, language);
+                        const fidelity = transcriptionFidelity(segment.text, trans.text);
+                        if (fidelity < TRANSCRIPTION_FIDELITY_FLOOR && verifyAttempt < 2) {
+                            echoError(
+                                `voice-gen: transcription fidelity ${fidelity.toFixed(2)} below ${TRANSCRIPTION_FIDELITY_FLOOR} (looped/garbage audio): "${trans.text.slice(0, 80)}..."`,
+                            );
+                            continue;
+                        }
+                        transcription = trans.text;
+                    } catch {
+                        // Transcription unavailable — keep audio; the QC audit backstop decides.
+                    }
+                }
+                // Voice dent gate: Qwen3-TTS stochastically collapses loudness mid-utterance on
+                // long text; a dip that survives to the last verify attempt is recorded for QC
+                // (dogfood 2026-09-08 A/B: same text regenerated clean, so retry usually clears).
+                const dip = detectLoudnessDip(attemptBytes);
+                loudnessDip = dip.dip;
+                if (dip.dip && verifyAttempt < 2) {
+                    echoError(`voice-gen: loudness dip detected at ${dip.dipAtSec ?? '?'}s — regenerating segment`);
+                    continue;
+                }
+                wavBytes = attemptBytes;
+            }
+            if (wavBytes === undefined) {
+                // Both network attempts and all verify attempts exhausted without audio.
+                throw lastError instanceof Error ? lastError : new Error(String(lastError ?? 'no audio'));
             }
 
-            const wavBytes = await client.downloadAudio(id);
             const gap = segment.gap_ms ?? 0;
 
             segmentWavs.push(wavBytes);
             segmentGaps.push(gap);
 
-            const segDuration = history.duration ?? 0;
             totalDuration += segDuration + gap / 1000;
             segmentMetadata.push({
-                generationId: id,
+                generationId,
                 profile: resolvedProfile.name,
                 speaker: segment.speaker,
                 emotion: rawEmotion,
                 duration: segDuration,
+                verifyRetries,
+                transcription,
+                loudnessDip,
             });
         }
 

package/plugins/generations/voice-gen/src/qc.ts

@@ -1,3 +1,4 @@
+import { type ParsedWav, parseWav } from './concat';
 import type { VoiceboxClient } from './voicebox-client';
 import type { VoiceScript, VoiceSegment } from './voicescript';
 
@@ -10,6 +11,8 @@ export interface SegmentQualityAudit {
     transcription?: string;
     repetitionDetected: boolean;
     durationAnomaly: boolean;
+    /** True when a loudness dip (voice dent) was detected in the final audio and survived regeneration. */
+    loudnessDip?: boolean;
     issues: string[];
     passed: boolean;
 }
@@ -50,6 +53,109 @@ export function detectRepetitions(text: string): boolean {
     return false;
 }
 
+/** Below this character-bigram coverage, audio is treated as looped/garbage (dogfood 2026-09-03). */
+export const TRANSCRIPTION_FIDELITY_FLOOR = 0.15;
+
+/**
+ * Character-bigram coverage of the source text by the transcription. Whisper zh ASR is
+ * lossy on good audio (0.23–0.8 across dogfood runs); looped/garbage TTS audio scores ≤0.05.
+ */
+export function transcriptionFidelity(source: string, transcription: string): number {
+    const bigrams = (t: string): Set<string> => {
+        const clean = t.replace(/\s+/g, '');
+        return new Set(Array.from({ length: Math.max(0, clean.length - 1) }, (_, i) => clean.slice(i, i + 2)));
+    };
+    const a = bigrams(source);
+    const b = bigrams(transcription);
+    if (a.size === 0) return 1;
+    let inter = 0;
+    for (const g of a) if (b.has(g)) inter++;
+    return inter / a.size;
+}
+
+/**
+ * A ~1s window whose RMS sits below this fraction of its louder neighbour is a dent.
+ * Calibrated on dogfood 2026-09-08: audible Qwen3-TTS collapses measured 0.24–0.37 of
+ * the louder neighbour (rms 0.033–0.048); normal quieter phrases sit ≥0.45.
+ */
+export const DIP_RATIO = 0.4;
+
+/** A window below this RMS is a real pause (silence), not a dent — dents are quiet-but-not-silent. */
+export const DIP_SILENCE_FLOOR = 0.008;
+
+/** Windows above this absolute RMS are normal quieter narration, not collapses (file p25 ≈ 0.08). */
+export const DIP_ABS_CEILING = 0.06;
+
+/** Neighbour windows below this RMS carry no speech signal to compare against. */
+export const DIP_NEIGHBOR_FLOOR = 0.02;
+
+/**
+ * Detect a loudness dip (voice dent): a ~1s window of quiet-but-not-silent audio
+ * sandwiched between louder speech on both sides. Qwen3-TTS (AR clone, MLX) stochastically
+ * collapses loudness mid-utterance on long text (dogfood 2026-09-08 A/B: same text regenerated
+ * clean; official docs list "loudness drops" among AR long-form instabilities). The detector
+ * returns the first dip position; voice-gen regenerates on it (stochastic → retry usually clears).
+ * Returns null when the wav is too short or unparseable.
+ */
+export function detectLoudnessDip(wav: Uint8Array): { dip: boolean; dipAtSec: number | null } {
+    let parsed: ParsedWav;
+    try {
+        parsed = parseWav(wav);
+    } catch {
+        return { dip: false, dipAtSec: null };
+    }
+    if (parsed.bitsPerSample !== 16 && parsed.bitsPerSample !== 8) return { dip: false, dipAtSec: null };
+    const rate = parsed.sampleRate;
+    if (rate <= 0) return { dip: false, dipAtSec: null };
+    const data = parsed.dataBytes;
+    const bytesPerSample = parsed.bitsPerSample === 8 ? 1 : 2;
+    const frameBytes = parsed.numChannels * bytesPerSample;
+    const totalFrames = Math.floor(data.length / frameBytes);
+    const windowFrames = rate; // 1s windows
+    if (totalFrames < windowFrames * 5) return { dip: false, dipAtSec: null };
+
+    const sampleAt = (frame: number, channel = 0): number => {
+        const off = (frame * parsed.numChannels + channel) * bytesPerSample;
+        if (bytesPerSample === 1) return ((data[off] ?? 0) - 128) / 128;
+        let s = (data[off] ?? 0) | ((data[off + 1] ?? 0) << 8);
+        if (s >= 0x8000) s -= 0x10000;
+        return s / 32768;
+    };
+
+    // Non-overlapping 1s windows (step = window): a dent spans ~1s → exactly one window,
+    // with loud speech before and after. Frame-stepped windows would smear a 1s dent into a
+    // 24000-index plateau with equal neighbours, invisible to any adjacent comparison.
+    const nWins = Math.floor(totalFrames / windowFrames);
+    if (nWins < 7) return { dip: false, dipAtSec: null };
+    const windowRms: number[] = [];
+    for (let w = 0; w < nWins; w++) {
+        const base = w * windowFrames;
+        let sum = 0;
+        for (let f = base; f < base + windowFrames; f++) {
+            const v = sampleAt(f);
+            sum += v * v;
+        }
+        windowRms.push(Math.sqrt(sum / windowFrames));
+    }
+
+    // A dent = one ~1s window of quiet-but-not-silent audio (rms 0.008..0.06) that is
+    // notably quieter than BOTH its immediate 1s neighbours (ratio < 0.4) — an isolated
+    // loudness collapse amid otherwise steady speech. Non-overlapping windows keep
+    // neighbour semantics exact (a frame-stepped window would smear a 1s dent into a
+    // many-index plateau with equal neighbours). Sentence pauses are excluded by the
+    // silence floor; normal quieter phrases by the absolute ceiling (calibrated on
+    // dogfood 2026-09-08 prod audio: audible dents 0.047/0.042/0.048 vs normal p25 0.080).
+    for (let j = 1; j < windowRms.length - 1; j++) {
+        const center = windowRms[j] ?? 0;
+        if (center <= DIP_SILENCE_FLOOR) continue; // real pause, not a dent
+        if (center >= DIP_ABS_CEILING) continue; // normal quieter narration
+        const nbr = Math.max(windowRms[j - 1] ?? 0, windowRms[j + 1] ?? 0);
+        if (nbr <= DIP_NEIGHBOR_FLOOR) continue;
+        if (center < nbr * DIP_RATIO) return { dip: true, dipAtSec: j };
+    }
+    return { dip: false, dipAtSec: null };
+}
+
 /**
  * Compute expected speech duration range given text and language.
  */
@@ -76,7 +182,14 @@ export function computeExpectedDurationRange(text: string, language = 'zh'): [nu
 export async function auditVoiceSegments(
     script: VoiceScript,
     segmentWavs: Uint8Array[],
-    segmentMetadata: Array<{ generationId: string; profile: string; duration: number }>,
+    segmentMetadata: Array<{
+        generationId: string;
+        profile: string;
+        duration: number;
+        transcription?: string;
+        verifyRetries?: number;
+        loudnessDip?: boolean;
+    }>,
     client?: VoiceboxClient,
 ): Promise<VoiceQualityReport> {
     const audits: SegmentQualityAudit[] = [];
@@ -109,22 +222,53 @@ export async function auditVoiceSegments(
         let transcriptionText: string | undefined;
         let repetitionDetected = false;
 
-        if (client && wavBytes && typeof client.transcribe === 'function') {
+        if (meta && typeof meta.transcription === 'string') {
+            // Precomputed during generation (verify step) — reuse instead of re-transcribing.
+            transcriptionText = meta.transcription;
+        } else if (client && wavBytes && typeof client.transcribe === 'function') {
             try {
                 const trans = await client.transcribe(wavBytes, lang);
                 transcriptionText = trans.text;
-                if (detectRepetitions(transcriptionText)) {
-                    repetitionDetected = true;
-                    issues.push(
-                        `Repetition hallucination detected in audio transcription: "${transcriptionText.slice(0, 80)}..."`,
-                    );
-                }
             } catch {
                 // Transcription service unavailable or optional
             }
         }
+        if (transcriptionText !== undefined && detectRepetitions(transcriptionText)) {
+            repetitionDetected = true;
+            // Whisper loops over trailing gap/silence and emits phantom repetitions whose
+            // audio duration stays normal; a real TTS loop always inflates duration.
+            // Fail only when the duration channel corroborates (dogfood 2026-09-03:
+            // 15/22 segments flagged with in-range durations = ASR artifacts).
+            if (durationAnomaly) {
+                issues.push(
+                    `Repetition hallucination corroborated by duration anomaly: "${transcriptionText.slice(0, 80)}..."`,
+                );
+            }
+        }
+
+        let lowFidelity = false;
+        if (transcriptionText !== undefined) {
+            const fidelity = transcriptionFidelity(segment.text, transcriptionText);
+            if (fidelity < TRANSCRIPTION_FIDELITY_FLOOR) {
+                lowFidelity = true;
+                issues.push(
+                    `Transcription fidelity ${fidelity.toFixed(2)} below ${TRANSCRIPTION_FIDELITY_FLOOR} — audio diverges from source text: "${transcriptionText.slice(0, 80)}..."`,
+                );
+            }
+        }
+
+        // Loudness dip (voice dent) that survived regeneration — record, don't fail the
+        // pipeline: the verify loop already regenerated on it, and a stubborn single dent
+        // is better surfaced than a whole-episode failure (dogfood 2026-09-08).
+        let loudnessDip = false;
+        if (meta?.loudnessDip === true) {
+            loudnessDip = true;
+            issues.push(
+                `Loudness dip persisted after regeneration (voice-gen auto-retried with varied seed; verifyRetries=${meta.verifyRetries ?? 0})`,
+            );
+        }
 
-        const passed = !durationAnomaly && !repetitionDetected;
+        const passed = !durationAnomaly && !(repetitionDetected && durationAnomaly) && !lowFidelity;
         if (!passed) {
             criticalIssues.push(`Segment ${i + 1} (${meta?.profile || 'Unknown'}): ${issues.join('; ')}`);
         }
@@ -138,6 +282,7 @@ export async function auditVoiceSegments(
             transcription: transcriptionText,
             repetitionDetected,
             durationAnomaly,
+            loudnessDip,
             issues,
             passed,
         });

package/plugins/ingestions/aihot-ingest/plugin.json

@@ -3,5 +3,5 @@
     "kind": "ingestion",
     "entry": "./src/index.ts",
     "version": "1.0.0",
-    "description": "Ingests curated AI news items from aihot.virxact.com into Doc[]"
+    "description": "Ingests fresh AI news from an RSS feed (default TechCrunch AI) into Doc[]; legacy aihot.virxact.com API behind AIHOT_SOURCE=api"
 }

package/plugins/ingestions/aihot-ingest/src/index.ts

@@ -6,9 +6,11 @@ import { echoError } from '@gobing-ai/ts-utils';
 import { z } from 'zod';
 import { type AihotDeps, createAihotClient } from './client';
 import { mapAihotItemsToDocs } from './mapper';
+import { AIHOT_FEED_URL_DEFAULT, fetchRssDocs, type RssFetchLike } from './rss';
 
 export * from './client';
 export * from './mapper';
+export * from './rss';
 
 export interface IngestionOptions {
     out: string;
@@ -38,8 +40,26 @@ function parseLimit(value: string): number {
 export async function processIngestionIO(
     options: IngestionOptions,
     depsOverride?: Partial<AihotDeps>,
+    rssFetchLike: RssFetchLike = fetch,
 ): Promise<IngestionResult> {
     const fs = createNodeFileSystem();
+
+    // RSS is the default source (2026-09-08 dogfood: the aihot.virxact.com API served a
+    // 2020–2025 archive — every item failed the 7-day freshness gate; see rss.ts). The API
+    // path stays reachable with AIHOT_SOURCE=api for when the CMS carries live curated items.
+    const source = process.env.AIHOT_SOURCE?.trim() || 'rss';
+    if (source === 'api') {
+        return processApiSource(options, fs, depsOverride);
+    }
+    return processRssSource(options, fs, rssFetchLike);
+}
+
+/** Legacy aihot.virxact.com API source (cursor/stateFile contract preserved). */
+async function processApiSource(
+    options: IngestionOptions,
+    fs: ReturnType<typeof createNodeFileSystem>,
+    depsOverride?: Partial<AihotDeps>,
+): Promise<IngestionResult> {
     const client = createAihotClient(depsOverride);
 
     let effectiveCursor = options.cursor?.trim();
@@ -68,30 +88,69 @@ export async function processIngestionIO(
     }
 
     const docs = DocListSchema.parse(mapAihotItemsToDocs(items));
-
-    const outDir = dirname(options.out);
-    if (outDir && outDir !== '.') {
-        await fs.ensureDir(outDir);
-    }
-    await atomicWriteJson(options.out, docs, fs);
+    await writeOutput(fs, options, docs);
 
     if (options.stateFile) {
-        const stateDir = dirname(options.stateFile);
-        if (stateDir && stateDir !== '.') {
-            await fs.ensureDir(stateDir);
-        }
-        const statePayload = {
+        await writeState(fs, options.stateFile, {
             cursor,
             asOf,
             lastSyncAt: new Date().toISOString(),
             itemCount: docs.length,
-        };
-        await atomicWriteJson(options.stateFile, statePayload, fs);
+        });
     }
 
     return { docs, cursor, asOf };
 }
 
+/** RSS feed source (default): stateless fetch, no cursor (feed order is newest-first). */
+async function processRssSource(
+    options: IngestionOptions,
+    fs: ReturnType<typeof createNodeFileSystem>,
+    rssFetchLike: RssFetchLike,
+): Promise<IngestionResult> {
+    const feedUrl = process.env.AIHOT_FEED_URL?.trim() || AIHOT_FEED_URL_DEFAULT;
+    const limit = options.limit ?? 10;
+    const { docs, asOf } = await fetchRssDocs(feedUrl, limit, rssFetchLike);
+    const validated = DocListSchema.parse(docs);
+    await writeOutput(fs, options, validated);
+
+    if (options.stateFile) {
+        await writeState(fs, options.stateFile, {
+            source: 'rss',
+            feedUrl,
+            asOf,
+            lastSyncAt: new Date().toISOString(),
+            itemCount: validated.length,
+        });
+    }
+
+    return { docs: validated, cursor: '', asOf };
+}
+
+async function writeOutput(
+    fs: ReturnType<typeof createNodeFileSystem>,
+    options: IngestionOptions,
+    docs: Doc[],
+): Promise<void> {
+    const outDir = dirname(options.out);
+    if (outDir && outDir !== '.') {
+        await fs.ensureDir(outDir);
+    }
+    await atomicWriteJson(options.out, docs, fs);
+}
+
+async function writeState(
+    fs: ReturnType<typeof createNodeFileSystem>,
+    stateFile: string,
+    payload: Record<string, unknown>,
+): Promise<void> {
+    const stateDir = dirname(stateFile);
+    if (stateDir && stateDir !== '.') {
+        await fs.ensureDir(stateDir);
+    }
+    await atomicWriteJson(stateFile, payload, fs);
+}
+
 export async function main(depsOverride?: Partial<AihotDeps>): Promise<number> {
     let values: { in?: string; out?: string; limit?: string; cursor?: string; 'state-file'?: string };
     try {

package/plugins/ingestions/aihot-ingest/src/mapper.ts

@@ -16,6 +16,7 @@ export function mapAihotItemToDoc(item: AihotItem): Doc {
     const body = `${item.summary?.trim() || item.title.trim()}\n`;
 
     const metadata: Record<string, unknown> = {
+        source: 'aihot',
         sourceName: item.source.name,
         publishedAt: item.publishedAt,
         discoveredAt: item.discoveredAt,

package/plugins/ingestions/aihot-ingest/src/rss.ts

@@ -0,0 +1,151 @@
+import { createHash } from 'node:crypto';
+import type { Doc } from '@gobing-ai/kk-core';
+
+/**
+ * RSS/Atom feed fetch + item → Doc mapping for the aihot-ingest RSS source.
+ *
+ * Why RSS (2026-09-08, dogfood): aihot.virxact.com's API served a 2020–2025 blog
+ * archive — all 10 items were dropped every run by the workflow's 7-day freshness
+ * gate, so the "curated" aihot source contributed zero usable items while carrying
+ * the highest selection tier (SOURCE_BASE aihot=30, importance 4). The RSS source
+ * restores aihot as a genuinely fresh feed; the API path stays selectable via
+ * AIHOT_SOURCE=api for when the CMS has live curated content.
+ *
+ * Keeps metadata.source='aihot' so planEpisode's aihot tier (and importance base 4)
+ * still applies to feed items.
+ */
+
+/** Default fresh AI-news feed (RSS 2.0, high volume, reachable without auth). */
+export const AIHOT_FEED_URL_DEFAULT = 'https://techcrunch.com/category/artificial-intelligence/feed/';
+
+export type RssFetchLike = (url: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+export interface RssEntry {
+    title: string;
+    link: string;
+    publishedAt: string | null; // ISO-8601 or null
+    summary: string;
+    category: string | null;
+}
+
+/** Strip XML/HTML tags and normalize common entities to plain text. */
+function stripMarkup(raw: string): string {
+    return raw
+        .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
+        .replace(/<[^>]+>/g, ' ')
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;|&apos;/g, "'")
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+
+/** RSS <pubDate> (RFC 822) or Atom <published>/<updated> (ISO-8601) → ISO-8601. */
+function toIsoDate(raw: string | undefined): string | null {
+    if (!raw?.trim()) return null;
+    const t = Date.parse(raw.trim().replace('UT', 'GMT'));
+    return Number.isFinite(t) ? new Date(t).toISOString() : null;
+}
+
+/** Extract a single sub-element's text from an element body. */
+function subElement(body: string, tag: string): string | undefined {
+    const re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`, 'i');
+    const m = body.match(re);
+    return m?.[1];
+}
+
+/** Extract a link: RSS <link>text</link> or Atom <link href="..."/> (first match wins). */
+function subLink(body: string): string | undefined {
+    const text = subElement(body, 'link');
+    if (text?.trim()) return text.trim();
+    const m = body.match(/<link[^>]*\bhref\s*=\s*"([^"]+)"/i);
+    return m?.[1];
+}
+
+/** Block-split RSS <item>… or Atom <entry>… elements and map each to an RssEntry. */
+export function parseFeed(xml: string): RssEntry[] {
+    const blocks: string[] = [];
+    const itemRe = /<item[\s>][\s\S]*?<\/item>/gi;
+    const entryRe = /<entry[\s>][\s\S]*?<\/entry>/gi;
+    let m = itemRe.exec(xml);
+    while (m !== null) {
+        blocks.push(m[0]);
+        m = itemRe.exec(xml);
+    }
+    if (blocks.length === 0) {
+        m = entryRe.exec(xml);
+        while (m !== null) {
+            blocks.push(m[0]);
+            m = entryRe.exec(xml);
+        }
+    }
+    return blocks.map((b) => {
+        const title = stripMarkup(subElement(b, 'title') ?? '');
+        const link = subLink(b) ?? '';
+        const category = stripMarkup(subElement(b, 'category') ?? '') || null;
+        const pubRaw = subElement(b, 'pubDate') ?? subElement(b, 'published') ?? subElement(b, 'updated');
+        const summary = stripMarkup(
+            subElement(b, 'content:encoded') ?? subElement(b, 'description') ?? subElement(b, 'summary') ?? title,
+        );
+        return { title, link, publishedAt: toIsoDate(pubRaw), summary, category };
+    });
+}
+
+export function computeRssDocId(seed: string): string {
+    return createHash('sha256').update(seed).digest('hex').slice(0, 16);
+}
+
+/** Map a feed entry to the kk Doc shape; source:'aihot' keeps the aihot plan tier. */
+export function mapRssEntryToDoc(entry: RssEntry, feedUrl: string): Doc {
+    const id = entry.link ? computeRssDocId(entry.link) : computeRssDocId(`${entry.title}${entry.summary}`);
+    let sourceName = 'aihot-feed';
+    try {
+        sourceName = new URL(feedUrl).hostname;
+    } catch {
+        // keep 'aihot-feed' fallback
+    }
+    return {
+        id,
+        title: entry.title || 'Untitled AI News',
+        sourceUri: entry.link || feedUrl,
+        body: `${entry.title}\n\n${entry.summary}\n`,
+        mediaType: 'text/markdown',
+        metadata: {
+            source: 'aihot',
+            sourceName,
+            publishedAt: entry.publishedAt ?? undefined,
+            discoveredAt: new Date().toISOString(),
+            originalUrl: entry.link || undefined,
+            category: entry.category ?? undefined,
+        },
+    };
+}
+
+export async function fetchRssDocs(
+    feedUrl: string,
+    limit: number,
+    fetchLike: RssFetchLike = fetch,
+): Promise<{ docs: Doc[]; asOf: string }> {
+    const res = await fetchLike(feedUrl, {
+        headers: { 'user-agent': 'aihot-ingest/1.0.0 (RSS source)' },
+    });
+    if (!res.ok) {
+        throw new Error(`aihot-ingest RSS fetch failed: HTTP ${res.status} from ${feedUrl}`);
+    }
+    const xml = await res.text();
+    const entries = parseFeed(xml);
+    // Feed order is newest-first for RSS; atom may vary — prefer publication order.
+    const seen = new Set<string>();
+    const docs: Doc[] = [];
+    for (const entry of entries) {
+        if (docs.length >= limit) break;
+        const key = entry.link || entry.title;
+        if (seen.has(key)) continue;
+        seen.add(key);
+        docs.push(mapRssEntryToDoc(entry, feedUrl));
+    }
+    return { docs, asOf: new Date().toISOString() };
+}

package/plugins/ingestions/horizon-ingest/package.json

@@ -0,0 +1,17 @@
+{
+    "name": "@gobing-ai/horizon-ingest",
+    "type": "module",
+    "private": true,
+    "scripts": {
+        "typecheck": "tsc --noEmit"
+    },
+    "dependencies": {
+        "@gobing-ai/kk-core": "workspace:*",
+        "@gobing-ai/ts-runtime": "catalog:",
+        "@gobing-ai/ts-utils": "catalog:",
+        "zod": "4.4.3"
+    },
+    "devDependencies": {
+        "@types/bun": "1.3.14"
+    }
+}