@gobing-ai/knowledge-kit 0.0.11 → 0.0.13

package/plugins/ingestions/aihot-ingest/plugin.json

@@ -1,7 +1,7 @@
 {
     "name": "aihot-ingest",
     "kind": "ingestion",
-    "entry": "./src/index.ts",
+    "entry": "./dist/index.js",
     "version": "1.0.0",
-    "description": "Ingests curated AI news items from aihot.virxact.com into Doc[]"
+    "description": "Ingests fresh AI news from an RSS feed (default TechCrunch AI) into Doc[]; legacy aihot.virxact.com API behind AIHOT_SOURCE=api"
 }

package/plugins/ingestions/aihot-ingest/src/index.ts

@@ -6,9 +6,11 @@ import { echoError } from '@gobing-ai/ts-utils';
 import { z } from 'zod';
 import { type AihotDeps, createAihotClient } from './client';
 import { mapAihotItemsToDocs } from './mapper';
+import { AIHOT_FEED_URL_DEFAULT, fetchRssDocs, type RssFetchLike } from './rss';
 
 export * from './client';
 export * from './mapper';
+export * from './rss';
 
 export interface IngestionOptions {
     out: string;
@@ -38,8 +40,26 @@ function parseLimit(value: string): number {
 export async function processIngestionIO(
     options: IngestionOptions,
     depsOverride?: Partial<AihotDeps>,
+    rssFetchLike: RssFetchLike = fetch,
 ): Promise<IngestionResult> {
     const fs = createNodeFileSystem();
+
+    // RSS is the default source (2026-09-08 dogfood: the aihot.virxact.com API served a
+    // 2020–2025 archive — every item failed the 7-day freshness gate; see rss.ts). The API
+    // path stays reachable with AIHOT_SOURCE=api for when the CMS carries live curated items.
+    const source = process.env.AIHOT_SOURCE?.trim() || 'rss';
+    if (source === 'api') {
+        return processApiSource(options, fs, depsOverride);
+    }
+    return processRssSource(options, fs, rssFetchLike);
+}
+
+/** Legacy aihot.virxact.com API source (cursor/stateFile contract preserved). */
+async function processApiSource(
+    options: IngestionOptions,
+    fs: ReturnType<typeof createNodeFileSystem>,
+    depsOverride?: Partial<AihotDeps>,
+): Promise<IngestionResult> {
     const client = createAihotClient(depsOverride);
 
     let effectiveCursor = options.cursor?.trim();
@@ -68,30 +88,69 @@ export async function processIngestionIO(
     }
 
     const docs = DocListSchema.parse(mapAihotItemsToDocs(items));
-
-    const outDir = dirname(options.out);
-    if (outDir && outDir !== '.') {
-        await fs.ensureDir(outDir);
-    }
-    await atomicWriteJson(options.out, docs, fs);
+    await writeOutput(fs, options, docs);
 
     if (options.stateFile) {
-        const stateDir = dirname(options.stateFile);
-        if (stateDir && stateDir !== '.') {
-            await fs.ensureDir(stateDir);
-        }
-        const statePayload = {
+        await writeState(fs, options.stateFile, {
             cursor,
             asOf,
             lastSyncAt: new Date().toISOString(),
             itemCount: docs.length,
-        };
-        await atomicWriteJson(options.stateFile, statePayload, fs);
+        });
     }
 
     return { docs, cursor, asOf };
 }
 
+/** RSS feed source (default): stateless fetch, no cursor (feed order is newest-first). */
+async function processRssSource(
+    options: IngestionOptions,
+    fs: ReturnType<typeof createNodeFileSystem>,
+    rssFetchLike: RssFetchLike,
+): Promise<IngestionResult> {
+    const feedUrl = process.env.AIHOT_FEED_URL?.trim() || AIHOT_FEED_URL_DEFAULT;
+    const limit = options.limit ?? 10;
+    const { docs, asOf } = await fetchRssDocs(feedUrl, limit, rssFetchLike);
+    const validated = DocListSchema.parse(docs);
+    await writeOutput(fs, options, validated);
+
+    if (options.stateFile) {
+        await writeState(fs, options.stateFile, {
+            source: 'rss',
+            feedUrl,
+            asOf,
+            lastSyncAt: new Date().toISOString(),
+            itemCount: validated.length,
+        });
+    }
+
+    return { docs: validated, cursor: '', asOf };
+}
+
+async function writeOutput(
+    fs: ReturnType<typeof createNodeFileSystem>,
+    options: IngestionOptions,
+    docs: Doc[],
+): Promise<void> {
+    const outDir = dirname(options.out);
+    if (outDir && outDir !== '.') {
+        await fs.ensureDir(outDir);
+    }
+    await atomicWriteJson(options.out, docs, fs);
+}
+
+async function writeState(
+    fs: ReturnType<typeof createNodeFileSystem>,
+    stateFile: string,
+    payload: Record<string, unknown>,
+): Promise<void> {
+    const stateDir = dirname(stateFile);
+    if (stateDir && stateDir !== '.') {
+        await fs.ensureDir(stateDir);
+    }
+    await atomicWriteJson(stateFile, payload, fs);
+}
+
 export async function main(depsOverride?: Partial<AihotDeps>): Promise<number> {
     let values: { in?: string; out?: string; limit?: string; cursor?: string; 'state-file'?: string };
     try {

package/plugins/ingestions/aihot-ingest/src/mapper.ts

@@ -16,6 +16,7 @@ export function mapAihotItemToDoc(item: AihotItem): Doc {
     const body = `${item.summary?.trim() || item.title.trim()}\n`;
 
     const metadata: Record<string, unknown> = {
+        source: 'aihot',
         sourceName: item.source.name,
         publishedAt: item.publishedAt,
         discoveredAt: item.discoveredAt,

package/plugins/ingestions/aihot-ingest/src/rss.ts

@@ -0,0 +1,151 @@
+import { createHash } from 'node:crypto';
+import type { Doc } from '@gobing-ai/kk-core';
+
+/**
+ * RSS/Atom feed fetch + item → Doc mapping for the aihot-ingest RSS source.
+ *
+ * Why RSS (2026-09-08, dogfood): aihot.virxact.com's API served a 2020–2025 blog
+ * archive — all 10 items were dropped every run by the workflow's 7-day freshness
+ * gate, so the "curated" aihot source contributed zero usable items while carrying
+ * the highest selection tier (SOURCE_BASE aihot=30, importance 4). The RSS source
+ * restores aihot as a genuinely fresh feed; the API path stays selectable via
+ * AIHOT_SOURCE=api for when the CMS has live curated content.
+ *
+ * Keeps metadata.source='aihot' so planEpisode's aihot tier (and importance base 4)
+ * still applies to feed items.
+ */
+
+/** Default fresh AI-news feed (RSS 2.0, high volume, reachable without auth). */
+export const AIHOT_FEED_URL_DEFAULT = 'https://techcrunch.com/category/artificial-intelligence/feed/';
+
+export type RssFetchLike = (url: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+export interface RssEntry {
+    title: string;
+    link: string;
+    publishedAt: string | null; // ISO-8601 or null
+    summary: string;
+    category: string | null;
+}
+
+/** Strip XML/HTML tags and normalize common entities to plain text. */
+function stripMarkup(raw: string): string {
+    return raw
+        .replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
+        .replace(/<[^>]+>/g, ' ')
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&amp;/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;|&apos;/g, "'")
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+
+/** RSS <pubDate> (RFC 822) or Atom <published>/<updated> (ISO-8601) → ISO-8601. */
+function toIsoDate(raw: string | undefined): string | null {
+    if (!raw?.trim()) return null;
+    const t = Date.parse(raw.trim().replace('UT', 'GMT'));
+    return Number.isFinite(t) ? new Date(t).toISOString() : null;
+}
+
+/** Extract a single sub-element's text from an element body. */
+function subElement(body: string, tag: string): string | undefined {
+    const re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`, 'i');
+    const m = body.match(re);
+    return m?.[1];
+}
+
+/** Extract a link: RSS <link>text</link> or Atom <link href="..."/> (first match wins). */
+function subLink(body: string): string | undefined {
+    const text = subElement(body, 'link');
+    if (text?.trim()) return text.trim();
+    const m = body.match(/<link[^>]*\bhref\s*=\s*"([^"]+)"/i);
+    return m?.[1];
+}
+
+/** Block-split RSS <item>… or Atom <entry>… elements and map each to an RssEntry. */
+export function parseFeed(xml: string): RssEntry[] {
+    const blocks: string[] = [];
+    const itemRe = /<item[\s>][\s\S]*?<\/item>/gi;
+    const entryRe = /<entry[\s>][\s\S]*?<\/entry>/gi;
+    let m = itemRe.exec(xml);
+    while (m !== null) {
+        blocks.push(m[0]);
+        m = itemRe.exec(xml);
+    }
+    if (blocks.length === 0) {
+        m = entryRe.exec(xml);
+        while (m !== null) {
+            blocks.push(m[0]);
+            m = entryRe.exec(xml);
+        }
+    }
+    return blocks.map((b) => {
+        const title = stripMarkup(subElement(b, 'title') ?? '');
+        const link = subLink(b) ?? '';
+        const category = stripMarkup(subElement(b, 'category') ?? '') || null;
+        const pubRaw = subElement(b, 'pubDate') ?? subElement(b, 'published') ?? subElement(b, 'updated');
+        const summary = stripMarkup(
+            subElement(b, 'content:encoded') ?? subElement(b, 'description') ?? subElement(b, 'summary') ?? title,
+        );
+        return { title, link, publishedAt: toIsoDate(pubRaw), summary, category };
+    });
+}
+
+export function computeRssDocId(seed: string): string {
+    return createHash('sha256').update(seed).digest('hex').slice(0, 16);
+}
+
+/** Map a feed entry to the kk Doc shape; source:'aihot' keeps the aihot plan tier. */
+export function mapRssEntryToDoc(entry: RssEntry, feedUrl: string): Doc {
+    const id = entry.link ? computeRssDocId(entry.link) : computeRssDocId(`${entry.title}${entry.summary}`);
+    let sourceName = 'aihot-feed';
+    try {
+        sourceName = new URL(feedUrl).hostname;
+    } catch {
+        // keep 'aihot-feed' fallback
+    }
+    return {
+        id,
+        title: entry.title || 'Untitled AI News',
+        sourceUri: entry.link || feedUrl,
+        body: `${entry.title}\n\n${entry.summary}\n`,
+        mediaType: 'text/markdown',
+        metadata: {
+            source: 'aihot',
+            sourceName,
+            publishedAt: entry.publishedAt ?? undefined,
+            discoveredAt: new Date().toISOString(),
+            originalUrl: entry.link || undefined,
+            category: entry.category ?? undefined,
+        },
+    };
+}
+
+export async function fetchRssDocs(
+    feedUrl: string,
+    limit: number,
+    fetchLike: RssFetchLike = fetch,
+): Promise<{ docs: Doc[]; asOf: string }> {
+    const res = await fetchLike(feedUrl, {
+        headers: { 'user-agent': 'aihot-ingest/1.0.0 (RSS source)' },
+    });
+    if (!res.ok) {
+        throw new Error(`aihot-ingest RSS fetch failed: HTTP ${res.status} from ${feedUrl}`);
+    }
+    const xml = await res.text();
+    const entries = parseFeed(xml);
+    // Feed order is newest-first for RSS; atom may vary — prefer publication order.
+    const seen = new Set<string>();
+    const docs: Doc[] = [];
+    for (const entry of entries) {
+        if (docs.length >= limit) break;
+        const key = entry.link || entry.title;
+        if (seen.has(key)) continue;
+        seen.add(key);
+        docs.push(mapRssEntryToDoc(entry, feedUrl));
+    }
+    return { docs, asOf: new Date().toISOString() };
+}