@gobing-ai/knowledge-kit 0.0.6 → 0.0.8

package/plugins/ingestions/karakeep-local/src/index.ts

@@ -1,8 +1,12 @@
 import { createHash } from 'node:crypto';
-import { mkdir, readdir, readFile, stat, writeFile } from 'node:fs/promises';
-import { dirname, join, relative } from 'node:path';
+import { dirname, relative } from 'node:path';
 import { parseArgs } from 'node:util';
 import type { Doc } from '@gobing-ai/kk-core';
+import { getLogger, initializeLogger } from '@gobing-ai/ts-infra';
+import { createNodeFileSystem, walkDir } from '@gobing-ai/ts-runtime';
+import { echoError } from '@gobing-ai/ts-utils';
+
+const log = getLogger('kk.ingestion.karakeep-local');
 
 interface ParsedMeta {
     title?: string;
@@ -17,27 +21,26 @@ function computeHash(str: string): string {
 }
 
 async function collectFiles(dir: string): Promise<string[]> {
-    const entries = await readdir(dir, { recursive: true });
-    const files: string[] = [];
-    for (const entry of entries) {
-        const fullPath = join(dir, entry);
-        const st = await stat(fullPath);
-        if (st.isFile()) {
-            files.push(fullPath);
-        }
-    }
-    return files.sort();
+    // walkDir is cycle-safe and root-confined: a directory symlink pointing back at
+    // an ancestor is walked at most once, and one escaping `dir` is skipped.
+    return (await walkDir(dir, createNodeFileSystem())).sort();
 }
 
 export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
+    const fs = createNodeFileSystem();
     try {
-        const inStat = await stat(inDir);
+        // The FileSystem seam returns null for a missing path where node's stat throws;
+        // both cases must still surface as "Inaccessible source path" below.
+        const inStat = await fs.stat(inDir);
+        if (inStat === null) {
+            throw new Error(`Source path does not exist: ${inDir}`);
+        }
         if (!inStat.isDirectory()) {
             throw new Error(`Source path is not a directory: ${inDir}`);
         }
     } catch (err: unknown) {
         const msg = `Inaccessible source path: ${inDir}`;
-        console.error(`Error: ${msg}`, err);
+        echoError(`Error: ${msg}`);
         throw new Error(msg, { cause: err });
     }
 
@@ -62,14 +65,14 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
 
         if (contentMd) {
             processedFiles.add(contentMd);
-            const mdText = await readFile(contentMd, 'utf-8');
+            const mdText = await fs.readFile(contentMd);
             let metaObj: ParsedMeta = {};
             if (metaJson) {
                 processedFiles.add(metaJson);
                 try {
-                    metaObj = JSON.parse(await readFile(metaJson, 'utf-8')) as ParsedMeta;
+                    metaObj = JSON.parse(await fs.readFile(metaJson)) as ParsedMeta;
                 } catch (err: unknown) {
-                    console.error(`Warning: corrupt/unparsable meta file at ${metaJson}`, err);
+                    log.warn(`Warning: corrupt/unparsable meta file at ${metaJson}`, { error: err });
                 }
             }
 
@@ -99,7 +102,7 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
 
         if (f.endsWith('.json')) {
             try {
-                const jsonText = await readFile(f, 'utf-8');
+                const jsonText = await fs.readFile(f);
                 const jsonObj = JSON.parse(jsonText) as ParsedMeta;
                 const { title, sourceUri, mediaType, body, ...extraMeta } = jsonObj;
                 if (typeof body === 'string') {
@@ -112,13 +115,13 @@ export async function processKarakeepFolder(inDir: string): Promise<Doc[]> {
                         metadata: Object.keys(extraMeta).length > 0 ? extraMeta : undefined,
                     });
                 } else {
-                    console.error(`Warning: JSON entry missing string body at ${f}`);
+                    log.warn(`Warning: JSON entry missing string body at ${f}`);
                 }
             } catch (err: unknown) {
-                console.error(`Warning: corrupt/unparsable JSON file at ${f}`, err);
+                log.warn(`Warning: corrupt/unparsable JSON file at ${f}`, { error: err });
             }
         } else if (f.endsWith('.md')) {
-            const mdText = await readFile(f, 'utf-8');
+            const mdText = await fs.readFile(f);
             docs.push({
                 id,
                 body: mdText,
@@ -140,31 +143,33 @@ export async function main() {
     });
 
     if (!values.in || !values.out) {
-        console.error('Error: Missing required arguments --in or --out');
+        echoError('Error: Missing required arguments --in or --out');
         process.exit(1);
     }
 
+    const fs = createNodeFileSystem();
     try {
         const candidateFiles = await collectFiles(values.in);
         const docs = await processKarakeepFolder(values.in);
 
         if (docs.length === 0 && candidateFiles.length > 0) {
-            console.error(`Fatal: source yielded zero ingestible documents at ${values.in}`);
+            echoError(`Fatal: source yielded zero ingestible documents at ${values.in}`);
             process.exit(1);
         }
 
         const outDir = dirname(values.out);
         if (outDir && outDir !== '.') {
-            await mkdir(outDir, { recursive: true });
+            await fs.ensureDir(outDir);
         }
 
-        await writeFile(values.out, JSON.stringify(docs, null, 2), 'utf-8');
+        await fs.writeFile(values.out, JSON.stringify(docs, null, 2));
     } catch (err: unknown) {
-        console.error('Fatal error during karakeep-local ingestion:', err);
+        echoError(`Fatal error during karakeep-local ingestion: ${err instanceof Error ? err.message : String(err)}`);
         process.exit(1);
     }
 }
 
 if (import.meta.main) {
+    await initializeLogger({ console: process.env.NODE_ENV !== 'test', json: true, level: 'warn' });
     main();
 }

package/plugins/ingestions/karakeep-local/tsconfig.json

@@ -0,0 +1,4 @@
+{
+    "extends": "../../../tooling/typescript/base.json",
+    "include": ["src", "tests"]
+}

package/plugins/ingestions/web-search/package.json

@@ -6,7 +6,10 @@
         "typecheck": "tsc --noEmit"
     },
     "dependencies": {
-        "@gobing-ai/kk-core": "workspace:*"
+        "@gobing-ai/kk-core": "workspace:*",
+        "@gobing-ai/ts-infra": "catalog:",
+        "@gobing-ai/ts-runtime": "catalog:",
+        "@gobing-ai/ts-utils": "catalog:"
     },
     "devDependencies": {
         "@types/bun": "1.3.14"

package/plugins/ingestions/web-search/src/index.ts

@@ -1,8 +1,12 @@
 import { createHash } from 'node:crypto';
-import { mkdir, readFile, writeFile } from 'node:fs/promises';
-import { dirname } from 'node:path';
+import { dirname, join } from 'node:path';
 import { parseArgs } from 'node:util';
 import { type Doc, DocListSchema } from '@gobing-ai/kk-core';
+import { getLogger, initializeLogger } from '@gobing-ai/ts-infra';
+import { createNodeFileSystem } from '@gobing-ai/ts-runtime';
+import { echoError } from '@gobing-ai/ts-utils';
+
+const log = getLogger('kk.ingestion.web-search');
 
 /** Validated `--in` input per the 0054 contract (`topic` required; `maxResults`/`fixture` optional). */
 interface InInput {
@@ -40,11 +44,269 @@ const FIXTURE_ENTRIES: FixtureEntry[] = [
     },
 ];
 
-/** Deterministic Doc id: sha256(sourceUri) hex prefix, same idea as karakeep-local (0054 Block 4). */
-function computeId(uri: string): string {
+/** Deterministic Doc id: sha256(sourceUri) hex prefix, same idea as karakeep-local (0054 Block 4).
+ * Exported so 0059's local-Doc mapping can stay consistent if a later helper imports it (R3). */
+export function computeId(uri: string): string {
     return createHash('sha256').update(uri).digest('hex').slice(0, 16);
 }
 
+/** Fetch signature the live path accepts, so tests can inject a fake (CI never hits the network). */
+export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
+
+/** markitdown normalizer — HTML in, markdown out (ADR-013 D3). Throws on non-zero exit or
+ * empty output; callers treat a throw as unusable content (per-URL fallback, D5). */
+export type MarkitdownRunner = (html: string) => Promise<string>;
+
+/** Live-path dependencies — fetch, env, clock, scrape, and markitdown are all injectable for tests. */
+export interface SearchDeps {
+    fetch: FetchLike;
+    env: Record<string, string | undefined>;
+    now: () => Date;
+    /** Scrape transport; defaults to `deps.fetch` so tests can route both Firecrawl calls separately. */
+    scrape?: FetchLike;
+    /** markitdown runner; when absent in live mode, `main` probes PATH and fails loud (R6). */
+    markitdown?: MarkitdownRunner;
+}
+
+/** Real markitdown runner: `markitdown -x .html`, HTML on stdin, markdown on stdout. The
+ * `-x .html` extension hint is required — bare stdin passes HTML through unconverted.
+ * Spawns the absolute path resolved via Bun.which (with an explicit PATH option — plain
+ * argv[0] lookup can resolve against a startup PATH snapshot, missing runtime edits). */
+export const realMarkitdown: MarkitdownRunner = async (html: string): Promise<string> => {
+    const bin = Bun.which('markitdown', { PATH: process.env.PATH });
+    if (!bin) {
+        throw new Error('markitdown is not on PATH');
+    }
+    const proc = Bun.spawn([bin, '-x', '.html'], { stdin: 'pipe', stdout: 'pipe', stderr: 'pipe' });
+    proc.stdin.write(html);
+    await proc.stdin.end();
+    const [stdout, stderrText, exitCode] = await Promise.all([
+        new Response(proc.stdout).text(),
+        new Response(proc.stderr).text(),
+        proc.exited,
+    ]);
+    if (exitCode !== 0) {
+        const detail = stderrText.trim();
+        throw new Error(`markitdown exited ${exitCode}${detail ? `: ${detail}` : ''}`);
+    }
+    const markdown = stdout.trim();
+    if (markdown.length === 0) {
+        throw new Error('markitdown produced empty output');
+    }
+    return markdown;
+};
+
+/** One Firecrawl `data.web[]` item — the subset of fields 0054 Block 3/4 consume. */
+interface FirecrawlWebItem {
+    url?: unknown;
+    title?: unknown;
+    markdown?: unknown;
+    position?: unknown;
+}
+
+interface FirecrawlSearchResponse {
+    success?: unknown;
+    error?: unknown;
+    data?: { web?: FirecrawlWebItem[] | null } | null;
+}
+
+/** One Firecrawl `/v2/scrape` body — the subset the cache pass consumes. */
+interface FirecrawlScrapeResponse {
+    success?: unknown;
+    data?: { html?: unknown; markdown?: unknown } | null;
+}
+
+/**
+ * Live Firecrawl search → Doc[] (0054 Blocks 2–5). Throws an Error whose message is the
+ * fail-loud diagnostic; `main` prefixes it with `web-search failed: ` and returns 1.
+ * Never writes `--out` on a thrown path (no DocListSchema.parse, no file write here).
+ */
+export async function searchFirecrawl(input: InInput, deps: SearchDeps): Promise<Doc[]> {
+    const key = deps.env.FIRECRAWL_API_KEY;
+    if (!key) {
+        // 0054 Block 5 row 2 — key required by this plugin's contract, not by Firecrawl.
+        throw new Error('FIRECRAWL_API_KEY is unset');
+    }
+
+    let res: Response;
+    try {
+        res = await deps.fetch('https://api.firecrawl.dev/v2/search', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                Authorization: `Bearer ${key}`,
+            },
+            body: JSON.stringify({
+                query: input.topic,
+                limit: input.maxResults,
+                sources: ['web'],
+                scrapeOptions: { formats: ['markdown'] },
+            }),
+        });
+    } catch (err: unknown) {
+        // 0054 Block 5 row 3 — transport error (DNS, refused, TLS, fetch throws).
+        throw new Error(`Firecrawl network error: ${err instanceof Error ? err.message : String(err)}`);
+    }
+
+    let bodyText: string;
+    try {
+        bodyText = await res.text();
+    } catch {
+        throw new Error(`Firecrawl HTTP ${res.status}: empty response body`);
+    }
+
+    if (!res.ok || res.status !== 200) {
+        // 0054 Block 5 rows 4–5 — 401/402/400/404/408/429/5xx. No silent retry in v1.
+        throw new Error(`Firecrawl HTTP ${res.status}: ${extractError(bodyText)}`);
+    }
+
+    let parsed: FirecrawlSearchResponse;
+    try {
+        parsed = JSON.parse(bodyText) as FirecrawlSearchResponse;
+    } catch {
+        // 200 but not JSON → malformed response (row 5).
+        throw new Error('Firecrawl malformed response: invalid JSON');
+    }
+
+    if (parsed.success === false) {
+        // 0054 Block 5 row 5 — `success:false` in a 200 body.
+        throw new Error(`Firecrawl HTTP 200: ${extractError(bodyText)}`);
+    }
+
+    if (parsed.success !== true) {
+        throw new Error('Firecrawl malformed response: success missing');
+    }
+
+    // Row 5 (P4): a 200 `success:true` with `data` absent/null is malformed, not zero hits.
+    if (parsed.data === null || parsed.data === undefined) {
+        throw new Error('Firecrawl malformed response: data missing');
+    }
+
+    const web = parsed.data.web ?? [];
+
+    // Row 6 — true zero hits: write `[]`, exit 0.
+    if (web.length === 0) {
+        return [];
+    }
+
+    const docs: Doc[] = [];
+    let skipped = 0;
+    for (const [rank, item] of web.entries()) {
+        const url = typeof item.url === 'string' ? item.url : undefined;
+        const markdown = typeof item.markdown === 'string' ? item.markdown : undefined;
+        const body = markdown?.trim() ?? '';
+        if (!url || body.length === 0) {
+            // Row 7 — partial item failure: skip + stderr warn, continue.
+            skipped += 1;
+            log.warn(`web-search: skipping result without usable markdown (url=${url ?? 'missing'})`);
+            continue;
+        }
+        docs.push({
+            id: computeId(url),
+            body,
+            title: typeof item.title === 'string' ? item.title : undefined,
+            sourceUri: url,
+            mediaType: 'text/markdown',
+            metadata: {
+                query: input.topic,
+                rank,
+                position: typeof item.position === 'number' ? item.position : undefined,
+                scraped_at: deps.now().toISOString(),
+            },
+        });
+    }
+
+    // Row 8 — hits > 0 but every item skipped: deliberate departure from C1 (0054 Block 6b).
+    if (docs.length === 0 && skipped > 0) {
+        throw new Error('Firecrawl returned hits but no usable markdown bodies');
+    }
+
+    return docs;
+}
+
+/** Best-effort `error` message from a Firecrawl error body; empty when unavailable. */
+function extractError(bodyText: string): string {
+    try {
+        const parsed = JSON.parse(bodyText) as { error?: unknown };
+        const error = parsed.error;
+        if (typeof error === 'string' && error.length > 0) {
+            return error;
+        }
+        if (error !== undefined && error !== null) {
+            return JSON.stringify(error);
+        }
+    } catch {
+        // Non-JSON error body — fall through to empty.
+    }
+    return '';
+}
+
+/** Scrape + normalize + cache pass over the search results (ADR-013 D1–D6). Sequential
+ * per Doc — deliberate, no parallel fan-out (rate-limit friendly, deterministic). Cache hit
+ * (`references/<doc-id>.md` exists) reuses the file with no new /v2/scrape call (R3/R4);
+ * otherwise scrape → normalize (html via markitdown, markdown as-is) → write cache → body
+ * swap. Per-URL failure or unusable content warns and keeps the snippet body (R5/R11) — never
+ * fatal. Returns a new Doc[]; input order and non-body fields are untouched. */
+export async function scrapeAndCacheDocs(docs: Doc[], deps: SearchDeps, outPath: string): Promise<Doc[]> {
+    const fs = createNodeFileSystem();
+    const cacheDir = join(dirname(outPath), 'references');
+    const scrapeFetch = deps.scrape ?? deps.fetch;
+    const out: Doc[] = [];
+    for (const doc of docs) {
+        const uri = doc.sourceUri;
+        if (!uri) {
+            out.push(doc);
+            continue;
+        }
+        const cacheFile = join(cacheDir, `${doc.id}.md`);
+        try {
+            const cached = await fs.readFile(cacheFile);
+            if (cached.trim().length > 0) {
+                out.push({ ...doc, body: cached });
+                continue;
+            }
+        } catch {
+            // Cache miss — fall through to the live scrape.
+        }
+        try {
+            const key = deps.env.FIRECRAWL_API_KEY;
+            if (!key) throw new Error('FIRECRAWL_API_KEY is unset');
+            const res = await scrapeFetch('https://api.firecrawl.dev/v2/scrape', {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    Authorization: `Bearer ${key}`,
+                },
+                body: JSON.stringify({ url: uri, formats: ['html', 'markdown'] }),
+            });
+            if (!res.ok || res.status !== 200) {
+                throw new Error(`Firecrawl HTTP ${res.status}`);
+            }
+            const parsed = JSON.parse(await res.text()) as FirecrawlScrapeResponse;
+            if (parsed.success !== true || parsed.data === null || parsed.data === undefined) {
+                throw new Error('Firecrawl malformed scrape response');
+            }
+            const html = typeof parsed.data.html === 'string' ? parsed.data.html : undefined;
+            const markdown = typeof parsed.data.markdown === 'string' ? parsed.data.markdown : undefined;
+            let normalized: string;
+            if (html && html.trim().length > 0) {
+                normalized = await (deps.markitdown ?? realMarkitdown)(html);
+            } else if (markdown && markdown.trim().length > 0) {
+                normalized = markdown.trim();
+            } else {
+                throw new Error('scrape returned no usable content');
+            }
+            await fs.ensureDir(cacheDir);
+            await fs.writeFile(cacheFile, normalized);
+            out.push({ ...doc, body: normalized });
+        } catch (err: unknown) {
+            log.warn(`web-search: scrape failed (url=${uri}): ${err instanceof Error ? err.message : String(err)}`);
+            out.push(doc);
+        }
+    }
+    return out;
+}
+
 /**
  * Build the fixture `Doc[]` for a topic (0054 Block 4 mapping). Always emits
  * canned docs — the v1 plugin has no live Firecrawl path.
@@ -95,6 +357,7 @@ export function parseInput(raw: string): InInput {
 }
 
 export async function main(): Promise<number> {
+    const fs = createNodeFileSystem();
     const { values } = parseArgs({
         options: {
             in: { type: 'string' },
@@ -103,42 +366,61 @@ export async function main(): Promise<number> {
     });
 
     if (!values.in || !values.out) {
-        console.error('Error: Missing required arguments --in and --out');
+        echoError('Error: Missing required arguments --in and --out');
         return 1;
     }
 
     let input: InInput;
     try {
-        input = parseInput(await readFile(values.in, 'utf-8'));
+        input = parseInput(await fs.readFile(values.in));
     } catch (err: unknown) {
-        console.error(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
+        echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
         return 1;
     }
 
     if (!input.fixture) {
-        // 0054 Block 1/Block 8: `fixture` absent or false selects the LIVE Firecrawl path; v1 has no
-        // live client. Fail loud (Block 5 row-2 semantics: exit 1, no --out) instead of silently
-        // fabricating research docs — an obvious failure beats a surprising recovery.
-        console.error(
-            'web-search failed: live Firecrawl not implemented in v1 (pass "fixture": true to use the fixture stub)',
-        );
-        return 1;
+        // 0054 Block 1/Block 8: `fixture` absent/false = LIVE Firecrawl path.
+        try {
+            const deps: SearchDeps = {
+                fetch: globalThis.fetch,
+                env: process.env,
+                now: () => new Date(),
+            };
+            // Bun.which with an explicit PATH — without the option it can ignore runtime
+            // process.env.PATH changes (observed on Bun 1.3.14).
+            if (!deps.markitdown && !Bun.which('markitdown', { PATH: deps.env.PATH })) {
+                // R6 — live mode requires the normalizer; fail before any network call.
+                throw new Error('markitdown is not on PATH');
+            }
+            const docs = await searchFirecrawl(input, deps);
+            const cached = await scrapeAndCacheDocs(docs, deps, values.out);
+            const outDir = dirname(values.out);
+            if (outDir && outDir !== '.') {
+                await fs.ensureDir(outDir);
+            }
+            await fs.writeFile(values.out, JSON.stringify(DocListSchema.parse(cached), null, 2));
+            return 0;
+        } catch (err: unknown) {
+            echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
+            return 1;
+        }
     }
 
     try {
         const docs = DocListSchema.parse(buildFixtureDocs(input.topic, input.maxResults));
         const outDir = dirname(values.out);
         if (outDir && outDir !== '.') {
-            await mkdir(outDir, { recursive: true });
+            await fs.ensureDir(outDir);
         }
-        await writeFile(values.out, JSON.stringify(docs, null, 2), 'utf-8');
+        await fs.writeFile(values.out, JSON.stringify(docs, null, 2));
         return 0;
     } catch (err: unknown) {
-        console.error(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
+        echoError(`web-search failed: ${err instanceof Error ? err.message : String(err)}`);
         return 1;
     }
 }
 
 if (import.meta.main) {
+    await initializeLogger({ console: process.env.NODE_ENV !== 'test', json: true, level: 'warn' });
     process.exit(await main());
 }

package/plugins/kk/README.md

@@ -7,11 +7,22 @@ It is a Claude Code / Superskill plugin, **not** a kk-core product plugin.
 |------|--------|
 | `skills/` | Fat skills (`SKILL.md`) |
 | `commands/` | Thin slash-command wrappers |
-| `agents/` | Thin subagent wrappers |
+| `agents/` | Thin subagent wrappers (currently empty — see below) |
 | `hooks/` | Hook definitions |
 | `rules/` | Agent rules |
+| `workflows/` | Design-time product YAML SSOT (ADR-009). Runtime dest is `$HOME/.config/kk/workflows` via `/workflow-run` |
+
+`agents/` is reserved for thin wrappers of fat skills. It is empty: `judge-tech` /
+`judge-tone` / `judge-compliance` were retired (2026-08-19). Rubric selection lives on
+`content-judge` (`agent.run` prompt or `--judge` on `itc-generating`). Do not re-add a
+persona wrapper unless a new skill owns a distinct run procedure.
 
 `discoverPlugins` skips this directory. Product plugins (`ingestion` / `generator` /
 `publish`) live under `plugins/ingestions|generations|publishings/<name>/`.
 
 Install: `superskill install kk`.
+
+Capability files **omit a leading `kk-`**. `superskill install` prefixes the plugin name, so
+`skills/content-judge` installs as `kk:content-judge` (not `kk:kk-content-judge`). Same for `topic`,
+`storm-research`, `itc-generating`, and `/workflow-run`. Product workflow YAML (`kk-storm-research.yaml`)
+keeps its existing name — that is a workflow stem, not an installable agent capability.