@phuetz/code-buddy 1.6.0 → 2.0.0

package/dist/agent/deep-research-ckg.js

@@ -0,0 +1,250 @@
+/**
+ * Deep Research — Phase D (Collective Knowledge Graph bridge).
+ *
+ * Phases A/B/C produce a self-contained cited report from FRESH web sources and
+ * then forget everything. The Collective Knowledge Graph (`memory/collective-
+ * knowledge-graph.ts`) is the collective's SHARED, cross-run/cross-agent memory
+ * — an append-only ledger whose `ingest()` auto-links a discovery to its nearest
+ * neighbours (supports / contradicts / related_to) and CORROBORATES a fact that
+ * several independent agents find, and whose `recallHybrid()` retrieves prior
+ * knowledge with local embeddings ($0, no LLM at retrieval).
+ *
+ * Today the two are DISCONNECTED: a Deep Research run neither feeds the graph nor
+ * consults what the collective already knows. Phase D bridges them, both ways and
+ * both OPT-IN:
+ *
+ *   1. INGEST (write)  — at the END of a run, the deduped web sources are ingested
+ *                        into the CKG (one `discovery` node each, url as stable
+ *                        name + a bounded excerpt as text). Benefits: accumulation
+ *                        across runs, auto-linking, and corroboration when several
+ *                        runs/agents surface the same fact. Idempotent by design:
+ *                        the CKG dedups on contentHash (same url + same excerpt →
+ *                        reinforce, not duplicate), and we fingerprint-dedup the
+ *                        batch first (reusing Phase-A `contentFingerprint`).
+ *   2. RECALL (read)   — at the START of a run, the collective's prior knowledge
+ *                        for the question is recalled (bounded top-K) and injected
+ *                        into the produced report as a DISTINCTLY-MARKED
+ *                        "## Mémoire collective" section with its own `[Mk]`
+ *                        citation namespace — never confused with a fresh web
+ *                        source.
+ *
+ * STRICTLY ADDITIVE / never-throws: with the bridge absent or `enabled: false`
+ * the wrapper delegates to the base A/B/C runner VERBATIM — no recall, no ingest,
+ * an untouched report/sources (byte-identical). The single side-effecting edge —
+ * the CKG itself — is the INJECTABLE `CkgBridge`, so the whole pipeline is unit-
+ * testable with zero ledger / zero network. Any bridge failure (recall or ingest)
+ * degrades silently: the research continues normally. The CKG is a bonus, never a
+ * hard dependency.
+ *
+ * @module agent/deep-research-ckg
+ */
+import { logger } from '../utils/logger.js';
+import { contentFingerprint, fingerprintSimilarity } from './deep-research.js';
+const DEFAULT_RECALL_LIMIT = 6;
+const RECALL_LIMIT_MAX = 20;
+const DEFAULT_EXCERPT_CHARS = 600;
+const EXCERPT_CHARS_MIN = 120;
+const EXCERPT_CHARS_MAX = 4000;
+/** Near-duplicate excerpts collapse in the ingest batch (reuse Phase-A dedup threshold). */
+const INGEST_DEDUP_THRESHOLD = 0.8;
+function clampInt(v, def, min, max) {
+    const n = typeof v === 'number' && Number.isFinite(v) ? Math.floor(v) : def;
+    return Math.max(min, Math.min(max, n));
+}
+function errMsg(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+function collapseWhitespace(s) {
+    return s.replace(/\s+/g, ' ').trim();
+}
+// ============================================================================
+// Recall (read) — bounded, never-throws
+// ============================================================================
+/**
+ * Recall prior collective knowledge for `query`, bounded to top-K. Never throws:
+ * a bridge failure (or an unusable payload) yields []. The results are cleaned
+ * (non-empty text) and hard-capped at K.
+ */
+export async function recallCollectiveMemory(bridge, query, limit) {
+    const k = clampInt(limit, DEFAULT_RECALL_LIMIT, 1, RECALL_LIMIT_MAX);
+    const q = (query ?? '').trim();
+    if (!q)
+        return [];
+    try {
+        const hits = await bridge.recall(q, k);
+        if (!Array.isArray(hits))
+            return [];
+        return hits
+            .filter((h) => !!h && typeof h.text === 'string' && h.text.trim().length > 0)
+            .slice(0, k);
+    }
+    catch (err) {
+        logger.debug(`[deep-research-ckg] recall failed: ${errMsg(err)}`);
+        return [];
+    }
+}
+// ============================================================================
+// Ingest (write) — bounded, idempotent, never-throws
+// ============================================================================
+/**
+ * Fingerprint-dedup + excerpt-bound an ingest batch (reuses the Phase-A
+ * `contentFingerprint`/Jaccard so a near-duplicate excerpt is not re-ingested).
+ * The batch is already URL-unique and Phase-A-deduped upstream; this is a cheap
+ * belt-and-suspenders pass. Fail-open: an unfingerprintable source is kept.
+ */
+export function prepareIngestBatch(sources, excerptChars) {
+    const out = [];
+    const prints = [];
+    const seenUrls = new Set();
+    for (const s of sources) {
+        const url = typeof s?.url === 'string' ? s.url.trim() : '';
+        const content = typeof s?.content === 'string' ? collapseWhitespace(s.content) : '';
+        if (!url || !content || seenUrls.has(url))
+            continue;
+        let print = [];
+        try {
+            print = contentFingerprint(content);
+        }
+        catch {
+            print = [];
+        }
+        let isDup = false;
+        if (print.length > 0) {
+            for (const prev of prints) {
+                if (fingerprintSimilarity(print, prev) >= INGEST_DEDUP_THRESHOLD) {
+                    isDup = true;
+                    break;
+                }
+            }
+        }
+        if (isDup)
+            continue;
+        seenUrls.add(url);
+        prints.push(print);
+        out.push({ url, title: collapseWhitespace(s.title ?? url) || url, content: content.slice(0, excerptChars) });
+    }
+    return out;
+}
+/**
+ * Ingest the deduped sources into the collective graph. Bounded to the batch
+ * size (itself capped by Phase A), never throws (a bridge failure yields 0).
+ */
+export async function ingestCollectedSources(bridge, sources, meta, excerptChars) {
+    const chars = clampInt(excerptChars, DEFAULT_EXCERPT_CHARS, EXCERPT_CHARS_MIN, EXCERPT_CHARS_MAX);
+    const batch = prepareIngestBatch(Array.isArray(sources) ? sources : [], chars);
+    if (batch.length === 0)
+        return 0;
+    try {
+        const n = await bridge.ingest(batch, meta);
+        return typeof n === 'number' && Number.isFinite(n) ? Math.max(0, Math.min(batch.length, Math.floor(n))) : 0;
+    }
+    catch (err) {
+        logger.debug(`[deep-research-ckg] ingest failed: ${errMsg(err)}`);
+        return 0;
+    }
+}
+// ============================================================================
+// Report augmentation — recalled memory injected, cited distinctly
+// ============================================================================
+/**
+ * Inject the recalled collective knowledge into the report as a DISTINCT
+ * "## Mémoire collective" section, placed just before "## Références" so the web
+ * references stay last. Each entry gets a `[Mk]` marker — a separate citation
+ * namespace from the web `[n]` markers — and carries its provenance (agent /
+ * source / similarity), so recalled knowledge is never confused with a fresh web
+ * source. Empty memory ⇒ the report is returned untouched.
+ */
+export function augmentReportWithMemory(report, memory) {
+    if (!Array.isArray(memory) || memory.length === 0)
+        return report;
+    const lines = [
+        '## Mémoire collective',
+        '',
+        '_Savoir déjà collecté par le collectif (graphe de connaissances), distinct des sources web fraîches ci-dessus._',
+        '',
+    ];
+    memory.forEach((m, i) => {
+        const who = m.agentId ? ` — par ${m.agentId}` : '';
+        const src = m.source ? `, source ${m.source}` : '';
+        const sim = typeof m.similarity === 'number' && Number.isFinite(m.similarity) ? ` (sim ${m.similarity.toFixed(2)})` : '';
+        lines.push(`- [M${i + 1}] ${collapseWhitespace(m.text)}${who}${src}${sim}`);
+    });
+    const block = lines.join('\n');
+    const idx = report.indexOf('## Références');
+    if (idx >= 0) {
+        const head = report.slice(0, idx).replace(/\s+$/, '');
+        const tail = report.slice(idx);
+        return `${head}\n\n${block}\n\n${tail}`;
+    }
+    return `${report.replace(/\s+$/, '')}\n\n${block}`;
+}
+/**
+ * Wrap a base Deep Research run (Phase A/B/C) with the CKG bridge. When Phase D
+ * is OFF (bridge absent or `enabled: false`) the base runner is delegated to
+ * VERBATIM — no recall, no ingest, the report/sources untouched (byte-identical).
+ * When ON: recall (start) → base run → ingest the deduped sources (end) → augment
+ * the report with the recalled memory, and attach a `ckg` outcome. Never throws:
+ * recall and ingest degrade silently, the base research is always returned.
+ */
+export async function runDeepResearchWithCkg(params) {
+    const { question, options, runBase, collectSourcesForIngest } = params;
+    // OFF / no bridge → byte-identical: base result verbatim, NO recall, NO ingest.
+    if (!options.enabled || !options.bridge) {
+        return runBase();
+    }
+    const bridge = options.bridge;
+    // RECALL (read) — at the START, bounded, never-throws.
+    const memory = await recallCollectiveMemory(bridge, question, options.recallLimit);
+    // Base A/B/C run (unchanged).
+    const result = await runBase();
+    // INGEST (write) — the deduped web sources, bounded to the run's source count.
+    let ingested = 0;
+    try {
+        const ingestable = collectSourcesForIngest(result);
+        ingested = await ingestCollectedSources(bridge, ingestable, {
+            question,
+            source: options.sourceTag ?? 'deep-research',
+            ...(options.agentId ? { agentId: options.agentId } : {}),
+        }, options.ingestExcerptChars);
+    }
+    catch (err) {
+        logger.debug(`[deep-research-ckg] ingest phase failed: ${errMsg(err)}`);
+        ingested = 0;
+    }
+    // Inject the recalled memory into the report, cited distinctly.
+    const report = augmentReportWithMemory(result.report, memory);
+    const outcome = { enabled: true, recalled: memory.length, ingested, memory };
+    return { ...result, report, ckg: outcome };
+}
+// ============================================================================
+// Small shared helpers
+// ============================================================================
+/**
+ * Resolve Phase-D activation from the CLI flag and/or the shared env gate.
+ * `--ckg` OR `CODEBUDDY_COLLECTIVE_MEMORY=true` enables the bridge — the SAME
+ * gate the rest of the app uses, so activation is consistent. Absent ⇒ off.
+ */
+export function resolveCkgEnabled(opts = {}, env = process.env) {
+    return Boolean(opts.ckg) || env.CODEBUDDY_COLLECTIVE_MEMORY === 'true';
+}
+/**
+ * Wrap a boundaries object so its `scrape` TEES the fetched content into `sink`
+ * (keyed by url), leaving every other boundary (llm/search/fingerprint/mapBatched
+ * and the STORM seams) untouched. This is how Phase D captures the per-source
+ * content the base pipeline drops from its `SourceRef[]` result — WITHOUT
+ * touching the A/B/C pipeline. Applied only on the CKG-enabled path. Never throws
+ * beyond what the underlying scrape throws (which the pipeline already guards).
+ */
+export function teeScrapeBoundary(boundaries, sink) {
+    const originalScrape = boundaries.scrape.bind(boundaries);
+    return {
+        ...boundaries,
+        scrape: async (url) => {
+            const content = await originalScrape(url);
+            if (typeof content === 'string' && content.trim().length > 0)
+                sink.set(url, content);
+            return content;
+        },
+    };
+}
+//# sourceMappingURL=deep-research-ckg.js.map

package/dist/agent/deep-research-storm.d.ts

@@ -0,0 +1,222 @@
+/**
+ * Deep Research — Phase C (STORM multi-perspective, Stanford-inspired).
+ *
+ * Phase A (`deep-research.ts runDeepResearchPipeline`) is ONE researcher on ONE
+ * angle. Phase B (`runDeepResearchLoop`) iterates that single angle to fill its
+ * own gaps. Both share a single point of view: the planner's decomposition of
+ * the question. STORM's insight is that a fact-heavy, encyclopedic article is
+ * better served by DIVERSE points of view, each researching the topic through
+ * its own lens, then co-writing an outline-first article with per-section
+ * citations — exactly what neither Phase A/B nor the council produces (the
+ * council's personas reason in pure LLM, they never touch the web).
+ *
+ * Phase C couples the two:
+ *
+ *   1. perspectives — instantiate N diversified perspectives (default 4) DERIVED
+ *                     from the council persona angles (`council/conductor.ts`
+ *                     ROLE_SETS: practitioner, skeptic, architect, reviewer,
+ *                     strategist, verifier) PLUS a STORM-signature historian /
+ *                     state-of-the-art angle. Injectable (`generatePerspectives`)
+ *                     for a topic-specific LLM derivation; deterministic default.
+ *   2. per-perspective research — each perspective frames the topic through its
+ *                     angle → runs the DETERMINISTIC Phase-A fan-out (reused:
+ *                     `planQueries` → `collectSources` → `dedupSources`) → its
+ *                     own cited sources. Perspectives run in PARALLEL (`mapBatched`).
+ *   3. cross-perspective merge — `mergeSources` (reused Phase A/B) into a SHARED
+ *                     citation registry: the same content found by two personas
+ *                     collapses to ONE source, ids stay contiguous.
+ *   4. outline-first co-writing — (a) build a structured table of contents from
+ *                     the accumulated perspectives + sources; (b) write EACH
+ *                     section grounded in its relevant cited sources → a
+ *                     structured article with inline [n] per section, a ToC, and
+ *                     a single renumbered "## Références" from the shared registry.
+ *
+ * STRICTLY ADDITIVE: nothing here runs on the Phase-A/B path. `runStormResearch`
+ * is a NEW entry point; `deep-research.ts` is untouched and only its exported
+ * building blocks are reused. Every stage is never-throws — a perspective that
+ * fails is dropped and the article is written from the others; an outline that
+ * fails degrades to the flat Phase-A `synthesize`. All side-effecting edges are
+ * INJECTABLE (`StormBoundaries`) so the whole pipeline is unit-testable offline.
+ *
+ * @module agent/deep-research-storm
+ */
+import { type DeepResearchBoundaries, type DeepResearchOptions, type DeepResearchResult, type CollectedSource, type SourceRef, type DeepLlmMessage } from './deep-research.js';
+export interface StormResearchOptions extends DeepResearchOptions {
+    /** Number of diversified perspectives (default 4, clamped [2, 6]). */
+    perspectives?: number;
+    /** Absolute cap on sources accumulated across ALL perspectives (default 24). */
+    maxTotalSources?: number;
+}
+/** Default perspective count when `--perspectives`/`--storm` asks for STORM. */
+export declare const DEFAULT_STORM_PERSPECTIVES = 4;
+export declare function resolveStormPerspectiveCount(n: number | undefined): number;
+/** One diversified point of view driving both research and section writing. */
+export interface StormPerspective {
+    id: string;
+    label: string;
+    /** The mission/angle that orients this perspective's queries + prose. */
+    angle: string;
+    /** What this perspective emphasises. */
+    focus: string[];
+}
+export interface ResearchOutlineSection {
+    title: string;
+    subsections?: string[];
+}
+export interface ResearchOutline {
+    title: string;
+    sections: ResearchOutlineSection[];
+}
+/** Per-perspective accounting for the result. */
+export interface StormPerspectiveResult {
+    perspective: StormPerspective;
+    /** Sources this perspective surfaced (pre cross-perspective merge). */
+    sourceCount: number;
+    /** Sub-questions this perspective's planner produced. */
+    subQuestions: number;
+    /** True when the perspective's research threw and it was dropped. */
+    failed: boolean;
+    /** True when the perspective's planner used the LLM (false ⇒ deterministic). */
+    plannerLlmUsed: boolean;
+}
+/** The Phase-C result — a superset of {@link DeepResearchResult}. */
+export interface StormResearchResult extends DeepResearchResult {
+    /** The diversified perspectives and what each contributed. */
+    perspectives: StormPerspectiveResult[];
+    /** The structured table of contents the article was written against. */
+    outline: ResearchOutline;
+    /** True when the outline came from the LLM (false ⇒ deterministic fallback). */
+    outlineLlmUsed: boolean;
+    /** True when the article was written outline-first (false ⇒ flat Phase-A fallback). */
+    coWritten: boolean;
+}
+export type StormStage = {
+    stage: 'perspectives';
+} | {
+    stage: 'perspectives-ready';
+    count: number;
+} | {
+    stage: 'perspective-planning';
+    perspective: string;
+} | {
+    stage: 'perspective-done';
+    perspective: string;
+    sources: number;
+    failed: boolean;
+} | {
+    stage: 'merged-perspectives';
+    total: number;
+    dropped: number;
+} | {
+    stage: 'outlining';
+} | {
+    stage: 'outlined';
+    sections: number;
+    llmUsed: boolean;
+} | {
+    stage: 'writing';
+} | {
+    stage: 'written';
+    sections: number;
+    coWritten: boolean;
+} | {
+    stage: 'storm-done';
+    sources: number;
+};
+/** Progress channel for the STORM path (distinct from the Phase-A/B `deep` channel). */
+export type StormProgress = {
+    type: 'storm';
+} & StormStage;
+export interface OutlineInput {
+    topic: string;
+    perspectives: StormPerspective[];
+    sources: SourceRef[];
+}
+export interface SectionWriteInput {
+    topic: string;
+    section: ResearchOutlineSection;
+    /** The sources most relevant to this section (a subset of the registry). */
+    relevant: CollectedSource[];
+    /** The FULL registry (a section may cite any id). */
+    registry: SourceRef[];
+}
+export interface StormBoundaries extends DeepResearchBoundaries {
+    /**
+     * Instantiate N topic-specific perspectives. Optional — when absent, the
+     * default derives them deterministically from the council persona angles.
+     * MAY throw (caller falls back to the deterministic seeds).
+     */
+    generatePerspectives?(topic: string, n: number): Promise<StormPerspective[]>;
+    /**
+     * Build the article outline (ToC). Optional — when absent, an LLM call through
+     * `llm` drives it with a deterministic fallback. MAY throw (caller degrades to
+     * the flat Phase-A synthesis).
+     */
+    buildOutline?(input: OutlineInput): Promise<ResearchOutline>;
+    /**
+     * Write one section grounded in its relevant cited sources. Optional — when
+     * absent, an LLM call through `llm` drives it with a deterministic cited
+     * fallback. MAY throw (caller degrades that section to the deterministic body).
+     */
+    writeSection?(input: SectionWriteInput): Promise<string>;
+}
+/**
+ * Deterministic, ORDERED perspective seeds derived from the council persona
+ * angles (`council/conductor.ts`) plus the historian angle. The order surfaces
+ * the most diverse lenses first so a small `n` still spans practitioner /
+ * skeptic / historian / architect. Built defensively — a missing council role
+ * is skipped, and a hardcoded floor guarantees ≥2 distinct perspectives.
+ */
+export declare function defaultStormPerspectives(n: number): StormPerspective[];
+/**
+ * Instantiate the perspectives. If a `generatePerspectives` boundary is injected
+ * it wins (falling back to the deterministic seeds on throw/empty). Otherwise an
+ * LLM derivation is attempted (topic-specific), again falling back to the seeds.
+ * Never throws; always returns between {@link STORM_MIN_PERSPECTIVES} and `n`.
+ */
+export declare function deriveStormPerspectives(topic: string, n: number, boundaries: StormBoundaries): Promise<StormPerspective[]>;
+/** Frame the topic through a perspective's angle so the planner biases its queries. */
+export declare function framePerspectiveQuestion(topic: string, p: StormPerspective): string;
+/**
+ * Build the outline. If a `buildOutline` boundary is injected it is used AS-IS
+ * (it MAY throw — the caller degrades to the flat synthesis). Otherwise an LLM
+ * call is attempted with a deterministic fallback derived from the perspectives.
+ * The default path never throws.
+ */
+export declare function buildResearchOutline(topic: string, perspectives: StormPerspective[], sources: CollectedSource[], boundaries: StormBoundaries): Promise<{
+    outline: ResearchOutline;
+    llmUsed: boolean;
+}>;
+/** Deterministic outline: background + one section per perspective + synthesis. */
+export declare function fallbackOutline(topic: string, perspectives: StormPerspective[]): ResearchOutline;
+/**
+ * Rank the registry by keyword overlap with the section heading/subsections and
+ * return the top-K sources most relevant to grounding this section. Falls back
+ * to the whole (bounded) registry when nothing overlaps, so a section is never
+ * left without citable material.
+ */
+export declare function selectRelevantSources(section: ResearchOutlineSection, sources: CollectedSource[], limit?: number): CollectedSource[];
+/**
+ * Write one section. If a `writeSection` boundary is injected it is used
+ * (falling back to the deterministic body on throw/empty). Otherwise an LLM call
+ * is attempted with a deterministic cited fallback. Never throws.
+ */
+export declare function writeSectionBody(topic: string, section: ResearchOutlineSection, relevant: CollectedSource[], registry: SourceRef[], boundaries: StormBoundaries): Promise<{
+    body: string;
+    llmUsed: boolean;
+}>;
+/** Deterministic table of contents rendered from the outline. */
+export declare function renderTableOfContents(outline: ResearchOutline): string;
+/**
+ * Run the full Phase-C STORM pipeline. Never throws: perspectives that fail are
+ * dropped, an outline that fails degrades to the flat Phase-A synthesis, and a
+ * total failure yields an honest non-conclusive cited report. Emits coarse
+ * progress via the optional callback (its own `storm` channel).
+ */
+export declare function runStormResearch(topic: string, boundaries: StormBoundaries, options?: StormResearchOptions, onProgress?: (s: StormStage) => void): Promise<StormResearchResult>;
+/**
+ * Drop a leading ATX heading that repeats the section title. The assembler
+ * already emits `## ${title}`; live GK33 STORM left `# Title` then `## Title`.
+ */
+export declare function stripLeadingSectionHeading(body: string, title: string): string;
+export type { DeepLlmMessage };