clitrigger 0.1.13 → 0.1.15

package/dist/server/services/memory-ingest.js

@@ -2,6 +2,9 @@ import { spawn } from 'child_process';
 import fs from 'fs';
 import path from 'path';
 import * as queries from '../db/queries.js';
+import { broadcaster } from '../websocket/broadcaster.js';
+import { debugLogger } from './debug-logger.js';
+import { dispatchWikiExport } from './wiki-exporter.js';
 const RAW_DIR_NAME = '.clitrigger';
 const RAW_SUBDIR = 'raw';
 const VALID_SOURCE_TYPES = new Set(['todo', 'discussion', 'manual']);
@@ -46,7 +49,9 @@ function writeRawSnapshot(project, sourceType, sourceId, fullText, titleHint) {
     try {
         const baseDir = path.join(project.path, RAW_DIR_NAME, RAW_SUBDIR, sourceType);
         fs.mkdirSync(baseDir, { recursive: true });
-        ensureGitignore(project.path, `${RAW_DIR_NAME}/`);
+        // Only ignore the raw snapshot subdirectory — leave wiki/ trackable in git
+        // so users can opt into committing the auto-exported markdown wiki.
+        ensureGitignore(project.path, `${RAW_DIR_NAME}/${RAW_SUBDIR}/`);
         const ts = timestampStr();
         const idPart = sourceId ? `-${sourceId.slice(0, 8)}` : '';
         const slug = slugify(titleHint || sourceType);
@@ -67,6 +72,7 @@ function writeRawSnapshot(project, sourceType, sourceId, fullText, titleHint) {
         return null;
     }
 }
+const WIKI_INDEX_TAG = '__wiki_index__';
 const DEFAULT_WIKI_SCHEMA = `# Wiki Schema
 
 ## Entity Types
@@ -99,21 +105,22 @@ function safeParseIngestOp(raw) {
     catch {
         const m = cleaned.match(/\{[\s\S]*\}/);
         if (!m)
-            return empty;
+            return { op: empty, parseFailed: true };
         try {
             parsed = JSON.parse(m[0]);
         }
         catch {
-            return empty;
+            return { op: empty, parseFailed: true };
         }
     }
-    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed))
-        return empty;
+    if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+        return { op: empty, parseFailed: true };
+    }
     const p = parsed;
     const create = Array.isArray(p.create) ? p.create : [];
     const update = Array.isArray(p.update) ? p.update : [];
     const edges = Array.isArray(p.edges) ? p.edges : [];
-    return { create, update, edges };
+    return { op: { create, update, edges }, parseFailed: false };
 }
 function safeParseLintIssues(raw) {
     const cleaned = stripCodeFences(raw);
@@ -154,7 +161,35 @@ function buildInvocation(cliTool) {
         default: return { command: 'claude', args: ['--print'] };
     }
 }
-function runHeadless(cliTool, prompt, timeoutMs = 180_000) {
+/**
+ * Open a debug session for a memory-ingest or lint run when the project has debug_logging enabled.
+ * Reuses the existing `.debug-logs/` directory and rotation policy.
+ * The synthetic todoId encodes intent (`mem-{kind}-{sourceType}-{sourceId|ts}`) so logs sort/filter
+ * cleanly alongside real todo logs.
+ */
+function startDebugSession(project, cliTool, sourceType, sourceId, kind) {
+    if (!project.debug_logging || !project.path)
+        return undefined;
+    const { command, args } = buildInvocation(cliTool);
+    const idPart = sourceId ? sourceId.slice(0, 8) : Date.now().toString(36);
+    const stypePart = sourceType ?? 'manual';
+    const todoId = `mem-${kind}-${stypePart}-${idPart}`;
+    try {
+        return debugLogger.startSession({
+            todoId,
+            projectPath: project.path,
+            cliTool,
+            command,
+            args,
+            workDir: process.env.HOME || process.env.USERPROFILE || '.',
+        });
+    }
+    catch (err) {
+        console.warn('[memory-ingest] failed to open debug session:', err);
+        return undefined;
+    }
+}
+export function runHeadless(cliTool, prompt, timeoutMs = 180_000, debugSession) {
     return new Promise((resolve, reject) => {
         const { command, args } = buildInvocation(cliTool);
         const isWin = process.platform === 'win32';
@@ -168,6 +203,11 @@ function runHeadless(cliTool, prompt, timeoutMs = 180_000) {
             env: { ...process.env },
             cwd: process.env.HOME || process.env.USERPROFILE || '.',
         });
+        if (debugSession) {
+            // tee returns a passthrough we don't need; the side-effect is appending to the debug log file.
+            debugSession.teeStdout(proc.stdout);
+            debugSession.teeStderr(proc.stderr);
+        }
         const timer = setTimeout(() => {
             if (settled)
                 return;
@@ -176,6 +216,10 @@ function runHeadless(cliTool, prompt, timeoutMs = 180_000) {
                 proc.kill();
             }
             catch { /* ignore */ }
+            try {
+                debugSession?.finalize(-1);
+            }
+            catch { /* ignore */ }
             reject(new Error('Memory ingest timed out'));
         }, timeoutMs);
         proc.stdout.on('data', (c) => { stdout += c.toString('utf8'); });
@@ -185,6 +229,10 @@ function runHeadless(cliTool, prompt, timeoutMs = 180_000) {
                 return;
             settled = true;
             clearTimeout(timer);
+            try {
+                debugSession?.finalize(-1);
+            }
+            catch { /* ignore */ }
             reject(err);
         });
         proc.on('close', (code) => {
@@ -192,6 +240,10 @@ function runHeadless(cliTool, prompt, timeoutMs = 180_000) {
                 return;
             settled = true;
             clearTimeout(timer);
+            try {
+                debugSession?.finalize(code ?? 0);
+            }
+            catch { /* ignore */ }
             if (code === 0)
                 resolve(stdout);
             else
@@ -200,12 +252,20 @@ function runHeadless(cliTool, prompt, timeoutMs = 180_000) {
         try {
             proc.stdin.write(prompt + '\n');
             proc.stdin.end();
+            try {
+                debugSession?.writeStdin(prompt);
+            }
+            catch { /* ignore */ }
         }
         catch (err) {
             if (settled)
                 return;
             settled = true;
             clearTimeout(timer);
+            try {
+                debugSession?.finalize(-1);
+            }
+            catch { /* ignore */ }
             try {
                 proc.kill();
             }
@@ -214,7 +274,7 @@ function runHeadless(cliTool, prompt, timeoutMs = 180_000) {
         }
     });
 }
-function resolveCliTool(value) {
+export function resolveCliTool(value) {
     if (value === 'claude' || value === 'gemini' || value === 'codex')
         return value;
     return 'claude';
@@ -235,11 +295,28 @@ function getOrCreateSchemaNode(projectId) {
     queries.createMemoryNode(projectId, 'Wiki Schema', DEFAULT_WIKI_SCHEMA, JSON.stringify([WIKI_SCHEMA_TAG]), 1);
     return DEFAULT_WIKI_SCHEMA;
 }
+const NODE_SUMMARY_FULL_CHAR_BUDGET = 6000;
+const NODE_SUMMARY_TITLES_FALLBACK_BUDGET = 1500;
+const NODE_SUMMARY_PINNED_BODY_PREVIEW = 300;
+function formatFullNodeLine(n) {
+    try {
+        const tags = JSON.parse(n.tags ?? '[]');
+        const tagStr = tags.filter(t => t !== WIKI_SCHEMA_TAG && t !== WIKI_INDEX_TAG).join(', ');
+        const pinnedStr = n.pinned ? ' [pinned]' : '';
+        const bodyPreview = n.pinned ? `\n  ${(n.body || '').slice(0, NODE_SUMMARY_PINNED_BODY_PREVIEW)}` : '';
+        return `- id="${n.id}" title="${n.title}"${tagStr ? ` tags=[${tagStr}]` : ''}${pinnedStr}${bodyPreview}`;
+    }
+    catch {
+        return `- id="${n.id}" title="${n.title}"`;
+    }
+}
 function buildNodeSummary(nodes) {
     const visible = nodes.filter(n => {
         try {
             const tags = JSON.parse(n.tags ?? '[]');
-            return !Array.isArray(tags) || !tags.includes(WIKI_SCHEMA_TAG);
+            if (!Array.isArray(tags))
+                return true;
+            return !tags.includes(WIKI_SCHEMA_TAG) && !tags.includes(WIKI_INDEX_TAG);
         }
         catch {
             return true;
@@ -247,20 +324,52 @@ function buildNodeSummary(nodes) {
     });
     if (visible.length === 0)
         return '(no existing pages)';
-    return visible.map(n => {
-        try {
-            const tags = JSON.parse(n.tags ?? '[]');
-            const tagStr = tags.filter(t => t !== WIKI_SCHEMA_TAG).join(', ');
-            const pinned = n.pinned ? ' [pinned]' : '';
-            const bodyPreview = n.pinned ? `\n  ${(n.body || '').slice(0, 300)}` : '';
-            return `- id="${n.id}" title="${n.title}"${tagStr ? ` tags=[${tagStr}]` : ''}${pinned}${bodyPreview}`;
+    // Pinned first (user-curated importance), then most-recently-updated.
+    const sorted = [...visible].sort((a, b) => {
+        const pinDiff = (b.pinned ? 1 : 0) - (a.pinned ? 1 : 0);
+        if (pinDiff !== 0)
+            return pinDiff;
+        return (b.updated_at ?? '').localeCompare(a.updated_at ?? '');
+    });
+    const lines = [];
+    let used = 0;
+    let inlined = 0;
+    for (const n of sorted) {
+        const line = formatFullNodeLine(n);
+        // Always include pinned nodes regardless of budget; gate non-pinned by char budget.
+        if (!n.pinned && used + line.length > NODE_SUMMARY_FULL_CHAR_BUDGET)
+            break;
+        lines.push(line);
+        used += line.length + 1;
+        inlined++;
+    }
+    const remaining = sorted.slice(inlined);
+    if (remaining.length > 0) {
+        // Titles-only tail so the LLM can still match by exact title (avoid duplicate creates),
+        // even though older nodes' IDs aren't shown so they cannot be updated in this batch.
+        const titles = [];
+        let titleBudget = NODE_SUMMARY_TITLES_FALLBACK_BUDGET;
+        for (const n of remaining) {
+            const piece = `"${n.title}"`;
+            if (titleBudget - piece.length - 2 < 0)
+                break;
+            titles.push(piece);
+            titleBudget -= piece.length + 2;
         }
-        catch {
-            return `- id="${n.id}" title="${n.title}"`;
+        const hiddenCount = remaining.length - titles.length;
+        if (titles.length > 0) {
+            const tail = hiddenCount > 0 ? ` … and ${hiddenCount} more` : '';
+            lines.push('');
+            lines.push(`(other existing titles — match exactly to avoid duplicates; IDs not shown so these are not updatable in this batch: ${titles.join(', ')}${tail})`);
         }
-    }).join('\n');
+        else {
+            lines.push('');
+            lines.push(`(${remaining.length} older entries omitted to fit context)`);
+        }
+    }
+    return lines.join('\n');
 }
-const INGEST_PROMPT_HEADER = `You are maintaining a project knowledge wiki using the LLM Wiki pattern.
+const INGEST_PROMPT_HEADER = `You are maintaining a project knowledge wiki using the LLM Wiki pattern.{CHUNK_PREAMBLE}
 
 ## Wiki Schema
 {SCHEMA}
@@ -268,7 +377,7 @@ const INGEST_PROMPT_HEADER = `You are maintaining a project knowledge wiki using
 ## Existing Wiki Pages
 {NODES}
 
-## New Source Material
+## New Source Material{CHUNK_NOTE}
 {SOURCE}
 
 ---
@@ -314,53 +423,90 @@ Issue types:
 Rules:
 - Maximum 10 issues. Only flag real problems.
 - Return [] if the wiki looks healthy.`;
-export async function ingestSource(projectId, sourceText, sourceType, sourceId, titleHint) {
-    const project = queries.getProjectById(projectId);
-    if (!project)
-        throw new Error('Project not found');
-    const cliTool = resolveCliTool(project.cli_tool);
-    // Step 1: persist raw snapshot (immutable). Failure is non-fatal.
-    let rawPath = null;
-    if (sourceType && VALID_SOURCE_TYPES.has(sourceType)) {
-        const hint = (titleHint && titleHint.trim()) || sourceText.split('\n').find(l => l.trim())?.trim().slice(0, 60) || sourceType;
-        rawPath = writeRawSnapshot(project, sourceType, sourceId, sourceText, hint);
+const CHUNK_CHARS = 7000;
+const CHUNK_THRESHOLD = 8000;
+const MAX_CHUNKS = 4;
+const VALID_RELATIONS = new Set(['related', 'precedes', 'example_of', 'counter_example', 'refines']);
+/**
+ * Split a long source into chunks at paragraph boundaries. Hard-splits any single paragraph
+ * that exceeds maxChars. Caps total chunks at maxChunks (later content is dropped).
+ * Returns a single-element array when text fits without splitting.
+ */
+function chunkSourceText(text, maxChars, maxChunks) {
+    if (text.length <= CHUNK_THRESHOLD)
+        return [text];
+    const paragraphs = text.split(/\n\s*\n/);
+    const chunks = [];
+    let cur = '';
+    for (const para of paragraphs) {
+        const p = para.trim();
+        if (!p)
+            continue;
+        if (chunks.length >= maxChunks)
+            break;
+        if (cur.length + p.length + 2 <= maxChars) {
+            cur = cur ? `${cur}\n\n${p}` : p;
+            continue;
+        }
+        if (cur) {
+            chunks.push(cur);
+            cur = '';
+            if (chunks.length >= maxChunks)
+                break;
+        }
+        if (p.length > maxChars) {
+            for (let i = 0; i < p.length && chunks.length < maxChunks; i += maxChars) {
+                chunks.push(p.slice(i, i + maxChars));
+            }
+        }
+        else {
+            cur = p;
+        }
     }
-    const schema = getOrCreateSchemaNode(projectId);
-    const nodes = queries.getMemoryNodesByProjectId(projectId);
-    const nodeSummary = buildNodeSummary(nodes);
-    const prompt = INGEST_PROMPT_HEADER
-        .replace('{SCHEMA}', schema)
-        .replace('{NODES}', nodeSummary)
-        .replace('{SOURCE}', sourceText.slice(0, 8000));
-    const raw = await runHeadless(cliTool, prompt);
-    const op = safeParseIngestOp(raw);
-    const titleToId = new Map(nodes.map(n => [n.title.toLowerCase(), n.id]));
-    const createdIds = [];
-    let edgesAdded = 0;
-    const VALID_RELATIONS = new Set(['related', 'precedes', 'example_of', 'counter_example', 'refines']);
+    if (cur && chunks.length < maxChunks)
+        chunks.push(cur);
+    return chunks.slice(0, maxChunks);
+}
+function applyIngestOp(ctx, op, existingNodes) {
+    ctx.skipped.proposedCreate += op.create.length;
+    ctx.skipped.proposedUpdate += op.update.length;
+    ctx.skipped.proposedEdges += op.edges.length;
     for (const c of op.create.slice(0, 10)) {
-        if (!c.title?.trim())
+        if (!c.title?.trim()) {
+            ctx.skipped.emptyTitle++;
             continue;
+        }
         const title = String(c.title).trim().slice(0, 120);
-        if (titleToId.has(title.toLowerCase()))
+        if (ctx.titleToId.has(title.toLowerCase())) {
+            ctx.skipped.duplicateTitle++;
             continue;
+        }
         try {
             const tags = Array.isArray(c.tags) ? JSON.stringify(c.tags.map(String).filter(Boolean)) : null;
-            const node = queries.createMemoryNode(projectId, title, typeof c.body === 'string' ? c.body : '', tags, 0, sourceType, sourceId, rawPath);
-            titleToId.set(title.toLowerCase(), node.id);
-            createdIds.push(node.id);
+            const node = queries.createMemoryNode(ctx.projectId, title, typeof c.body === 'string' ? c.body : '', tags, 0, ctx.sourceType, ctx.sourceId, ctx.rawPath);
+            ctx.titleToId.set(title.toLowerCase(), node.id);
+            ctx.createdIds.push(node.id);
         }
-        catch {
-            /* skip on UNIQUE conflict */
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (msg.includes('UNIQUE'))
+                ctx.skipped.uniqueConflict++;
+            else {
+                ctx.skipped.uniqueConflict++;
+                console.warn('[memory-ingest] createMemoryNode failed:', msg);
+            }
         }
     }
-    let updatedCount = 0;
     for (const u of op.update.slice(0, 10)) {
-        if (!u.id)
+        if (!u.id) {
+            ctx.skipped.invalidUpdateId++;
             continue;
-        const existing = nodes.find(n => n.id === u.id);
-        if (!existing)
+        }
+        const existing = existingNodes.find(n => n.id === u.id);
+        if (!existing) {
+            ctx.skipped.invalidUpdateId++;
             continue;
+        }
         const upd = {};
         if (typeof u.body === 'string')
             upd.body = u.body;
@@ -368,30 +514,189 @@ export async function ingestSource(projectId, sourceText, sourceType, sourceId,
             upd.tags = JSON.stringify(u.tags.map(String).filter(Boolean));
         if (Object.keys(upd).length > 0) {
             queries.updateMemoryNode(u.id, upd);
-            updatedCount++;
+            ctx.updatedIds.add(u.id);
         }
     }
     for (const e of op.edges.slice(0, 20)) {
-        const fromId = titleToId.get(String(e.from_title || '').toLowerCase());
-        const toId = titleToId.get(String(e.to_title || '').toLowerCase());
-        if (!fromId || !toId || fromId === toId)
+        const fromId = ctx.titleToId.get(String(e.from_title || '').toLowerCase());
+        const toId = ctx.titleToId.get(String(e.to_title || '').toLowerCase());
+        if (!fromId || !toId) {
+            ctx.skipped.invalidEdgeRef++;
             continue;
+        }
+        if (fromId === toId) {
+            ctx.skipped.selfEdge++;
+            continue;
+        }
         const rt = VALID_RELATIONS.has(e.relation_type ?? '') ? e.relation_type : 'related';
         try {
-            queries.createMemoryEdge(projectId, fromId, toId, rt, e.label ?? null);
-            edgesAdded++;
+            queries.createMemoryEdge(ctx.projectId, fromId, toId, rt, e.label ?? null);
+            ctx.edgesAdded++;
         }
-        catch {
-            /* skip duplicates */
+        catch (err) {
+            const msg = err instanceof Error ? err.message : String(err);
+            if (msg.includes('UNIQUE'))
+                ctx.skipped.edgeUniqueConflict++;
+            else {
+                ctx.skipped.edgeUniqueConflict++;
+                console.warn('[memory-ingest] createMemoryEdge failed:', msg);
+            }
+        }
+    }
+}
+export async function ingestSource(projectId, sourceText, sourceType, sourceId, titleHint, locale) {
+    const project = queries.getProjectById(projectId);
+    if (!project)
+        throw new Error('Project not found');
+    const cliTool = resolveCliTool(project.cli_tool);
+    // Step 1: persist raw snapshot (immutable). Failure is non-fatal.
+    let rawPath = null;
+    if (sourceType && VALID_SOURCE_TYPES.has(sourceType)) {
+        const hint = (titleHint && titleHint.trim()) || sourceText.split('\n').find(l => l.trim())?.trim().slice(0, 60) || sourceType;
+        rawPath = writeRawSnapshot(project, sourceType, sourceId, sourceText, hint);
+    }
+    const schema = getOrCreateSchemaNode(projectId);
+    const langRule = locale === 'en'
+        ? '- Write all titles, body text, tags, and edge labels in English.'
+        : '- Write all titles, body text, tags, and edge labels in Korean (한국어).';
+    const chunks = chunkSourceText(sourceText, CHUNK_CHARS, MAX_CHUNKS);
+    const total = chunks.length;
+    const ctx = {
+        projectId,
+        sourceType,
+        sourceId,
+        rawPath,
+        titleToId: new Map(),
+        createdIds: [],
+        updatedIds: new Set(),
+        edgesAdded: 0,
+        skipped: {
+            parseFailed: false,
+            proposedCreate: 0, proposedUpdate: 0, proposedEdges: 0,
+            duplicateTitle: 0, uniqueConflict: 0, emptyTitle: 0,
+            invalidUpdateId: 0, invalidEdgeRef: 0, selfEdge: 0, edgeUniqueConflict: 0,
+        },
+        lastRaw: '',
+    };
+    for (let i = 0; i < total; i++) {
+        // Re-fetch nodes between chunks so dedup sees nodes added by earlier chunks.
+        const nodes = queries.getMemoryNodesByProjectId(projectId);
+        ctx.titleToId = new Map(nodes.map(n => [n.title.toLowerCase(), n.id]));
+        const nodeSummary = buildNodeSummary(nodes);
+        const chunkPreamble = total > 1
+            ? `\n\nThis source has been split into ${total} parts due to length. You are processing part ${i + 1}. Earlier parts may have added new pages — see "Existing Wiki Pages" for the current state. Avoid creating duplicates of pages already added in earlier parts.`
+            : '';
+        const chunkNote = total > 1 ? ` (part ${i + 1} of ${total})` : '';
+        const prompt = INGEST_PROMPT_HEADER
+            .replace('Rules:\n', `Rules:\n${langRule}\n`)
+            .replace('{CHUNK_PREAMBLE}', chunkPreamble)
+            .replace('{CHUNK_NOTE}', chunkNote)
+            .replace('{SCHEMA}', schema)
+            .replace('{NODES}', nodeSummary)
+            .replace('{SOURCE}', chunks[i]);
+        const debugSession = startDebugSession(project, cliTool, sourceType, sourceId, total > 1 ? `ingest-${i + 1}of${total}` : 'ingest');
+        const raw = await runHeadless(cliTool, prompt, 180_000, debugSession);
+        ctx.lastRaw = raw;
+        const { op, parseFailed } = safeParseIngestOp(raw);
+        if (parseFailed)
+            ctx.skipped.parseFailed = true;
+        applyIngestOp(ctx, op, nodes);
+    }
+    const created = ctx.createdIds.length;
+    const updated = ctx.updatedIds.size;
+    const edgesAdded = ctx.edgesAdded;
+    if (ctx.skipped.parseFailed || (created === 0 && updated === 0 && edgesAdded === 0)) {
+        console.warn(`[memory-ingest] no-op result project=${projectId} cli=${cliTool} chunks=${total} ` +
+            `parseFailed=${ctx.skipped.parseFailed} ` +
+            `proposed(c/u/e)=${ctx.skipped.proposedCreate}/${ctx.skipped.proposedUpdate}/${ctx.skipped.proposedEdges} ` +
+            `skip=dup:${ctx.skipped.duplicateTitle}/uniq:${ctx.skipped.uniqueConflict}/badId:${ctx.skipped.invalidUpdateId}/` +
+            `badEdge:${ctx.skipped.invalidEdgeRef}/empty:${ctx.skipped.emptyTitle}`);
+        if (ctx.skipped.parseFailed) {
+            console.warn('[memory-ingest] last raw response head:', ctx.lastRaw.slice(0, 500));
         }
     }
+    // Mirror DB → `.clitrigger/wiki/` markdown files (best-effort, fire-and-forget).
+    dispatchWikiExport(projectId);
+    // Project-scoped activity log entry — feeds the Wiki Activity tab.
+    try {
+        const applied = created + updated + edgesAdded;
+        let severity = 'info';
+        let message;
+        if (ctx.skipped.parseFailed) {
+            severity = 'error';
+            message = `Ingest failed to parse model output (${total} chunk${total > 1 ? 's' : ''})`;
+        }
+        else if (applied === 0) {
+            severity = 'warning';
+            message = `Ingest produced no changes — nothing new in source`;
+        }
+        else {
+            message = `Ingested ${created} new, ${updated} updated, ${edgesAdded} edge${edgesAdded === 1 ? '' : 's'}`;
+        }
+        queries.createMemoryLog(projectId, 'ingest', message, {
+            severity,
+            sourceType: sourceType ?? 'manual',
+            sourceId,
+            sourceTitle: titleHint ?? null,
+            metadata: {
+                cliTool,
+                chunks: total,
+                created,
+                updated,
+                edgesAdded,
+                skipped: ctx.skipped,
+            },
+        });
+    }
+    catch (err) {
+        console.warn('[memory-ingest] failed to write memory_logs entry:', err);
+    }
     return {
-        created: createdIds.length,
-        updated: updatedCount,
+        created,
+        updated,
         edgesAdded,
-        nodeIds: createdIds,
+        nodeIds: ctx.createdIds,
+        skipped: ctx.skipped,
+        rawResponseSnippet: ctx.skipped.parseFailed ? ctx.lastRaw.slice(0, 500) : undefined,
     };
 }
+const LINT_CHUNK_CHARS = 10000;
+const LINT_MAX_CHUNKS = 5;
+function chunkLintEntries(entries, maxChars, maxChunks) {
+    const chunks = [];
+    let cur = '';
+    for (const entry of entries) {
+        if (chunks.length >= maxChunks)
+            break;
+        const sep = cur ? '\n\n' : '';
+        if (cur && cur.length + sep.length + entry.length > maxChars) {
+            chunks.push(cur);
+            if (chunks.length >= maxChunks) {
+                cur = '';
+                break;
+            }
+            cur = entry.length > maxChars ? entry.slice(0, maxChars) : entry;
+        }
+        else {
+            cur = `${cur}${sep}${entry.length > maxChars ? entry.slice(0, maxChars) : entry}`;
+        }
+    }
+    if (cur && chunks.length < maxChunks)
+        chunks.push(cur);
+    return chunks;
+}
+function dedupeLintIssues(issues) {
+    const seen = new Set();
+    const out = [];
+    for (const issue of issues) {
+        const key = `${issue.type}::${[...issue.node_titles].map(s => s.toLowerCase()).sort().join('|')}::${issue.message.toLowerCase()}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(issue);
+    }
+    return out;
+}
 export async function lintWiki(projectId) {
     const project = queries.getProjectById(projectId);
     if (!project)
@@ -401,7 +706,9 @@ export async function lintWiki(projectId) {
     const visible = nodes.filter(n => {
         try {
             const tags = JSON.parse(n.tags ?? '[]');
-            return !Array.isArray(tags) || !tags.includes(WIKI_SCHEMA_TAG);
+            if (!Array.isArray(tags))
+                return true;
+            return !tags.includes(WIKI_SCHEMA_TAG) && !tags.includes(WIKI_INDEX_TAG);
         }
         catch {
             return true;
@@ -411,14 +718,88 @@ export async function lintWiki(projectId) {
         return [];
     const edges = queries.getMemoryEdgesByProjectId(projectId);
     const edgeSet = new Set(edges.flatMap(e => [e.from_node_id, e.to_node_id]));
-    const nodeText = visible.map(n => {
+    const entries = visible.map(n => {
         const body = (n.body || '').slice(0, 400);
         const hasEdge = edgeSet.has(n.id) ? '' : ' [no-edges]';
         return `### ${n.title}${hasEdge}\n${body}`;
-    }).join('\n\n');
-    const prompt = LINT_PROMPT_HEADER.replace('{NODES}', nodeText.slice(0, 12000));
-    const raw = await runHeadless(cliTool, prompt);
-    return safeParseLintIssues(raw);
+    });
+    // Larger wikis used to be silently truncated to 12KB — the tail nodes never
+    // got linted. Split into chunks so every node is seen by at least one pass.
+    // Cross-chunk duplicates aren't detected (each chunk only sees its own
+    // entries), but orphan/stale/contradiction within a chunk still work.
+    const chunks = chunkLintEntries(entries, LINT_CHUNK_CHARS, LINT_MAX_CHUNKS);
+    const truncated = entries.length > 0 && chunks.join('\n\n').length < entries.join('\n\n').length;
+    const collected = [];
+    for (let i = 0; i < chunks.length; i++) {
+        const prompt = LINT_PROMPT_HEADER.replace('{NODES}', chunks[i]);
+        const kind = chunks.length > 1 ? `lint-${i + 1}of${chunks.length}` : 'lint';
+        const debugSession = startDebugSession(project, cliTool, null, null, kind);
+        const raw = await runHeadless(cliTool, prompt, 180_000, debugSession);
+        collected.push(...safeParseLintIssues(raw));
+    }
+    const issues = dedupeLintIssues(collected);
+    try {
+        const counts = {};
+        for (const issue of issues) {
+            counts[issue.type] = (counts[issue.type] ?? 0) + 1;
+        }
+        const summary = issues.length === 0
+            ? `Wiki looks healthy — no issues found${chunks.length > 1 ? ` (${chunks.length} chunks)` : ''}`
+            : `Lint found ${issues.length} issue${issues.length === 1 ? '' : 's'}: ${Object.entries(counts).map(([k, v]) => `${k}=${v}`).join(', ')}${chunks.length > 1 ? ` (across ${chunks.length} chunks)` : ''}`;
+        queries.createMemoryLog(projectId, 'lint', summary, {
+            severity: issues.length === 0 ? 'info' : 'warning',
+            metadata: {
+                cliTool,
+                total: issues.length,
+                counts,
+                nodeCount: visible.length,
+                chunks: chunks.length,
+                truncated,
+            },
+        });
+    }
+    catch (err) {
+        console.warn('[memory-lint] failed to write memory_logs entry:', err);
+    }
+    return issues;
+}
+/**
+ * Run an auto-ingest and broadcast the result over WebSocket so the client can show a toast.
+ * Errors are swallowed (auto-ingest is best-effort) but reported as a failure event.
+ */
+export function runAutoIngestAndBroadcast(projectId, sourceType, sourceId, sourceTitle, sourceText) {
+    ingestSource(projectId, sourceText, sourceType, sourceId, sourceTitle).then((res) => {
+        broadcaster.broadcast({
+            type: 'memory:ingest-finished',
+            projectId,
+            sourceType,
+            sourceId,
+            sourceTitle,
+            created: res.created,
+            updated: res.updated,
+            edgesAdded: res.edgesAdded,
+            skipped: res.skipped,
+        });
+    }).catch((err) => {
+        console.error(`[memory-ingest] auto-ingest failed (${sourceType}):`, err);
+        broadcaster.broadcast({
+            type: 'memory:ingest-finished',
+            projectId,
+            sourceType,
+            sourceId,
+            sourceTitle,
+            created: 0,
+            updated: 0,
+            edgesAdded: 0,
+            skipped: {
+                parseFailed: false,
+                proposedCreate: 0, proposedUpdate: 0, proposedEdges: 0,
+                duplicateTitle: 0, uniqueConflict: 0, emptyTitle: 0,
+                invalidUpdateId: 0, invalidEdgeRef: 0, selfEdge: 0, edgeUniqueConflict: 0,
+            },
+            error: err instanceof Error ? err.message : String(err),
+        });
+    });
 }
 export function buildSourceTextFromTodo(todoId) {
     const todo = queries.getTodoById(todoId);