@meetless/mla 0.1.4

package/dist/lib/enrichment/ingest.js

@@ -0,0 +1,301 @@
+"use strict";
+// `enrich ingest`: load the authoritative run record (the agent supplies only
+// {runId, results}, never trusted plan data), re-verify it, then validate + persist each
+// scout's candidates. Security-critical: realpath containment, exist-at-HEAD via the
+// tracked set, and commit-allowlist membership all live here (plan §5, §5b, §6, §6b, §9).
+//
+// HTTP is injected as a Persister so this module is fully unit-testable without a live
+// intel server; the command wires the real kb-add POST. The filesystem/git probe is
+// likewise injectable, default-built from the repo root.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultProbe = defaultProbe;
+exports.verifyCandidate = verifyCandidate;
+exports.renderCandidateDocument = renderCandidateDocument;
+exports.statePath = statePath;
+exports.loadState = loadState;
+exports.writeState = writeState;
+exports.ingestRun = ingestRun;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const protocol_1 = require("./protocol");
+const plan_1 = require("./plan");
+const SCOUT_SLOTS = ["documentation", "history"];
+function safeRealpath(p) {
+    try {
+        return (0, node_fs_1.realpathSync)(p);
+    }
+    catch {
+        return p;
+    }
+}
+function defaultProbe(repoRoot, gitRunner = (0, plan_1.defaultGitRunner)(repoRoot)) {
+    const repoRealpath = safeRealpath(repoRoot);
+    let tracked = null;
+    return {
+        repoRealpath,
+        realpath: (absPath) => (0, node_fs_1.realpathSync)(absPath),
+        lineCount: (absPath) => (0, node_fs_1.readFileSync)(absPath, "utf8").split("\n").length,
+        isTracked: (relPath) => {
+            if (!tracked) {
+                try {
+                    tracked = new Set(gitRunner(["ls-files"])
+                        .split("\n")
+                        .map((l) => l.trim())
+                        .filter(Boolean));
+                }
+                catch {
+                    tracked = new Set();
+                }
+            }
+            return tracked.has(relPath);
+        },
+    };
+}
+// --- impure candidate verification (shape already validated upstream) -------------
+function verifyFileEvidence(ev, probe, push) {
+    const raw = ev.path.trim();
+    if ((0, node_path_1.isAbsolute)(raw)) {
+        push("path_traversal", `file path must be repo-relative: ${raw}`);
+        return;
+    }
+    const norm = raw.replace(/\\/g, "/").replace(/^\.\//, "");
+    if (norm.split("/").includes("..")) {
+        push("path_traversal", `file path may not contain "..": ${raw}`);
+        return;
+    }
+    if (!probe.isTracked(norm)) {
+        push("untracked_path", `file is not tracked at HEAD: ${norm}`);
+        return;
+    }
+    const abs = (0, node_path_1.join)(probe.repoRealpath, norm);
+    let real;
+    try {
+        real = probe.realpath(abs);
+    }
+    catch {
+        push("missing_file", `file does not exist: ${norm}`);
+        return;
+    }
+    if (real !== probe.repoRealpath && !real.startsWith(probe.repoRealpath + node_path_1.sep)) {
+        push("escapes_repo", `file resolves outside the repository: ${norm}`);
+        return;
+    }
+    let lines;
+    try {
+        lines = probe.lineCount(real);
+    }
+    catch {
+        push("missing_file", `file is unreadable: ${norm}`);
+        return;
+    }
+    if (ev.endLine > lines) {
+        push("line_out_of_range", `endLine ${ev.endLine} exceeds file length ${lines}: ${norm}`);
+    }
+}
+// Verifies a single shape-valid candidate against the filesystem + commit allowlist.
+// Rejects the whole candidate if ANY anchor fails (a citation is only as trustworthy as
+// its weakest anchor). Returns all errors for reporting.
+function verifyCandidate(candidate, run, probe, index) {
+    const errors = [];
+    const push = (code, message) => {
+        errors.push({ index, code, message });
+    };
+    const allowlist = (0, protocol_1.commitAllowlist)(run);
+    for (const ev of candidate.evidence) {
+        if (ev.type === "file") {
+            verifyFileEvidence(ev, probe, push);
+        }
+        else if ((0, protocol_1.resolveAllowedCommit)(allowlist, ev.commit) === null) {
+            push("commit_not_in_allowlist", `commit is not in the plan's allowlist: ${ev.commit}`);
+        }
+    }
+    return errors;
+}
+// --- candidate -> governed document ----------------------------------------------
+function renderCandidateDocument(candidate) {
+    const lines = [];
+    lines.push(candidate.statement.trim());
+    lines.push("");
+    lines.push(`Kind: ${candidate.kind}. Source: ${candidate.sourceScout} scout (onboarding enrichment, advisory; pending human review).`);
+    lines.push("");
+    lines.push("## Evidence");
+    for (const ev of candidate.evidence) {
+        if (ev.type === "file") {
+            lines.push(`- \`${ev.path}\` lines ${ev.startLine}-${ev.endLine}`);
+        }
+        else {
+            lines.push(`- commit \`${ev.commit}\`${ev.path ? ` (\`${ev.path}\`)` : ""}`);
+        }
+    }
+    return lines.join("\n") + "\n";
+}
+// --- per-scout state persistence (§6) --------------------------------------------
+function statePath(home, workspaceId) {
+    return (0, node_path_1.join)(home, "workspaces", workspaceId, "onboarding-state.json");
+}
+function loadState(home, workspaceId) {
+    const path = statePath(home, workspaceId);
+    if (!(0, node_fs_1.existsSync)(path))
+        return null;
+    try {
+        const parsed = JSON.parse((0, node_fs_1.readFileSync)(path, "utf8"));
+        if (parsed?.schemaVersion !== 1)
+            return null;
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+function writeState(home, state) {
+    const dir = (0, node_path_1.join)(home, "workspaces", state.workspaceId);
+    (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+    (0, node_fs_1.writeFileSync)(statePath(home, state.workspaceId), JSON.stringify(state, null, 2), "utf8");
+}
+function emptyScoutState() {
+    return { status: "not_started" };
+}
+// --- orchestration ---------------------------------------------------------------
+async function ingestRun(input) {
+    const { env, request, persist, now } = input;
+    const envelope = (0, protocol_1.validateIngestRequestShape)(request);
+    if (!envelope.ok)
+        return { ok: false, rejectionReason: envelope.error, outcomes: [] };
+    const { runId, results } = envelope.request;
+    const run = (0, plan_1.loadRunRecord)(env.home, env.workspaceId, runId);
+    if (!run)
+        return { ok: false, rejectionReason: `unknown run: ${runId}`, outcomes: [], runId };
+    if (run.workspaceId !== env.workspaceId) {
+        return { ok: false, rejectionReason: "run record workspace mismatch", outcomes: [], runId };
+    }
+    if (safeRealpath(run.repositoryRoot) !== safeRealpath(env.repositoryRoot)) {
+        return { ok: false, rejectionReason: "run record repository mismatch", outcomes: [], runId };
+    }
+    if ((0, protocol_1.computePlanDigest)(run) !== run.planDigest) {
+        return { ok: false, rejectionReason: "plan digest mismatch (run record corrupt)", outcomes: [], runId };
+    }
+    const probe = input.probe ?? defaultProbe(env.repositoryRoot, input.gitRunner);
+    // Resume: a scout already "complete" is never re-processed (its candidates are
+    // immutable; §6). Carry prior state forward.
+    const prior = loadState(env.home, env.workspaceId);
+    const scoutState = {
+        documentation: prior?.scouts.documentation ?? emptyScoutState(),
+        history: prior?.scouts.history ?? emptyScoutState(),
+    };
+    const outcomes = [];
+    let totalAccepted = 0;
+    const cap = run.limits.maxCandidatesTotal;
+    for (const rawResult of results) {
+        const shape = (0, protocol_1.validateScoutResultShape)(rawResult);
+        if (!shape.ok) {
+            // Try to attribute a malformed envelope to a slot for retry; else surface loose.
+            const guessed = guessScoutName(rawResult);
+            if (guessed)
+                scoutState[guessed] = { status: "malformed", error: shape.error };
+            outcomes.push({
+                scout: guessed ?? "documentation",
+                received: 0,
+                accepted: 0,
+                rejected: 0,
+                persisted: 0,
+                errors: [{ index: -1, code: "malformed_envelope", message: shape.error }],
+            });
+            continue;
+        }
+        const result = shape.result;
+        const scout = result.scout;
+        if (scoutState[scout].status === "complete") {
+            outcomes.push({
+                scout,
+                received: 0,
+                accepted: 0,
+                rejected: 0,
+                persisted: 0,
+                errors: [{ index: -1, code: "already_complete", message: "scout already complete; skipped" }],
+            });
+            continue;
+        }
+        // The agent reports the scout did not finish: record it, persist nothing (rerun
+        // re-runs it). Avoids partial-persist duplication for unfinished scouts.
+        if (result.status !== "complete") {
+            scoutState[scout] = { status: result.status, error: result.error };
+            outcomes.push({
+                scout,
+                received: result.candidates.length,
+                accepted: 0,
+                rejected: 0,
+                persisted: 0,
+                errors: [{ index: -1, code: result.status, message: result.error ?? `scout ${result.status}` }],
+            });
+            continue;
+        }
+        // Complete + valid envelope: validate each candidate independently.
+        const accepted = [];
+        const errors = [];
+        result.candidates.forEach((raw, i) => {
+            if (totalAccepted + accepted.length >= cap) {
+                errors.push({ index: i, code: "candidate_cap_exceeded", message: `run candidate cap (${cap}) reached` });
+                return;
+            }
+            const shapeRes = (0, protocol_1.validateCandidateShape)(raw, i);
+            if (!shapeRes.ok) {
+                errors.push(...shapeRes.errors);
+                return;
+            }
+            const verifyErrors = verifyCandidate(shapeRes.candidate, run, probe, i);
+            if (verifyErrors.length > 0) {
+                errors.push(...verifyErrors);
+                return;
+            }
+            accepted.push(shapeRes.candidate);
+        });
+        // Dedup identical candidates (same id -> same path) before POST.
+        const docsByPath = new Map();
+        for (const c of accepted) {
+            const relPath = (0, protocol_1.candidateRelPath)(c);
+            if (!docsByPath.has(relPath))
+                docsByPath.set(relPath, { relPath, content: renderCandidateDocument(c) });
+        }
+        const docs = [...docsByPath.values()];
+        let persisted = 0;
+        let status = "complete";
+        if (docs.length > 0) {
+            try {
+                persisted = (await persist(docs)).persisted;
+            }
+            catch (e) {
+                status = "persistence_failed";
+                errors.push({ index: -1, code: "persistence_failed", message: e instanceof Error ? e.message : String(e) });
+            }
+        }
+        totalAccepted += accepted.length;
+        scoutState[scout] =
+            status === "complete" ? { status: "complete", candidateCount: accepted.length } : { status, error: "kb-add persistence failed" };
+        outcomes.push({
+            scout,
+            received: result.candidates.length,
+            accepted: accepted.length,
+            rejected: result.candidates.length - accepted.length,
+            persisted,
+            errors,
+        });
+    }
+    const allComplete = SCOUT_SLOTS.every((s) => scoutState[s].status === "complete");
+    const state = {
+        workspaceId: env.workspaceId,
+        schemaVersion: 1,
+        status: allComplete ? "complete" : "partial",
+        updatedAt: now,
+        scouts: { documentation: scoutState.documentation, history: scoutState.history },
+    };
+    writeState(env.home, state);
+    return { ok: true, runId, outcomes, state };
+}
+function guessScoutName(raw) {
+    if (raw && typeof raw === "object" && "scout" in raw) {
+        const s = raw.scout;
+        if (s === "documentation" || s === "history")
+            return s;
+    }
+    return null;
+}

package/dist/lib/enrichment/plan.js

@@ -0,0 +1,253 @@
+"use strict";
+// `enrich plan` builder: turns the local repo into an immutable OnboardingRun record.
+// Three jobs (plan §5b, §8, §14): (1) rank documentation targets the doc scout should
+// read; (2) prepare a bounded git-history allowlist + context for the history scout;
+// (3) assemble + digest + persist the run record, pruning stale ones.
+//
+// The nondeterministic dependency (git) is injected as a GitRunner so the parsing and
+// byte-bounding logic is unit-testable without a real repo, mirroring the scanner's
+// injected-clock idiom. Identity/digest math lives in protocol.ts.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultGitRunner = defaultGitRunner;
+exports.buildDocumentationTargets = buildDocumentationTargets;
+exports.prepareGitEvidence = prepareGitEvidence;
+exports.buildOnboardingRun = buildOnboardingRun;
+exports.runsDir = runsDir;
+exports.runRecordPath = runRecordPath;
+exports.writeRunRecord = writeRunRecord;
+exports.loadRunRecord = loadRunRecord;
+exports.pruneOldRuns = pruneOldRuns;
+exports.createPlan = createPlan;
+const node_child_process_1 = require("node:child_process");
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const score_1 = require("../scanner/score");
+const protocol_1 = require("./protocol");
+// Defensive per-commit caps (not pinned by §8, which bounds the total). Conservative.
+const MAX_BODY_CHARS = 1000;
+const MAX_CHANGED_FILES_PER_COMMIT = 100;
+// Sentinels for single-pass `git log` parsing. Chosen to be vanishingly unlikely to
+// appear at the start of a commit-message line; a collision would garble at most one
+// commit (bounded, all downstream candidates are PENDING + human-rejectable anyway).
+const COMMIT_MARK = "@@MLA-ENRICH-COMMIT@@";
+const META_END_MARK = "@@MLA-ENRICH-ENDMETA@@";
+function defaultGitRunner(repoRoot) {
+    return (args) => (0, node_child_process_1.execFileSync)("git", args, { cwd: repoRoot, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 });
+}
+const TIER_RANK = { T1: 0, T2: 1, T3: 99, T4: 2 };
+// Rank the doc targets the documentation scout should read: T1 instruction files first,
+// then T2 decision docs, then T4 legacy notes; within a tier, deterministic by path.
+// T3 (grounding-only) and unclassified files are excluded. Capped to the limit.
+function buildDocumentationTargets(repoRoot, limit, gitRunner = defaultGitRunner(repoRoot)) {
+    let tracked;
+    try {
+        tracked = gitRunner(["ls-files"])
+            .split("\n")
+            .map((l) => l.trim())
+            .filter(Boolean);
+    }
+    catch {
+        return []; // not a git repo / git unavailable: no targets, never throw
+    }
+    const scored = [];
+    for (const path of tracked) {
+        const tier = (0, score_1.classifyTier)(path);
+        if (!tier || tier === "T3")
+            continue;
+        scored.push({ path, tier });
+    }
+    scored.sort((a, b) => TIER_RANK[a.tier] - TIER_RANK[b.tier] || a.path.localeCompare(b.path));
+    return scored.slice(0, Math.max(0, limit)).map((s, i) => ({ path: s.path, tier: s.tier, rank: i + 1 }));
+}
+// Prepare a bounded slice of recent git history: the commit allowlist (full SHAs) plus
+// enough context (subject, bounded body, changed paths + statuses, rename info) for the
+// history scout to distil decisions. No raw unbounded logs (§8). Merge commits are
+// skipped: their diffs are mechanical noise; authored commits carry the decisions.
+// A bounded diff excerpt is intentionally omitted in the MVP (the scout reads files at
+// HEAD); it is a future toggle.
+function prepareGitEvidence(repoRoot, opts) {
+    const gitRunner = opts.gitRunner ?? defaultGitRunner(repoRoot);
+    let raw;
+    try {
+        raw = gitRunner([
+            "log",
+            `-n`,
+            String(Math.max(0, opts.maxCommits)),
+            "--no-merges",
+            "--date=iso-strict",
+            "--name-status",
+            `--pretty=format:${COMMIT_MARK}%n%H%n%cI%n%s%n%b%n${META_END_MARK}`,
+        ]);
+    }
+    catch {
+        return { evidence: [], truncated: false }; // empty history / not a repo: no evidence
+    }
+    const parsed = parseGitLog(raw);
+    const evidence = [];
+    let bytes = 0;
+    let truncated = false;
+    for (const commit of parsed) {
+        if (evidence.length >= opts.maxCommits) {
+            truncated = true;
+            break;
+        }
+        const size = Buffer.byteLength(JSON.stringify(commit), "utf8");
+        if (bytes + size > opts.maxBytes && evidence.length > 0) {
+            truncated = true;
+            break; // keep at least one commit even if it alone exceeds the byte budget
+        }
+        bytes += size;
+        evidence.push(commit);
+    }
+    if (parsed.length > evidence.length)
+        truncated = true;
+    return { evidence, truncated };
+}
+// Single-pass parser for the sentinel-framed `git log --name-status` output above.
+function parseGitLog(raw) {
+    const lines = raw.split("\n");
+    const out = [];
+    let i = 0;
+    while (i < lines.length) {
+        if (lines[i] !== COMMIT_MARK) {
+            i++;
+            continue;
+        }
+        // header block: hash, committer ISO, subject, then body until META_END_MARK
+        const commit = (lines[i + 1] ?? "").trim();
+        const timestamp = (lines[i + 2] ?? "").trim();
+        const subject = lines[i + 3] ?? "";
+        i += 4;
+        const bodyLines = [];
+        while (i < lines.length && lines[i] !== META_END_MARK && lines[i] !== COMMIT_MARK) {
+            bodyLines.push(lines[i]);
+            i++;
+        }
+        if (lines[i] === META_END_MARK)
+            i++; // consume the end marker
+        // name-status lines until the next commit marker (or EOF); skip blanks
+        const changedFiles = [];
+        while (i < lines.length && lines[i] !== COMMIT_MARK) {
+            const line = lines[i];
+            i++;
+            if (!line.trim())
+                continue;
+            if (changedFiles.length >= MAX_CHANGED_FILES_PER_COMMIT)
+                continue;
+            const parts = line.split("\t");
+            const status = parts[0]?.trim();
+            if (!status)
+                continue;
+            if (/^[RC]/.test(status) && parts.length >= 3) {
+                changedFiles.push({ path: parts[2], status, renamedFrom: parts[1] });
+            }
+            else if (parts.length >= 2) {
+                changedFiles.push({ path: parts[1], status });
+            }
+        }
+        if (!commit)
+            continue; // guard against a garbled record
+        out.push({
+            commit: commit.toLowerCase(),
+            timestamp,
+            subject,
+            body: bodyLines.join("\n").trim().slice(0, MAX_BODY_CHARS),
+            changedFiles,
+        });
+    }
+    return out;
+}
+// Assemble the run record: compute the deadline from the injected clock + budget and the
+// plan digest over the integrity-bearing content. Pure (clock + runId injected).
+function buildOnboardingRun(input) {
+    const limits = input.limits ?? (0, protocol_1.defaultLimits)();
+    const deadlineAt = new Date(Date.parse(input.now) + limits.budgetMs).toISOString();
+    const partial = {
+        protocolVersion: protocol_1.PROTOCOL_VERSION,
+        workspaceId: input.workspaceId,
+        repositoryRoot: input.repositoryRoot,
+        limits,
+        documentationTargets: input.documentationTargets,
+        historyEvidence: input.historyEvidence,
+    };
+    return {
+        ...partial,
+        runId: input.runId,
+        createdAt: input.now,
+        deadlineAt,
+        planDigest: (0, protocol_1.computePlanDigest)(partial),
+    };
+}
+// --- Persistence: ~/.meetless/workspaces/<ws>/onboarding-runs/<runId>.json ---------
+function runsDir(home, workspaceId) {
+    return (0, node_path_1.join)(home, "workspaces", workspaceId, "onboarding-runs");
+}
+function runRecordPath(home, workspaceId, runId) {
+    return (0, node_path_1.join)(runsDir(home, workspaceId), `${runId}.json`);
+}
+function writeRunRecord(home, run) {
+    const dir = runsDir(home, run.workspaceId);
+    (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+    const path = runRecordPath(home, run.workspaceId, run.runId);
+    (0, node_fs_1.writeFileSync)(path, JSON.stringify(run, null, 2), "utf8");
+    return path;
+}
+function loadRunRecord(home, workspaceId, runId) {
+    const path = runRecordPath(home, workspaceId, runId);
+    if (!(0, node_fs_1.existsSync)(path))
+        return null;
+    try {
+        const parsed = JSON.parse((0, node_fs_1.readFileSync)(path, "utf8"));
+        if (parsed?.protocolVersion !== protocol_1.PROTOCOL_VERSION || parsed.runId !== runId)
+            return null;
+        return parsed;
+    }
+    catch {
+        return null;
+    }
+}
+// Keep only the current active run record; drop older ones (§5b: no run-history
+// retention). runId collisions are impossible (random), so "older" == "any other".
+function pruneOldRuns(home, workspaceId, currentRunId) {
+    const dir = runsDir(home, workspaceId);
+    if (!(0, node_fs_1.existsSync)(dir))
+        return 0;
+    let removed = 0;
+    for (const name of (0, node_fs_1.readdirSync)(dir)) {
+        if (!name.endsWith(".json") || name === `${currentRunId}.json`)
+            continue;
+        try {
+            (0, node_fs_1.unlinkSync)((0, node_path_1.join)(dir, name));
+            removed++;
+        }
+        catch {
+            // best-effort cleanup; a leftover record is harmless (ingest loads by runId)
+        }
+    }
+    return removed;
+}
+// Orchestration helper used by the `enrich plan` command: scan the repo into targets +
+// git evidence, assemble + persist the record, prune stale ones. Returns the run record
+// (the command prints it as the plan envelope).
+function createPlan(input) {
+    const limits = (0, protocol_1.defaultLimits)(input.budgetMs);
+    const gitRunner = input.gitRunner ?? defaultGitRunner(input.repositoryRoot);
+    const documentationTargets = buildDocumentationTargets(input.repositoryRoot, limits.maxDocumentTargets, gitRunner);
+    const { evidence: historyEvidence, truncated: historyTruncated } = prepareGitEvidence(input.repositoryRoot, {
+        maxCommits: limits.maxHistoryCommits,
+        maxBytes: limits.maxPreparedInputBytes,
+        gitRunner,
+    });
+    const run = buildOnboardingRun({
+        runId: input.runId,
+        workspaceId: input.workspaceId,
+        repositoryRoot: input.repositoryRoot,
+        now: input.now,
+        limits,
+        documentationTargets,
+        historyEvidence,
+    });
+    const recordPath = writeRunRecord(input.home, run);
+    const pruned = pruneOldRuns(input.home, input.workspaceId, input.runId);
+    return { run, recordPath, pruned, historyTruncated };
+}