agentera 3.0.0-dev.4 → 3.0.0-dev.46

package/dist/analytics/extractCorpus/corpus.js

@@ -0,0 +1,167 @@
+import fs from "node:fs";
+import { ADAPTER_VERSION, FAMILIES, isoNow, runtimeStatus, discoverRuntimeStore, COPILOT_SPARSE_REMEDIATION, } from "./core.js";
+import { isPlainObject } from "./core.js";
+import { extractInstructionDocuments, extractProjectConfigSignals } from "./filesystemSources.js";
+import { extractCodexSessions, extractClaudeProjectSessions } from "./jsonlSessions.js";
+import { extractOpencodeSessions, PermissionDeniedError } from "./sqliteSessions.js";
+import { extractCopilotSessions } from "./copilotSessions.js";
+import { extractCursorSessions, extractCursorAgentSessions, } from "./cursorSessions.js";
+import { resolvePath } from "../../core/paths.js";
+import { applyTruncationToStatus } from "./sqliteCaps.js";
+export class ExtractionNotImplementedError extends Error {
+}
+function extractRuntimeStore(runtime, storePath, errors, extractor, sqliteCaps, provenance = {
+    sourceProduct: runtime ?? "unknown",
+    sourceClass: "active_runtime",
+    activeRuntime: true,
+}) {
+    const discovery = discoverRuntimeStore(runtime, storePath, provenance);
+    if (discovery.status !== "available")
+        return [[], discovery];
+    const errorStart = errors.length;
+    const ctx = { sqliteCaps };
+    let records;
+    try {
+        records = extractor(storePath, errors, ctx);
+    }
+    catch (exc) {
+        const fc = (discovery.file_count ?? null); // cast: discovery payload IO boundary
+        if (exc instanceof ExtractionNotImplementedError) {
+            return [[], runtimeStatus(runtime, { status: "degraded", reason: "extractor_unimplemented", storePath, fileCount: fc, recordCount: 0, errorCount: 0, ...provenance })];
+        }
+        if (exc instanceof PermissionDeniedError) {
+            return [[], runtimeStatus(runtime, { status: "degraded", reason: "store_locked", storePath, fileCount: fc, ...provenance })];
+        }
+        return [[], runtimeStatus(runtime, { status: "degraded", reason: "store_unreadable", storePath, fileCount: fc, ...provenance })];
+    }
+    const fc = (discovery.file_count ?? null); // cast: discovery payload IO boundary
+    const errorCount = errors.length - errorStart;
+    if (errorCount) {
+        return [records, runtimeStatus(runtime, { status: "degraded", reason: "schema_divergent", storePath, fileCount: fc, recordCount: records.length, errorCount, ...provenance })];
+    }
+    if (records.length === 0) {
+        return [records, runtimeStatus(runtime, {
+                status: "sparse",
+                reason: "no_matching_records",
+                storePath,
+                fileCount: fc,
+                recordCount: 0,
+                remediationLabels: runtime === "copilot" ? [COPILOT_SPARSE_REMEDIATION] : null,
+                ...provenance,
+            })];
+    }
+    return [
+        records,
+        applyTruncationToStatus(runtimeStatus(runtime, { status: "ok", reason: "records_extracted", storePath, fileCount: fc, recordCount: records.length, errorCount: 0, ...provenance }), ctx.truncation),
+    ];
+}
+export function dedupeRecords(records) {
+    const byId = new Map();
+    for (const item of records)
+        byId.set(item.source_id, item);
+    const actorOrder = (item) => {
+        const actor = isPlainObject(item.data) ? item.data.actor : null;
+        return actor === "user" ? 0 : actor === "assistant" ? 1 : 2;
+    };
+    return Array.from(byId.values()).sort((a, b) => {
+        const at = (a.timestamp ?? "");
+        const bt = (b.timestamp ?? "");
+        if (at !== bt)
+            return at < bt ? -1 : 1;
+        const ak = (a.source_kind ?? "");
+        const bk = (b.source_kind ?? "");
+        if (ak !== bk)
+            return ak < bk ? -1 : 1;
+        const ao = actorOrder(a);
+        const bo = actorOrder(b);
+        if (ao !== bo)
+            return ao - bo;
+        const ai = (a.source_id ?? "");
+        const bi = (b.source_id ?? "");
+        return ai < bi ? -1 : ai > bi ? 1 : 0;
+    });
+}
+export function buildMetadata(records, errors, runtimeStatuses, coverage) {
+    const counts = new Map();
+    for (const item of records) {
+        const sk = item.source_kind;
+        if (FAMILIES.includes(sk))
+            counts.set(sk, (counts.get(sk) ?? 0) + 1);
+    }
+    const families = {};
+    for (const family of FAMILIES) {
+        const count = counts.get(family) ?? 0;
+        families[family] = { count, status: count ? "ok" : "missing" };
+        if (count === 0)
+            families[family].error = "no records extracted for this family";
+    }
+    const runtimes = Array.from(new Set(records.filter((item) => item.active_runtime === true && item.runtime).map((item) => String(item.runtime)))).sort();
+    const sourceClasses = Array.from(new Set(records.map((item) => String(item.source_class)))).sort();
+    const sourceProducts = Array.from(new Set(records.map((item) => String(item.source_product)))).sort();
+    const coverageFields = coverage ?? {
+        available_runtimes: [],
+        selected_runtimes: [],
+        available_but_not_selected: [],
+    };
+    return {
+        extracted_at: isoNow(),
+        runtimes,
+        adapter_version: ADAPTER_VERSION,
+        families,
+        runtime_statuses: runtimeStatuses,
+        available_runtimes: coverageFields.available_runtimes,
+        selected_runtimes: coverageFields.selected_runtimes,
+        available_but_not_selected: coverageFields.available_but_not_selected,
+        total_records: records.length,
+        errors,
+        source_classes: sourceClasses,
+        source_products: sourceProducts,
+    };
+}
+export function buildCorpus(opts) {
+    const errors = [];
+    const normalizedRoots = [];
+    for (const root of opts.projectRoots) {
+        if (fs.existsSync(root))
+            normalizedRoots.push(resolvePath(root));
+        else
+            errors.push(`${root}: project root does not exist`);
+    }
+    const records = [];
+    records.push(...extractInstructionDocuments(normalizedRoots, errors));
+    records.push(...extractProjectConfigSignals(normalizedRoots, errors));
+    const runtimeStatuses = [];
+    const runtimes = [
+        ["codex", "codex", "active_runtime", true, opts.codexSessionsDir, extractCodexSessions, undefined],
+        ["cursor", "cursor", "active_runtime", true, opts.cursorProjectsDir ?? null, (sp, err) => extractCursorSessions(sp, err, normalizedRoots), undefined],
+        [
+            "cursor",
+            "cursor-agent",
+            "active_runtime",
+            true,
+            opts.cursorChatsDir ?? null,
+            (sp, err) => extractCursorAgentSessions(sp, err, normalizedRoots, opts.cursorProjectsDir ?? null),
+            "store.db",
+        ],
+        ["opencode", "opencode", "active_runtime", true, opts.opencodeConversationsDir ?? null, extractOpencodeSessions, undefined],
+        ["copilot", "github-copilot", "active_runtime", true, opts.copilotConversationsDir ?? null, extractCopilotSessions, undefined],
+    ];
+    if (opts.claudeProjectsDir !== null) {
+        runtimes.push([
+            null,
+            "claude-code",
+            "historical_import",
+            false,
+            opts.claudeProjectsDir,
+            extractClaudeProjectSessions,
+            "*.jsonl",
+        ]);
+    }
+    for (const [runtime, sourceProduct, sourceClass, activeRuntime, storePath, extractor, pattern] of runtimes) {
+        const [runtimeRecords, status] = extractRuntimeStore(runtime, storePath, errors, extractor, opts.sqliteCaps, { sourceProduct, sourceClass, activeRuntime, pattern });
+        records.push(...runtimeRecords);
+        runtimeStatuses.push(status);
+    }
+    const deduped = dedupeRecords(records);
+    return { metadata: buildMetadata(deduped, errors, runtimeStatuses, opts.coverage), records: deduped };
+}

package/dist/analytics/extractCorpus/coverageAudit.js

@@ -0,0 +1,260 @@
+import { discoverRuntimeStore, eventTimestamp, isoFromMtime, iterJsonl, rglob, } from "./core.js";
+import { isFilePath, isDir } from "./core.js";
+import { resolveCopilotStorePath, resolveCursorChatsPath, resolveCursorProjectsPath, resolveOpencodeDbPath, } from "./cursorSessions.js";
+import { openSqlite, PermissionDeniedError, sqliteTimestamp, tableColumns, firstColumn, } from "./sqliteSessions.js";
+export const COVERAGE_EXIT_FLAGGED = 4;
+export function corpusEnvelopeCoverage(audit) {
+    return {
+        available_runtimes: [...audit.available_runtimes],
+        selected_runtimes: [...audit.selected_runtimes],
+        available_but_not_selected: audit.skipped_available.map((item) => ({
+            runtime: item.runtime,
+            reason: item.reason,
+            store_path: item.store_path,
+        })),
+    };
+}
+export function resolveRuntimeStoreConfigs(args, env = process.env, _platform = process.platform) {
+    const configs = [
+        { runtime: "codex", sourceProduct: "codex", sourceClass: "active_runtime", activeRuntime: true, storePath: args.codexSessionsDir, selected: !args.noCodex, skipReason: args.noCodex ? "disabled_by_flag" : null },
+        { runtime: "cursor", sourceProduct: "cursor", sourceClass: "active_runtime", activeRuntime: true, storePath: args.cursorProjectsDir || resolveCursorProjectsPath(env), selected: !args.noCursor, skipReason: args.noCursor ? "disabled_by_flag" : null },
+        { runtime: "cursor", sourceProduct: "cursor-agent", sourceClass: "active_runtime", activeRuntime: true, pattern: "store.db", storePath: args.cursorChatsDir || resolveCursorChatsPath(env), selected: !args.noCursor, skipReason: args.noCursor ? "disabled_by_flag" : null },
+        { runtime: "opencode", sourceProduct: "opencode", sourceClass: "active_runtime", activeRuntime: true, storePath: args.opencodeConversationsDir || resolveOpencodeDbPath(env), selected: !args.noOpencode, skipReason: args.noOpencode ? "disabled_by_flag" : null },
+        { runtime: "copilot", sourceProduct: "github-copilot", sourceClass: "active_runtime", activeRuntime: true, storePath: args.copilotConversationsDir || resolveCopilotStorePath(env), selected: !args.noCopilot, skipReason: args.noCopilot ? "disabled_by_flag" : null },
+    ];
+    if ((args.importSources ?? []).includes("claude")) {
+        configs.push({
+            runtime: null,
+            sourceProduct: "claude-code",
+            sourceClass: "historical_import",
+            activeRuntime: false,
+            pattern: "*.jsonl",
+            storePath: args.claudeProjectsDir,
+            selected: true,
+            skipReason: null,
+        });
+    }
+    return configs.map((config) => ({ ...config, storePath: config.storePath ?? null }));
+}
+function trackEarliest(current, candidate) {
+    if (!candidate)
+        return current;
+    if (current === null)
+        return candidate;
+    return candidate < current ? candidate : current;
+}
+function trackLatest(current, candidate) {
+    if (!candidate)
+        return current;
+    if (current === null)
+        return candidate;
+    return candidate > current ? candidate : current;
+}
+function probeJsonlTimestamps(storePath) {
+    let earliest = null;
+    let latest = null;
+    for (const filePath of rglob(storePath, "*.jsonl")) {
+        const fallback = isoFromMtime(filePath);
+        let sawEvent = false;
+        for (const event of iterJsonl(filePath, [])) {
+            sawEvent = true;
+            const ts = eventTimestamp(event, fallback);
+            earliest = trackEarliest(earliest, ts);
+            latest = trackLatest(latest, ts);
+        }
+        if (!sawEvent) {
+            earliest = trackEarliest(earliest, fallback);
+            latest = trackLatest(latest, fallback);
+        }
+    }
+    return { earliest, latest };
+}
+function probeSqliteTimestamps(storePath, sourceProduct) {
+    const dbPaths = sourceProduct === "opencode"
+        ? [storePath]
+        : sourceProduct === "github-copilot"
+            ? isFilePath(storePath)
+                ? [storePath]
+                : rglob(storePath, "session-store.db")
+            : rglob(storePath, "store.db");
+    let earliest = null;
+    let latest = null;
+    for (const dbPath of dbPaths) {
+        if (!isFilePath(dbPath))
+            continue;
+        const fallback = isoFromMtime(dbPath);
+        let conn;
+        try {
+            conn = openSqlite(dbPath);
+        }
+        catch (exc) {
+            if (exc instanceof PermissionDeniedError)
+                continue;
+            continue;
+        }
+        try {
+            if (sourceProduct === "opencode") {
+                const cols = tableColumns(conn, "session");
+                const timeCol = firstColumn(cols, ["time_created", "time", "timestamp", "created_at", "createdAt"]);
+                if (timeCol) {
+                    const row = conn
+                        .prepare(`SELECT MIN("${timeCol.replace(/"/g, '""')}") AS min_ts, MAX("${timeCol.replace(/"/g, '""')}") AS max_ts FROM session`)
+                        .get();
+                    if (row) {
+                        const minTs = sqliteTimestamp(row.min_ts, fallback);
+                        const maxTs = sqliteTimestamp(row.max_ts, fallback);
+                        earliest = trackEarliest(earliest, minTs);
+                        latest = trackLatest(latest, maxTs);
+                    }
+                }
+            }
+            else if (sourceProduct === "github-copilot") {
+                const sessionCols = tableColumns(conn, "sessions");
+                const turnCols = tableColumns(conn, "turns");
+                const sessionTime = firstColumn(sessionCols, ["time", "timestamp", "created_at", "createdAt"]);
+                const turnTime = firstColumn(turnCols, ["time", "timestamp", "created_at", "createdAt"]);
+                const timeCol = turnTime ?? sessionTime;
+                const table = turnTime ? "turns" : sessionTime ? "sessions" : null;
+                if (timeCol && table) {
+                    const escaped = timeCol.replace(/"/g, '""');
+                    const row = conn.prepare(`SELECT MIN("${escaped}") AS min_ts, MAX("${escaped}") AS max_ts FROM ${table}`).get();
+                    if (row) {
+                        const minTs = sqliteTimestamp(row.min_ts, fallback);
+                        const maxTs = sqliteTimestamp(row.max_ts, fallback);
+                        earliest = trackEarliest(earliest, minTs);
+                        latest = trackLatest(latest, maxTs);
+                    }
+                }
+            }
+            else if (sourceProduct === "cursor-agent") {
+                const rows = conn.prepare("SELECT data FROM blobs ORDER BY id").all();
+                for (const row of rows) {
+                    const payload = row.data;
+                    let raw = "";
+                    if (payload instanceof Uint8Array)
+                        raw = Buffer.from(payload).toString("utf-8");
+                    else if (typeof payload === "string")
+                        raw = payload;
+                    if (!raw)
+                        continue;
+                    try {
+                        const parsed = JSON.parse(raw); // cast: IO boundary — parsed subprocess stdout JSON
+                        const ts = eventTimestamp(parsed, fallback);
+                        earliest = trackEarliest(earliest, ts);
+                        latest = trackLatest(latest, ts);
+                    }
+                    catch {
+                        earliest = trackEarliest(earliest, fallback);
+                        latest = trackLatest(latest, fallback);
+                    }
+                }
+                if (rows.length === 0) {
+                    earliest = trackEarliest(earliest, fallback);
+                    latest = trackLatest(latest, fallback);
+                }
+            }
+        }
+        catch {
+            earliest = trackEarliest(earliest, fallback);
+            latest = trackLatest(latest, fallback);
+        }
+        finally {
+            conn.close();
+        }
+    }
+    return { earliest, latest };
+}
+function probeRuntimeTimestamps(sourceProduct, storePath) {
+    if (sourceProduct === "opencode" || sourceProduct === "github-copilot" || sourceProduct === "cursor-agent") {
+        return probeSqliteTimestamps(storePath, sourceProduct);
+    }
+    if (isDir(storePath))
+        return probeJsonlTimestamps(storePath);
+    if (isFilePath(storePath))
+        return probeSqliteTimestamps(storePath, sourceProduct);
+    return { earliest: null, latest: null };
+}
+export function runCoverageAudit(args, env = process.env, platform = process.platform, acceptCoverageGap = false) {
+    const runtimes = [];
+    const skippedAvailable = [];
+    for (const config of resolveRuntimeStoreConfigs(args, env, platform)) {
+        const discovery = discoverRuntimeStore(config.runtime, config.storePath, {
+            sourceProduct: config.sourceProduct,
+            sourceClass: config.sourceClass,
+            activeRuntime: config.activeRuntime,
+            pattern: config.pattern,
+        });
+        const available = discovery.status === "available";
+        let earliest = null;
+        let latest = null;
+        if (available && config.storePath) {
+            const bounds = probeRuntimeTimestamps(config.sourceProduct, config.storePath);
+            earliest = bounds.earliest;
+            latest = bounds.latest;
+        }
+        const skipReason = available && !config.selected ? (config.skipReason ?? "disabled_by_flag") : null;
+        if (skipReason && config.storePath) {
+            skippedAvailable.push({ runtime: config.runtime, reason: skipReason, store_path: config.storePath });
+        }
+        runtimes.push({
+            runtime: config.runtime,
+            source_product: config.sourceProduct,
+            source_class: config.sourceClass,
+            active_runtime: config.activeRuntime,
+            store_path: config.storePath,
+            selected: config.selected,
+            discovery_status: String(discovery.status),
+            discovery_reason: String(discovery.reason),
+            available,
+            earliest_session: earliest,
+            latest_session: latest,
+            skip_reason: skipReason,
+        });
+    }
+    const availableRuntimes = [...new Set(runtimes.filter((r) => r.active_runtime && r.available).map((r) => r.runtime))];
+    const selectedRuntimes = [...new Set(runtimes.filter((r) => r.active_runtime && r.selected).map((r) => r.runtime))];
+    const coverageGapFlagged = skippedAvailable.length > 0 && !acceptCoverageGap;
+    return {
+        runtimes,
+        available_runtimes: availableRuntimes,
+        selected_runtimes: selectedRuntimes,
+        skipped_available: skippedAvailable,
+        coverage_gap_flagged: coverageGapFlagged,
+        exit_signal: skippedAvailable.length > 0 ? (acceptCoverageGap ? "complete" : "flagged") : "complete",
+    };
+}
+export function formatCoverageSummaryText(audit) {
+    const lines = ["Coverage Audit (pre-extraction)"];
+    for (const entry of audit.runtimes) {
+        const store = entry.store_path ?? "(none)";
+        if (entry.available) {
+            const span = entry.earliest_session && entry.latest_session
+                ? `${entry.earliest_session} .. ${entry.latest_session}`
+                : "timestamps unavailable";
+            const label = entry.active_runtime ? entry.runtime : `${entry.source_product} [historical import]`;
+            lines.push(`  ${label}: available store=${store} sessions=${span} selected=${entry.selected ? "yes" : "no"}`);
+        }
+        else {
+            lines.push(`  ${entry.active_runtime ? entry.runtime : `${entry.source_product} [historical import]`}: ${entry.discovery_status} (${entry.discovery_reason}) store=${store} selected=${entry.selected ? "yes" : "no"}`);
+        }
+    }
+    if (audit.skipped_available.length > 0) {
+        lines.push("Skipped available runtimes:");
+        for (const item of audit.skipped_available) {
+            lines.push(`  - ${item.runtime}: ${item.reason} (${item.store_path})`);
+        }
+        if (audit.coverage_gap_flagged) {
+            lines.push("Coverage gap flagged (EX2): pass --accept-coverage-gap to proceed with partial extraction.");
+        }
+        else {
+            lines.push("Coverage gap accepted: proceeding with selected runtimes only.");
+        }
+    }
+    else if (audit.available_runtimes.length > 0) {
+        lines.push("All available runtimes are selected; no coverage gap.");
+    }
+    else {
+        lines.push("No available runtime stores detected on this host.");
+    }
+    return lines.join("\n");
+}