agentera 3.0.0-dev.5 → 3.0.0-dev.51

package/dist/analytics/extractCorpus/cli.js

@@ -0,0 +1,190 @@
+import os from "node:os";
+import path from "node:path";
+import { defaultOutputPath, } from "./core.js";
+import { defaultTiersDir, publishEvidenceTiers } from "./evidenceTiers.js";
+import { buildCorpus } from "./corpus.js";
+import { COVERAGE_EXIT_FLAGGED, corpusEnvelopeCoverage, formatCoverageSummaryText, runCoverageAudit, } from "./coverageAudit.js";
+import { formatTruncationWarnings, resolveSqliteCaps } from "./sqliteCaps.js";
+import { resolveCopilotStorePath, resolveCursorChatsPath, resolveCursorProjectsPath, resolveOpencodeDbPath, } from "./cursorSessions.js";
+export function parseExtractArgs(argv, env = process.env, platform = process.platform) {
+    const home = env.HOME || (platform === "win32" ? env.USERPROFILE : undefined) || os.homedir();
+    const args = {
+        output: defaultOutputPath(env, platform),
+        tierOutput: null,
+        projectRoot: [],
+        codexSessionsDir: path.join(home, ".codex", "sessions"),
+        claudeProjectsDir: path.join(home, ".claude", "projects"),
+        opencodeConversationsDir: null,
+        copilotConversationsDir: null,
+        cursorProjectsDir: null,
+        cursorChatsDir: null,
+        noCodex: false,
+        importSources: [],
+        noOpencode: false,
+        noCopilot: false,
+        noCursor: false,
+        acceptCoverageGap: false,
+        coverageAuditOnly: false,
+        format: "text",
+    };
+    let claudeProjectsDirExplicit = false;
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        const val = (name) => {
+            if (a === name)
+                return argv[++i] ?? null;
+            if (a.startsWith(name + "="))
+                return a.slice(name.length + 1);
+            return null;
+        };
+        let v;
+        // `--output` is a deprecated no-op for back-compat: it targeted the
+        // eliminated monolithic corpus.json write. Refresh now publishes bounded
+        // tiers via `--tier-output` (default: `defaultTiersDir`). `--output` is
+        // accepted to keep older callers from hard-crashing but has no effect.
+        if ((v = val("--output")) !== null)
+            args.output = v;
+        else if ((v = val("--tier-output")) !== null)
+            args.tierOutput = v;
+        else if ((v = val("--project-root")) !== null)
+            args.projectRoot.push(v);
+        else if ((v = val("--codex-sessions-dir")) !== null)
+            args.codexSessionsDir = v;
+        else if ((v = val("--claude-projects-dir")) !== null) {
+            args.claudeProjectsDir = v;
+            claudeProjectsDirExplicit = true;
+        }
+        else if ((v = val("--import-source")) !== null) {
+            if (v !== "claude")
+                throw new Error(`extract-corpus: unsupported historical import source '${v}'; valid source: claude`);
+            if (!args.importSources.includes(v))
+                args.importSources.push(v);
+        }
+        else if ((v = val("--opencode-conversations-dir")) !== null)
+            args.opencodeConversationsDir = v;
+        else if ((v = val("--copilot-conversations-dir")) !== null)
+            args.copilotConversationsDir = v;
+        else if ((v = val("--cursor-projects-dir")) !== null)
+            args.cursorProjectsDir = v;
+        else if ((v = val("--cursor-chats-dir")) !== null)
+            args.cursorChatsDir = v;
+        else if (a === "--no-codex")
+            args.noCodex = true;
+        else if (a === "--no-opencode")
+            args.noOpencode = true;
+        else if (a === "--no-copilot")
+            args.noCopilot = true;
+        else if (a === "--no-cursor")
+            args.noCursor = true;
+        else if (a === "--accept-coverage-gap")
+            args.acceptCoverageGap = true;
+        else if (a === "--coverage-audit-only")
+            args.coverageAuditOnly = true;
+        else if ((v = val("--max-sqlite-sessions")) !== null) {
+            args.maxSqliteSessions = Number.parseInt(v, 10);
+            if (!Number.isFinite(args.maxSqliteSessions) || args.maxSqliteSessions < 1) {
+                throw new Error(`extract-corpus: invalid --max-sqlite-sessions: ${v}`);
+            }
+        }
+        else if ((v = val("--max-sqlite-rows")) !== null) {
+            args.maxSqliteRows = Number.parseInt(v, 10);
+            if (!Number.isFinite(args.maxSqliteRows) || args.maxSqliteRows < 1) {
+                throw new Error(`extract-corpus: invalid --max-sqlite-rows: ${v}`);
+            }
+        }
+        else if ((v = val("--format")) !== null) {
+            if (v !== "text" && v !== "json")
+                throw new Error(`extract-corpus: unsupported format '${v}'`);
+            args.format = v;
+        }
+        else
+            throw new Error(`extract-corpus: unrecognized argument: ${a}`);
+    }
+    if (claudeProjectsDirExplicit && !args.importSources.includes("claude")) {
+        throw new Error("extract-corpus: --claude-projects-dir requires explicit --import-source claude");
+    }
+    return args;
+}
+/** Engine entry point mirroring scripts/extract_corpus.py main(). */
+export function extractCorpusMain(argv, io = {}) {
+    const out = io.out ?? ((t) => process.stdout.write(t + "\n"));
+    const err = io.err ?? ((t) => process.stderr.write(t + "\n"));
+    const env = io.env ?? process.env;
+    const platform = io.platform ?? process.platform;
+    const cwd = io.cwd ?? process.cwd();
+    let args;
+    try {
+        args = parseExtractArgs(argv, env, platform);
+    }
+    catch (exc) {
+        err(exc.message);
+        return 2;
+    }
+    const projectRoots = args.projectRoot.length > 0 ? args.projectRoot : [cwd];
+    if (args.importSources.includes("claude")) {
+        err("Historical import warning: Claude transcripts can contain secrets, file contents, and command output. " +
+            "Import is local and read-only; records are labeled historical_import and excluded from active-runtime analytics.");
+    }
+    const audit = runCoverageAudit(args, env, platform, args.acceptCoverageGap);
+    if (args.format === "json") {
+        out(JSON.stringify({ coverage_audit: audit }, null, 2));
+    }
+    else {
+        out(formatCoverageSummaryText(audit));
+    }
+    if (audit.coverage_gap_flagged) {
+        err("coverage gap flagged (EX2): available runtime(s) skipped; pass --accept-coverage-gap to proceed");
+        return COVERAGE_EXIT_FLAGGED;
+    }
+    if (args.coverageAuditOnly)
+        return 0;
+    const sqliteCaps = resolveSqliteCaps(env, {
+        maxSessions: args.maxSqliteSessions,
+        maxRows: args.maxSqliteRows,
+    });
+    const skipCursor = args.noCursor;
+    const corpus = buildCorpus({
+        projectRoots,
+        codexSessionsDir: args.noCodex ? null : args.codexSessionsDir,
+        claudeProjectsDir: args.importSources.includes("claude") ? args.claudeProjectsDir : null,
+        opencodeConversationsDir: args.noOpencode ? null : args.opencodeConversationsDir || resolveOpencodeDbPath(env),
+        copilotConversationsDir: args.noCopilot ? null : args.copilotConversationsDir || resolveCopilotStorePath(env),
+        cursorProjectsDir: skipCursor ? null : args.cursorProjectsDir || resolveCursorProjectsPath(env),
+        cursorChatsDir: skipCursor ? null : args.cursorChatsDir || resolveCursorChatsPath(env),
+        coverage: corpusEnvelopeCoverage(audit),
+        sqliteCaps,
+    });
+    const truncationWarning = formatTruncationWarnings(corpus.metadata.runtime_statuses);
+    if (truncationWarning)
+        err(truncationWarning);
+    const total = corpus.metadata.total_records;
+    const familyBits = Object.entries(corpus.metadata.families)
+        .map(([name, summary]) => `${name}=${summary.count}`)
+        .join(", ");
+    // Publish bounded evidence tiers as the canonical bounded publication output.
+    // The monolithic corpus envelope write is eliminated: all bounded consumers
+    // (usage, report, prime coverage, startup analysis, profile synthesis) read
+    // the signal tier and full-evidence shards directly. Legacy monolithic state
+    // receives deterministic refresh guidance rather than read compatibility.
+    const tiersDir = args.tierOutput ?? defaultTiersDir(env, platform);
+    const publication = publishEvidenceTiers(corpus.records, {
+        tiersDir,
+        adapterVersion: corpus.metadata.adapter_version,
+        runtimeStatuses: corpus.metadata.runtime_statuses,
+        env,
+        platform,
+        corpusMetadata: {
+            extracted_at: corpus.metadata.extracted_at,
+            runtime_statuses: corpus.metadata.runtime_statuses,
+            coverage_envelope: {
+                available_runtimes: corpus.metadata.available_runtimes,
+                selected_runtimes: corpus.metadata.selected_runtimes,
+                available_but_not_selected: corpus.metadata.available_but_not_selected,
+            },
+        },
+    });
+    out(`published tiers: ${tiersDir} (generation ${publication.generation}; ` +
+        `${publication.shard_count} shards; ${publication.signal_count} signals; ` +
+        `${publication.signal_bytes} bytes${publication.signal_selection.capped ? "; selection applied" : ""}) [${total} records; ${familyBits}]`);
+    return 0;
+}

package/dist/analytics/extractCorpus/copilotSessions.js

@@ -0,0 +1,275 @@
+import fs from "node:fs";
+import { resolvePath } from "../../core/paths.js";
+import { isoFromMtime, record, signalType, textFromContent, } from "./core.js";
+import { isPlainObject, rglob, isFilePath } from "./core.js";
+import { resolveSqliteCaps, } from "./sqliteCaps.js";
+import { PermissionDeniedError, openSqlite, sqliteTimestamp, tableColumns, firstColumn, qualified, } from "./sqliteSessions.js";
+function copilotDbCandidates(storePath) {
+    if (isFilePath(storePath))
+        return [storePath];
+    return rglob(storePath, "session-store.db");
+}
+function copilotRows(conn, caps) {
+    const tables = new Set(conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map((r) => String(r.name)));
+    for (const t of ["sessions", "turns"]) {
+        if (!tables.has(t)) {
+            const missing = ["sessions", "turns"].filter((x) => !tables.has(x)).sort().join(",");
+            throw new Error(`missing copilot tables: ${missing}`);
+        }
+    }
+    const sessionCols = tableColumns(conn, "sessions");
+    const turnCols = tableColumns(conn, "turns");
+    const sessionId = firstColumn(sessionCols, ["id", "session_id", "sessionID"]);
+    const turnId = firstColumn(turnCols, ["id", "turn_id", "turnID"]);
+    const turnSession = firstColumn(turnCols, ["session_id", "sessionID", "session", "sessionId"]);
+    if (!(sessionId && turnSession))
+        throw new Error("missing copilot join columns");
+    const roleCol = firstColumn(turnCols, ["role", "actor", "author"]);
+    const textCol = firstColumn(turnCols, ["content", "text", "message", "prompt", "response"]);
+    if (roleCol === null)
+        throw new Error("missing copilot turn role column");
+    if (textCol === null)
+        throw new Error("missing copilot turn text column");
+    const sessionTime = firstColumn(sessionCols, ["time", "timestamp", "created_at", "createdAt"]);
+    const turnTime = firstColumn(turnCols, ["time", "timestamp", "created_at", "createdAt"]);
+    const turnOrder = firstColumn(turnCols, ["turn_index", "turnIndex", "idx", "position", "sequence"]);
+    const projectCol = firstColumn(sessionCols, ["cwd", "project_path", "projectPath", "directory", "path"]);
+    const turnData = firstColumn(turnCols, ["data", "payload", "json"]);
+    const turnType = firstColumn(turnCols, ["type", "kind"]);
+    const toolNameCol = firstColumn(turnCols, ["tool_name", "toolName", "tool", "name", "command_name", "commandName"]);
+    const toolArgs = firstColumn(turnCols, [
+        "arguments", "args", "input", "tool_input", "toolInput", "command", "command_line", "commandLine",
+    ]);
+    const orderExpr = turnOrder ? `t."${turnOrder}"` : `t."${turnTime || turnSession}"`;
+    const idExpr = turnId ? `t."${turnId}"` : "t.rowid";
+    const query = `
+        SELECT
+            ${qualified("s", sessionId, "session_id")},
+            ${qualified("s", projectCol, "project_path")},
+            ${qualified("t", turnId, "turn_id")},
+            ${qualified("t", roleCol, "role")},
+            ${qualified("t", turnTime, "turn_time")},
+            ${qualified("s", sessionTime, "session_time")},
+            ${qualified("t", textCol, "turn_text")},
+            ${qualified("t", turnData, "turn_data")},
+            ${qualified("t", turnType, "turn_type")},
+            ${qualified("t", toolNameCol, "tool_name")},
+            ${qualified("t", toolArgs, "tool_args")}
+        FROM turns t
+        JOIN sessions s ON t."${turnSession}" = s."${sessionId}"
+        ORDER BY COALESCE(${orderExpr}, t."${turnSession}"),
+                 CASE LOWER(t."${roleCol}") WHEN 'user' THEN 0 WHEN 'assistant' THEN 1 ELSE 2 END,
+                 ${idExpr}
+        LIMIT ?
+    `;
+    return conn.prepare(query).all(caps.maxRows);
+}
+export function probeCopilotTruncation(conn, caps, fallback) {
+    const tables = new Set(conn.prepare("SELECT name FROM sqlite_master WHERE type = 'table'").all().map((r) => String(r.name)));
+    if (!tables.has("sessions") || !tables.has("turns"))
+        return null;
+    const sessionCols = tableColumns(conn, "sessions");
+    const turnCols = tableColumns(conn, "turns");
+    const sessionId = firstColumn(sessionCols, ["id", "session_id", "sessionID"]);
+    const turnSession = firstColumn(turnCols, ["session_id", "sessionID", "session", "sessionId"]);
+    const roleCol = firstColumn(turnCols, ["role", "actor", "author"]);
+    if (!(sessionId && turnSession && roleCol))
+        return null;
+    const turnId = firstColumn(turnCols, ["id", "turn_id", "turnID"]);
+    const sessionTime = firstColumn(sessionCols, ["time", "timestamp", "created_at", "createdAt"]);
+    const turnTime = firstColumn(turnCols, ["time", "timestamp", "created_at", "createdAt"]);
+    const turnOrder = firstColumn(turnCols, ["turn_index", "turnIndex", "idx", "position", "sequence"]);
+    const orderExpr = turnOrder ? `t."${turnOrder}"` : `t."${turnTime || turnSession}"`;
+    const idExpr = turnId ? `t."${turnId}"` : "t.rowid";
+    const turnCount = Number(conn.prepare("SELECT COUNT(*) AS c FROM turns t JOIN sessions s ON t.\"" + turnSession + "\" = s.\"" + sessionId + "\"").get()?.c ?? 0);
+    if (turnCount <= caps.maxRows)
+        return null;
+    const timeExpr = `COALESCE(t."${turnTime || turnSession}", s."${sessionTime || sessionId}")`;
+    const query = `
+    SELECT MIN(turn_ts) AS truncated_at
+    FROM (
+      SELECT ${timeExpr} AS turn_ts
+      FROM turns t
+      JOIN sessions s ON t."${turnSession}" = s."${sessionId}"
+      ORDER BY COALESCE(${orderExpr}, t."${turnSession}"),
+               CASE LOWER(t."${roleCol}") WHEN 'user' THEN 0 WHEN 'assistant' THEN 1 ELSE 2 END,
+               ${idExpr}
+      LIMIT -1 OFFSET ?
+    )
+  `;
+    const row = conn.prepare(query).get(caps.maxRows);
+    const truncatedAt = sqliteTimestamp(row?.truncated_at, fallback);
+    return { truncatedAt, cap: "rows", limit: caps.maxRows };
+}
+function copilotArgumentDict(value) {
+    if (isPlainObject(value)) {
+        const stateInput = value.input;
+        if (isPlainObject(stateInput) && !("command" in value))
+            return stateInput;
+        return value;
+    }
+    if (typeof value === "string" && value) {
+        try {
+            const parsed = JSON.parse(value);
+            return isPlainObject(parsed) ? parsed : { value: parsed };
+        }
+        catch {
+            return { command: value };
+        }
+    }
+    return {};
+}
+function copilotJsonTools(value, errors, malformedLabel) {
+    if (typeof value === "string" && value) {
+        const raw = value;
+        try {
+            value = JSON.parse(value);
+        }
+        catch {
+            if (errors && malformedLabel && /^\s*[{[]/.test(raw))
+                errors.push(malformedLabel);
+            return [];
+        }
+    }
+    if (Array.isArray(value)) {
+        const tools = [];
+        for (const item of value)
+            tools.push(...copilotJsonTools(item, errors, malformedLabel));
+        return tools;
+    }
+    if (!isPlainObject(value))
+        return [];
+    const toolName = value.tool_name || value.toolName || value.tool || value.name;
+    const valueType = String(value.type || value.kind || "").toLowerCase();
+    if (typeof toolName === "string" &&
+        toolName &&
+        ["", "tool", "tool_call", "function_call", "tool_use", "command"].includes(valueType)) {
+        return [
+            {
+                tool_name: toolName,
+                arguments: copilotArgumentDict(value.arguments || value.args || value.input || value.state || {}),
+            },
+        ];
+    }
+    const tools = [];
+    for (const key of ["tool_calls", "toolCalls", "tools", "content", "parts", "items"]) {
+        const nested = value[key];
+        if (isPlainObject(nested) || Array.isArray(nested))
+            tools.push(...copilotJsonTools(nested, errors, malformedLabel));
+    }
+    return tools;
+}
+function copilotRowTools(row, errors) {
+    const tools = copilotJsonTools(row.turn_data, errors, "copilot malformed json tool payload");
+    tools.push(...copilotJsonTools(row.turn_text));
+    const toolName = row.tool_name;
+    if (typeof toolName === "string" && toolName) {
+        tools.push({ tool_name: toolName, arguments: copilotArgumentDict(row.tool_args) });
+    }
+    else if (String(row.turn_type || "").toLowerCase() === "command") {
+        const command = row.tool_args || row.turn_text;
+        if (typeof command === "string" && command)
+            tools.push({ tool_name: "bash", arguments: { command } });
+    }
+    return tools.filter((t) => typeof t.tool_name === "string" && t.tool_name);
+}
+export function extractCopilotSessions(storePath, errors, ctx) {
+    if (storePath === null || !fs.existsSync(storePath))
+        return [];
+    const caps = ctx?.sqliteCaps ?? resolveSqliteCaps();
+    const records = [];
+    for (const dbPath of copilotDbCandidates(storePath).slice(0, 1)) {
+        const fallbackTimestamp = isoFromMtime(dbPath);
+        let rows;
+        let conn = null;
+        try {
+            conn = openSqlite(dbPath);
+            const truncation = probeCopilotTruncation(conn, caps, fallbackTimestamp);
+            if (truncation && ctx)
+                ctx.truncation = truncation;
+            rows = copilotRows(conn, caps);
+        }
+        catch (exc) {
+            const msg = exc.message || "";
+            if (/lock|busy/i.test(msg))
+                throw new PermissionDeniedError(msg);
+            errors.push(`${dbPath}: copilot sqlite read failed: ${exc.name || "Error"}`);
+            continue;
+        }
+        finally {
+            try {
+                conn?.close();
+            }
+            catch {
+                /* ignore */
+            }
+        }
+        let previousAssistant = "";
+        let index = 0;
+        for (const row of rows) {
+            index += 1;
+            const role = String(row.role).toLowerCase();
+            if (role !== "user" && role !== "assistant")
+                continue;
+            const content = textFromContent(row.turn_text);
+            const toolItems = copilotRowTools(row, errors);
+            if (!content && toolItems.length === 0)
+                continue;
+            const projectPath = row.project_path ? String(row.project_path) : null;
+            const timestamp = sqliteTimestamp(row.turn_time || row.session_time, fallbackTimestamp);
+            if (content) {
+                const data = { actor: role, content };
+                if (role === "user") {
+                    if (previousAssistant)
+                        data.preceding_context = previousAssistant.slice(-2000);
+                    const sig = signalType(content);
+                    if (sig)
+                        data.signal_type = sig;
+                }
+                else {
+                    previousAssistant = content;
+                }
+                records.push(record({
+                    sourceKind: "conversation_turn",
+                    timestamp,
+                    projectPath,
+                    runtime: "copilot",
+                    sourceProduct: "github-copilot",
+                    sourceParts: [resolvePath(dbPath), index, role, content.slice(0, 80)],
+                    sessionId: String(row.session_id),
+                    data,
+                }));
+            }
+            let toolIndex = 0;
+            for (const toolItem of toolItems) {
+                toolIndex += 1;
+                records.push(record({
+                    sourceKind: "tool_call",
+                    timestamp,
+                    projectPath,
+                    runtime: "copilot",
+                    sourceProduct: "github-copilot",
+                    sourceParts: [resolvePath(dbPath), index, toolIndex, "tool", toolItem.tool_name, row.turn_id],
+                    sessionId: String(row.session_id),
+                    data: { tool_name: String(toolItem.tool_name), arguments: toolItem.arguments || {} },
+                }));
+            }
+            if (role === "user") {
+                const sig = signalType(content);
+                if (sig) {
+                    records.push(record({
+                        sourceKind: "history_prompt",
+                        timestamp,
+                        projectPath,
+                        runtime: "copilot",
+                        sourceProduct: "github-copilot",
+                        sourceParts: [resolvePath(dbPath), index, "history", content.slice(0, 120)],
+                        sessionId: String(row.session_id),
+                        data: { prompt: content, signal_type: sig },
+                    }));
+                }
+            }
+        }
+    }
+    return records;
+}