agentera 3.0.0-dev.5 → 3.0.0-dev.51

package/dist/analytics/extractCorpus/extractCorpusParity.js

@@ -0,0 +1,105 @@
+import fs from "node:fs";
+import path from "node:path";
+import { ADAPTER_VERSION, COPILOT_SPARSE_REMEDIATION, FAMILIES, MAX_SQLITE_ROWS, MAX_SQLITE_SESSIONS, MAX_TOOL_ARG_TEXT, RUNTIME_STORE_GLOBS, discoverRuntimeStore, } from "./core.js";
+import { buildCorpus } from "./corpus.js";
+import { runCoverageAudit } from "./coverageAudit.js";
+import { extractOpencodeSessions } from "./sqliteSessions.js";
+export const EXTRACT_CORPUS_PARITY_SCHEMA = "agentera-extract-corpus-parity-v1";
+export function buildExtractCorpusParityManifest() {
+    return {
+        schema_version: EXTRACT_CORPUS_PARITY_SCHEMA,
+        adapter_version: ADAPTER_VERSION,
+        max_sqlite_sessions: MAX_SQLITE_SESSIONS,
+        max_sqlite_rows: MAX_SQLITE_ROWS,
+        max_tool_arg_text: MAX_TOOL_ARG_TEXT,
+        copilot_sparse_remediation: COPILOT_SPARSE_REMEDIATION,
+        runtime_store_globs: { ...RUNTIME_STORE_GLOBS },
+        families: [...FAMILIES],
+    };
+}
+function trackEarliest(current, candidate) {
+    if (!candidate)
+        return current;
+    if (current === null)
+        return candidate;
+    return candidate < current ? candidate : current;
+}
+function trackLatest(current, candidate) {
+    if (!candidate)
+        return current;
+    if (current === null)
+        return candidate;
+    return candidate > current ? candidate : current;
+}
+function boundsFromRecords(records) {
+    let earliest = null;
+    let latest = null;
+    for (const item of records) {
+        const ts = typeof item.timestamp === "string" ? item.timestamp : null;
+        earliest = trackEarliest(earliest, ts);
+        latest = trackLatest(latest, ts);
+    }
+    return { earliest, latest };
+}
+/** Production opencode parity snapshot compared with the generated independent Python oracle. */
+export function opencodeParitySnapshot(dbPath) {
+    if (!fs.existsSync(dbPath)) {
+        throw new Error(`extract-corpus parity: missing opencode db at ${dbPath}`);
+    }
+    const audit = runCoverageAudit({
+        output: dbPath,
+        tierOutput: null,
+        projectRoot: [path.dirname(dbPath)],
+        codexSessionsDir: path.join(path.dirname(dbPath), "unused-codex"),
+        claudeProjectsDir: null,
+        opencodeConversationsDir: dbPath,
+        copilotConversationsDir: null,
+        cursorProjectsDir: null,
+        cursorChatsDir: null,
+        noCodex: true,
+        importSources: [],
+        noOpencode: false,
+        noCopilot: true,
+        noCursor: true,
+        acceptCoverageGap: false,
+        coverageAuditOnly: false,
+        format: "text",
+    }, {});
+    const coverageEntry = audit.runtimes.find((entry) => entry.runtime === "opencode");
+    const coverageProbe = {
+        record_count: 0,
+        earliest: coverageEntry?.earliest_session ?? null,
+        latest: coverageEntry?.latest_session ?? null,
+    };
+    const errors = [];
+    const extracted = extractOpencodeSessions(dbPath, errors);
+    const extractionBounds = boundsFromRecords(extracted);
+    const extractionProbe = {
+        record_count: extracted.length,
+        earliest: extractionBounds.earliest,
+        latest: extractionBounds.latest,
+    };
+    const corpus = buildCorpus({
+        projectRoots: [],
+        codexSessionsDir: null,
+        claudeProjectsDir: null,
+        opencodeConversationsDir: dbPath,
+    });
+    const runtimeStatus = corpus.metadata.runtime_statuses.find((status) => status.runtime === "opencode");
+    const recordCount = runtimeStatus?.record_count ?? extracted.length;
+    const discovery = discoverRuntimeStore("opencode", dbPath);
+    const discoveryShape = {
+        status: String(discovery.status),
+        file_count: typeof discovery.file_count === "number" ? discovery.file_count : null,
+    };
+    return {
+        record_count: recordCount,
+        earliest: coverageProbe.earliest,
+        latest: coverageProbe.latest,
+        probe_shapes: {
+            coverage: coverageProbe,
+            extraction: extractionProbe,
+            discovery: discoveryShape,
+        },
+    };
+}

package/dist/analytics/extractCorpus/filesystemSources.js

@@ -0,0 +1,118 @@
+import fs from "node:fs";
+import path from "node:path";
+import { resolvePath } from "../../core/paths.js";
+import { isPlainObject, isoFromMtime, record, splitLines, } from "./core.js";
+export function extractInstructionDocuments(projectRoots, errors) {
+    const records = [];
+    const docNames = [["AGENTS.md", "agents_md"]];
+    for (const root of projectRoots) {
+        for (const [filename, docType] of docNames) {
+            const p = path.join(root, filename);
+            if (!fs.existsSync(p))
+                continue;
+            let content;
+            try {
+                content = fs.readFileSync(p, "utf-8");
+            }
+            catch (exc) {
+                errors.push(`${p}: cannot read instruction document: ${exc.message}`);
+                continue;
+            }
+            records.push(record({
+                sourceKind: "instruction_document",
+                timestamp: isoFromMtime(p),
+                projectPath: root,
+                runtime: "filesystem",
+                sourceParts: [resolvePath(p)],
+                data: { doc_type: docType, name: filename, content, scope: "project" },
+            }));
+        }
+    }
+    return records;
+}
+function packageJsonSignals(p) {
+    const data = JSON.parse(fs.readFileSync(p, "utf-8"));
+    if (!isPlainObject(data))
+        return [];
+    const signals = [];
+    const name = data.name;
+    if (typeof name === "string" && name)
+        signals.push(`name=${name}`);
+    for (const section of ["scripts", "dependencies", "devDependencies"]) {
+        const sectionData = data[section];
+        if (!isPlainObject(sectionData))
+            continue;
+        for (const key of Object.keys(sectionData).sort().slice(0, 30)) {
+            signals.push(`${section}:${key}`);
+        }
+    }
+    return signals;
+}
+function textConfigSignals(p, configType) {
+    let lines;
+    try {
+        lines = splitLines(fs.readFileSync(p, "utf-8"));
+    }
+    catch {
+        return [];
+    }
+    const signals = [];
+    for (const rawLine of lines) {
+        const line = rawLine.trim();
+        if (!line || line.startsWith("#") || line.startsWith("//"))
+            continue;
+        if (configType === "gomod" && (line.startsWith("module ") || line.startsWith("go ") || line.startsWith("require "))) {
+            signals.push(line);
+        }
+        else if (configType === "pyproject" &&
+            (line.startsWith("[") || line.startsWith("requires-python") || line.startsWith("dependencies") || line.startsWith("name"))) {
+            signals.push(line);
+        }
+        else if (configType === "cargo_toml" && (line.startsWith("[") || line.startsWith("name") || line.startsWith("edition"))) {
+            signals.push(line);
+        }
+        else if (configType === "lefthook" && line.includes(":")) {
+            signals.push(line);
+        }
+        if (signals.length >= 40)
+            break;
+    }
+    return signals;
+}
+export function extractProjectConfigSignals(projectRoots, errors) {
+    const records = [];
+    const configFiles = [
+        ["package.json", "package_json", packageJsonSignals],
+        ["pyproject.toml", "pyproject", (p) => textConfigSignals(p, "pyproject")],
+        ["go.mod", "gomod", (p) => textConfigSignals(p, "gomod")],
+        ["Cargo.toml", "cargo_toml", (p) => textConfigSignals(p, "cargo_toml")],
+        [".lefthook.yml", "lefthook", (p) => textConfigSignals(p, "lefthook")],
+        ["lefthook.yml", "lefthook", (p) => textConfigSignals(p, "lefthook")],
+    ];
+    for (const root of projectRoots) {
+        for (const [filename, configType, extractor] of configFiles) {
+            const p = path.join(root, filename);
+            if (!fs.existsSync(p))
+                continue;
+            let signals;
+            try {
+                signals = extractor(p);
+            }
+            catch (exc) {
+                errors.push(`${p}: cannot extract config signals: ${exc.message}`);
+                continue;
+            }
+            if (signals.length === 0)
+                continue;
+            records.push(record({
+                sourceKind: "project_config_signal",
+                timestamp: isoFromMtime(p),
+                projectPath: root,
+                runtime: "filesystem",
+                sourceParts: [resolvePath(p), configType],
+                data: { config_type: configType, file_path: path.relative(root, p), signals },
+            }));
+        }
+    }
+    return records;
+}

package/dist/analytics/extractCorpus/index.js

@@ -0,0 +1,14 @@
+export { ADAPTER_VERSION, FAMILIES, RUNTIME_STORE_GLOBS, isoNow, isoFromMtime, stableId, projectIdFromPath, defaultAgenteraHome, defaultProfileDir, defaultOutputPath, runtimeStatus, discoverRuntimeStore, record, payloadItem, eventKind, eventTimestamp, textFromContent, claudeContentItems, iterJsonl, signalType, } from "./core.js";
+export { extractInstructionDocuments, extractProjectConfigSignals } from "./filesystemSources.js";
+export { extractCodexSessions, extractClaudeProjectSessions } from "./jsonlSessions.js";
+export { extractOpencodeSessions, PermissionDeniedError } from "./sqliteSessions.js";
+export { extractCopilotSessions } from "./copilotSessions.js";
+export { resolveOpencodeDbPath, resolveCopilotStorePath, resolveCursorProjectsPath, resolveCursorChatsPath, cursorWorkspaceHash, cursorProjectDirSlug, extractCursorSessions, extractCursorAgentSessions, } from "./cursorSessions.js";
+export { ExtractionNotImplementedError, dedupeRecords, buildMetadata, buildCorpus } from "./corpus.js";
+export { formatTruncationWarnings, resolveSqliteCaps, } from "./sqliteCaps.js";
+export { COVERAGE_EXIT_FLAGGED, corpusEnvelopeCoverage, formatCoverageSummaryText, resolveRuntimeStoreConfigs, runCoverageAudit, } from "./coverageAudit.js";
+export { parseExtractArgs, extractCorpusMain } from "./cli.js";
+export { TIER_SCHEMA_VERSION, CURRENT_POINTER_VERSION, defaultTiersDir, familyOf, deriveSignalRecords, shardFullEvidence, selectSignalsForBound, generationId, publishEvidenceTiers, readCurrentPointer, readCurrentGeneration, readSignalTier, getFullRecord, resolveEvidenceAnchor, evidenceTierCompatibility, iterTierRecords, readTierCorpusMetadata, } from "./evidenceTiers.js";
+export { tiersDirForCorpusPath, resolveTiersDir, isAnalyzable, recoveryForState, assessTiers, readBoundedMetadata, iterBoundedRecords, legacyCorpusReadable, } from "./tierReader.js";
+export { EXTRACT_CORPUS_PARITY_SCHEMA, buildExtractCorpusParityManifest, opencodeParitySnapshot, } from "./extractCorpusParity.js";
+export { readProfileSignals, resolveProfileEvidence, profileSignalsStatus, assessProfileSufficiency, } from "../profileSignals.js";

package/dist/analytics/extractCorpus/jsonlSessions.js

@@ -0,0 +1,204 @@
+import fs from "node:fs";
+import path from "node:path";
+import { resolvePath } from "../../core/paths.js";
+import { claudeContentItems, eventKind, eventTimestamp, isPlainObject, isoFromMtime, iterJsonl, payloadItem, record, rglob, signalType, textFromContent, toolCallRecord, toolCallRecordFromItem, } from "./core.js";
+function pathStem(p) {
+    const base = path.basename(p);
+    const ext = path.extname(base);
+    return ext ? base.slice(0, -ext.length) : base;
+}
+export function extractCodexSessions(sessionsDir, errors) {
+    if (sessionsDir === null || !fs.existsSync(sessionsDir))
+        return [];
+    const records = [];
+    for (const p of rglob(sessionsDir, "*.jsonl")) {
+        const fallbackTimestamp = isoFromMtime(p);
+        let sessionId = pathStem(p);
+        let projectPath = null;
+        let previousAssistant = "";
+        let index = 0;
+        for (const event of iterJsonl(p, errors)) {
+            index += 1;
+            const kind = eventKind(event);
+            const payload = isPlainObject(event.payload) ? event.payload : {};
+            if (kind === "session_meta") {
+                const sid = payload.id || event.id;
+                if (typeof sid === "string" && sid)
+                    sessionId = sid;
+                const cwd = payload.cwd || payload.working_directory;
+                if (typeof cwd === "string" && cwd)
+                    projectPath = cwd;
+                continue;
+            }
+            if (kind === "turn_context") {
+                const cwd = payload.cwd || payload.working_directory;
+                if (typeof cwd === "string" && cwd)
+                    projectPath = cwd;
+                continue;
+            }
+            const toolRecord = toolCallRecord({ event, fallbackTimestamp, projectPath, runtime: "codex", sourcePath: p, index, sessionId });
+            if (toolRecord !== null) {
+                records.push(toolRecord);
+                continue;
+            }
+            const item = payloadItem(event);
+            const itemType = item.type;
+            let role = item.role || item.actor;
+            if (kind === "user_msg")
+                role = "user";
+            if (role !== "user" && role !== "assistant")
+                continue;
+            const skipType = itemType !== undefined && itemType !== null && itemType !== "message";
+            if (skipType && !["response_item", "user_msg"].includes(kind))
+                continue;
+            const content = textFromContent(item.content || item.text || item.message);
+            if (!content)
+                continue;
+            const timestamp = eventTimestamp(event, fallbackTimestamp);
+            const data = { actor: role, content };
+            if (role === "user") {
+                if (previousAssistant)
+                    data.preceding_context = previousAssistant.slice(-2000);
+                const sig = signalType(content);
+                if (sig)
+                    data.signal_type = sig;
+            }
+            else {
+                previousAssistant = content;
+            }
+            records.push(record({
+                sourceKind: "conversation_turn",
+                timestamp,
+                projectPath,
+                runtime: "codex",
+                sourceParts: [resolvePath(p), index, role, content.slice(0, 80)],
+                sessionId,
+                data,
+            }));
+            if (role === "user") {
+                const sig = signalType(content);
+                if (sig) {
+                    records.push(record({
+                        sourceKind: "history_prompt",
+                        timestamp,
+                        projectPath,
+                        runtime: "codex",
+                        sourceParts: [resolvePath(p), index, "history", content.slice(0, 120)],
+                        sessionId,
+                        data: { prompt: content, signal_type: sig },
+                    }));
+                }
+            }
+        }
+    }
+    return records;
+}
+export function extractClaudeProjectSessions(projectsDir, errors) {
+    if (projectsDir === null || !fs.existsSync(projectsDir))
+        return [];
+    const records = [];
+    for (const p of rglob(projectsDir, "*.jsonl")) {
+        const fallbackTimestamp = isoFromMtime(p);
+        let sessionId = pathStem(p);
+        let projectPath = null;
+        let previousAssistant = "";
+        let index = 0;
+        for (const event of iterJsonl(p, errors)) {
+            index += 1;
+            const sid = event.sessionId || event.session_id || event.sessionID;
+            if (typeof sid === "string" && sid)
+                sessionId = sid;
+            const cwd = event.cwd || event.workingDirectory || event.working_directory;
+            if (typeof cwd === "string" && cwd)
+                projectPath = cwd;
+            const toolRecord = toolCallRecord({
+                event,
+                fallbackTimestamp,
+                projectPath,
+                runtime: null,
+                sourceClass: "historical_import",
+                sourceProduct: "claude-code",
+                activeRuntime: false,
+                sourcePath: p,
+                index,
+                sessionId,
+            });
+            if (toolRecord !== null) {
+                records.push(toolRecord);
+                continue;
+            }
+            let contentIndex = 0;
+            for (const item of claudeContentItems(event)) {
+                contentIndex += 1;
+                const tr = toolCallRecordFromItem({
+                    item,
+                    event,
+                    fallbackTimestamp,
+                    projectPath,
+                    runtime: null,
+                    sourceClass: "historical_import",
+                    sourceProduct: "claude-code",
+                    activeRuntime: false,
+                    sourcePath: p,
+                    index: index * 1000 + contentIndex,
+                    sessionId,
+                });
+                if (tr !== null)
+                    records.push(tr);
+            }
+            let role = event.role || event.type;
+            if (role !== "user" && role !== "assistant") {
+                const message = event.message;
+                if (isPlainObject(message))
+                    role = message.role;
+            }
+            if (role !== "user" && role !== "assistant")
+                continue;
+            const content = textFromContent(event.content || event.text || (isPlainObject(event.message) ? event.message : null));
+            if (!content)
+                continue;
+            const timestamp = eventTimestamp(event, fallbackTimestamp);
+            const data = { actor: role, content };
+            if (role === "user") {
+                if (previousAssistant)
+                    data.preceding_context = previousAssistant.slice(-2000);
+                const sig = signalType(content);
+                if (sig)
+                    data.signal_type = sig;
+            }
+            else {
+                previousAssistant = content;
+            }
+            records.push(record({
+                sourceKind: "conversation_turn",
+                timestamp,
+                projectPath,
+                runtime: null,
+                sourceClass: "historical_import",
+                sourceProduct: "claude-code",
+                activeRuntime: false,
+                sourceParts: [resolvePath(p), index, role, content.slice(0, 80)],
+                sessionId,
+                data,
+            }));
+            if (role === "user") {
+                const sig = signalType(content);
+                if (sig) {
+                    records.push(record({
+                        sourceKind: "history_prompt",
+                        timestamp,
+                        projectPath,
+                        runtime: null,
+                        sourceClass: "historical_import",
+                        sourceProduct: "claude-code",
+                        activeRuntime: false,
+                        sourceParts: [resolvePath(p), index, "history", content.slice(0, 120)],
+                        sessionId,
+                        data: { prompt: content, signal_type: sig },
+                    }));
+                }
+            }
+        }
+    }
+    return records;
+}

package/dist/analytics/extractCorpus/sqliteCaps.js

@@ -0,0 +1,43 @@
+import { MAX_SQLITE_ROWS, MAX_SQLITE_SESSIONS, } from "./core.js";
+function parsePositiveCap(value, fallback, label) {
+    if (value === undefined || value === "")
+        return fallback;
+    const parsed = Number.parseInt(value, 10);
+    if (!Number.isFinite(parsed) || parsed < 1) {
+        throw new Error(`extract-corpus: invalid ${label}: ${value}`);
+    }
+    return parsed;
+}
+export function resolveSqliteCaps(env = process.env, cliOverrides) {
+    return {
+        maxSessions: cliOverrides?.maxSessions ??
+            parsePositiveCap(env.AGENTERA_EXTRACT_MAX_SQLITE_SESSIONS, MAX_SQLITE_SESSIONS, "max sqlite sessions cap"),
+        maxRows: cliOverrides?.maxRows ??
+            parsePositiveCap(env.AGENTERA_EXTRACT_MAX_SQLITE_ROWS, MAX_SQLITE_ROWS, "max sqlite rows cap"),
+    };
+}
+export function formatTruncationWarnings(runtimeStatuses) {
+    const lines = [];
+    for (const status of runtimeStatuses) {
+        const truncatedAt = status.truncated_at;
+        if (typeof truncatedAt !== "string" || !truncatedAt)
+            continue;
+        const runtime = typeof status.runtime === "string" ? status.runtime : "unknown";
+        const cap = typeof status.truncation_cap === "string" ? status.truncation_cap : "cap";
+        const limit = status.truncation_limit ?? "?";
+        lines.push(`  ${runtime}: truncated at ${truncatedAt} (${cap} limit=${limit})`);
+    }
+    if (lines.length === 0)
+        return "";
+    return ["SQLite extraction truncated (history beyond cap omitted):", ...lines].join("\n");
+}
+export function applyTruncationToStatus(status, truncation) {
+    if (!truncation)
+        return status;
+    return {
+        ...status,
+        truncated_at: truncation.truncatedAt,
+        truncation_cap: truncation.cap,
+        truncation_limit: truncation.limit,
+    };
+}