agentera 3.0.0-dev.5 → 3.0.0-dev.51

package/dist/analytics/extractCorpus/core.js

@@ -0,0 +1,392 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { resolveProfileDirOverride, resolveXdgDataHome } from "../../core/envPaths.js";
+import { expanduser } from "../../core/paths.js";
+import crypto from "node:crypto";
+import { resolvePath } from "../../core/paths.js";
+export const ADAPTER_VERSION = "agentera-v3-corpus-2";
+export const FAMILIES = [
+    "instruction_document",
+    "history_prompt",
+    "conversation_turn",
+    "tool_call",
+    "project_config_signal",
+];
+export const RUNTIME_STORE_GLOBS = {
+    codex: "*.jsonl",
+    cursor: "*.jsonl",
+    opencode: "opencode.db",
+    copilot: "session-store.db",
+};
+export const HISTORICAL_IMPORT_STORE_GLOBS = {
+    "claude-code": "*.jsonl",
+};
+const ACTIVE_ANALYTICS_RUNTIMES = new Set(["opencode", "codex", "cursor", "copilot"]);
+export const MAX_TOOL_ARG_TEXT = 500;
+export const MAX_SQLITE_ROWS = 100_000;
+export const MAX_SQLITE_SESSIONS = 60;
+export const COPILOT_SPARSE_REMEDIATION = "/chronicle reindex";
+const DECISION_RE = /\b(decide|decision|prefer|preference|instead|avoid|don't|do not|should|trade[- ]?off|scope|plan|commit|review|fix|why|question|blocked|stuck|approve|reject|change|keep|remove)\b/i;
+const CORRECTION_RE = /\b(no|not quite|actually|rather|instead|wrong|correction|that's not|that is not|don't|do not)\b/i;
+const QUESTION_RE = /\?|^\s*(why|what|how|should|can|could|would)\b/i;
+export function isoNow() {
+    // Python datetime.now(utc).isoformat() -> microseconds + "+00:00" -> "Z".
+    // JS gives milliseconds; corpus extracted_at is a wall-clock stamp, not a
+    // parity-critical record value.
+    return new Date().toISOString().replace(/Z$/, "Z");
+}
+export function isoFromMtime(p) {
+    const ms = fs.statSync(p).mtimeMs;
+    return new Date(ms).toISOString().replace(/Z$/, "Z");
+}
+export function stableId(...parts) {
+    const raw = parts.map((p) => pyStr(p)).join("\0");
+    return crypto.createHash("sha256").update(raw, "utf-8").digest("hex").slice(0, 24);
+}
+/** Python str() for the scalar/None values stable_id receives. */
+function pyStr(value) {
+    if (value === null || value === undefined)
+        return "None";
+    if (value === true)
+        return "True";
+    if (value === false)
+        return "False";
+    return String(value);
+}
+export function projectIdFromPath(p) {
+    if (p === null)
+        return "global";
+    const name = path.basename(p) || p;
+    const slug = name
+        .replace(/[^A-Za-z0-9._-]+/g, "-")
+        .replace(/^-+/, "")
+        .replace(/-+$/, "")
+        .toLowerCase();
+    return slug || "global";
+}
+export function defaultAgenteraHome(env = process.env, platform = process.platform) {
+    const override = env.AGENTERA_HOME;
+    if (override)
+        return expanduser(override);
+    if (platform === "darwin")
+        return path.join(os.homedir(), "Library", "Application Support", "agentera");
+    if (platform === "win32") {
+        const appdata = env.APPDATA || path.join(os.homedir(), "AppData", "Roaming");
+        return path.join(expanduser(appdata), "agentera");
+    }
+    return path.join(resolveXdgDataHome(env), "agentera");
+}
+export function defaultProfileDir(env = process.env, platform = process.platform) {
+    const override = resolveProfileDirOverride(env);
+    if (override)
+        return expanduser(override);
+    return defaultAgenteraHome(env, platform);
+}
+export function defaultOutputPath(env = process.env, platform = process.platform) {
+    return path.join(defaultProfileDir(env, platform), "intermediate", "corpus.json");
+}
+export function runtimeStatus(runtime, opts) {
+    const activeRuntime = opts.activeRuntime ?? (runtime !== null && ACTIVE_ANALYTICS_RUNTIMES.has(runtime));
+    const item = {
+        runtime,
+        source_class: opts.sourceClass ?? (activeRuntime ? "active_runtime" : "historical_import"),
+        source_product: opts.sourceProduct ?? runtime ?? "unknown",
+        active_runtime: activeRuntime,
+        status: opts.status,
+        reason: opts.reason,
+    };
+    if (opts.storePath !== null && opts.storePath !== undefined)
+        item.store_path = opts.storePath;
+    if (opts.fileCount !== null && opts.fileCount !== undefined)
+        item.file_count = opts.fileCount;
+    if (opts.recordCount !== null && opts.recordCount !== undefined)
+        item.record_count = opts.recordCount;
+    if (opts.errorCount !== null && opts.errorCount !== undefined)
+        item.error_count = opts.errorCount;
+    if (opts.remediationLabels && opts.remediationLabels.length > 0)
+        item.remediation_labels = opts.remediationLabels;
+    if (opts.truncatedAt)
+        item.truncated_at = opts.truncatedAt;
+    if (opts.truncationCap)
+        item.truncation_cap = opts.truncationCap;
+    if (opts.truncationLimit !== null && opts.truncationLimit !== undefined)
+        item.truncation_limit = opts.truncationLimit;
+    return item;
+}
+function isDir(p) {
+    try {
+        return fs.statSync(p).isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+function isFilePath(p) {
+    try {
+        return fs.statSync(p).isFile();
+    }
+    catch {
+        return false;
+    }
+}
+/** Recursive glob for a simple pattern ("*.jsonl" or an exact filename). */
+function rglob(root, pattern) {
+    const out = [];
+    const matchesExt = pattern.startsWith("*.") ? pattern.slice(1) : null; // ".jsonl"
+    const exact = matchesExt ? null : pattern;
+    const walk = (dir) => {
+        let entries;
+        try {
+            entries = fs.readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            const full = path.join(dir, e.name);
+            if (e.isDirectory())
+                walk(full);
+            else if (e.isFile()) {
+                if (matchesExt ? e.name.endsWith(matchesExt) : e.name === exact)
+                    out.push(full);
+            }
+        }
+    };
+    walk(root);
+    return out.sort();
+}
+export function discoverRuntimeStore(runtime, storePath, opts = {}) {
+    const statusOpts = {
+        sourceProduct: opts.sourceProduct,
+        sourceClass: opts.sourceClass,
+        activeRuntime: opts.activeRuntime,
+    };
+    if (storePath === null) {
+        return runtimeStatus(runtime, { status: "skipped", reason: "disabled", storePath: null, ...statusOpts });
+    }
+    if (!fs.existsSync(storePath)) {
+        return runtimeStatus(runtime, {
+            status: "missing",
+            reason: "store_absent",
+            storePath,
+            remediationLabels: runtime === "copilot" ? [COPILOT_SPARSE_REMEDIATION] : null,
+            ...statusOpts,
+        });
+    }
+    if (isFilePath(storePath)) {
+        return runtimeStatus(runtime, {
+            status: "available",
+            reason: "candidate_files_found",
+            storePath,
+            fileCount: 1,
+            ...statusOpts,
+        });
+    }
+    if (!isDir(storePath)) {
+        return runtimeStatus(runtime, { status: "degraded", reason: "store_not_directory", storePath });
+    }
+    let candidates;
+    try {
+        const pattern = opts.pattern ?? (runtime === null ? undefined : RUNTIME_STORE_GLOBS[runtime]);
+        if (!pattern)
+            throw new Error("no declared store pattern");
+        candidates = rglob(storePath, pattern);
+    }
+    catch {
+        return runtimeStatus(runtime, { status: "degraded", reason: "store_unreadable", storePath });
+    }
+    if (candidates.length === 0) {
+        return runtimeStatus(runtime, {
+            status: "sparse",
+            reason: "no_candidate_files",
+            storePath,
+            fileCount: 0,
+            remediationLabels: runtime === "copilot" ? [COPILOT_SPARSE_REMEDIATION] : null,
+            ...statusOpts,
+        });
+    }
+    return runtimeStatus(runtime, {
+        status: "available",
+        reason: "candidate_files_found",
+        storePath,
+        fileCount: candidates.length,
+        ...statusOpts,
+    });
+}
+// ── record builders + content helpers ──────────────────────────────
+export function splitLines(text) {
+    const parts = text.split(/\r\n|\r|\n/);
+    if (parts.length > 0 && parts[parts.length - 1] === "")
+        parts.pop();
+    return parts;
+}
+export function record(opts) {
+    const activeRuntime = opts.activeRuntime ?? (opts.runtime !== null && ACTIVE_ANALYTICS_RUNTIMES.has(opts.runtime));
+    const item = {
+        source_id: stableId(opts.sourceKind, ...opts.sourceParts),
+        timestamp: opts.timestamp,
+        project_id: projectIdFromPath(opts.projectPath),
+        source_kind: opts.sourceKind,
+        runtime: opts.runtime,
+        source_class: opts.sourceClass ?? (activeRuntime ? "active_runtime" : "project"),
+        source_product: opts.sourceProduct ?? opts.runtime ?? "unknown",
+        active_runtime: activeRuntime,
+        adapter_version: ADAPTER_VERSION,
+        data: opts.data,
+    };
+    if (opts.projectPath !== null && opts.projectPath !== undefined)
+        item.project_path = opts.projectPath;
+    if (opts.sessionId) {
+        item.session_id = opts.sessionId;
+        item.conversation_key = opts.sessionId;
+    }
+    return item;
+}
+function isPlainObject(v) {
+    return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+export function payloadItem(event) {
+    const payload = event.payload;
+    if (isPlainObject(payload)) {
+        const nested = payload.item;
+        if (isPlainObject(nested))
+            return nested;
+        return payload;
+    }
+    return event;
+}
+export function eventKind(event) {
+    for (const key of ["type", "event", "name"]) {
+        const value = event[key];
+        if (typeof value === "string")
+            return value;
+    }
+    return "";
+}
+export function eventTimestamp(item, fallback) {
+    const payload = isPlainObject(item.payload) ? item.payload : {};
+    const nested = isPlainObject(payload.item) ? payload.item : {};
+    for (const source of [item, payload, nested]) {
+        for (const key of ["timestamp", "created_at", "createdAt", "time"]) {
+            const value = source[key];
+            if (typeof value === "string" && value)
+                return value;
+            if (typeof value === "number") {
+                return new Date(value * 1000).toISOString();
+            }
+        }
+    }
+    return fallback;
+}
+export function textFromContent(value) {
+    if (value === null || value === undefined)
+        return "";
+    if (typeof value === "string")
+        return value;
+    if (Array.isArray(value)) {
+        const parts = value.map((it) => textFromContent(it));
+        return parts.filter((p) => p).join("\n");
+    }
+    if (isPlainObject(value)) {
+        for (const key of ["text", "input_text", "output_text", "message", "content"]) {
+            const text = textFromContent(value[key]);
+            if (text)
+                return text;
+        }
+        return "";
+    }
+    return pyStr(value);
+}
+export function claudeContentItems(event) {
+    const items = [];
+    const msg = isPlainObject(event.message) ? event.message : null;
+    for (const source of [event, msg]) {
+        if (!isPlainObject(source))
+            continue;
+        const content = source.content;
+        if (Array.isArray(content)) {
+            for (const it of content)
+                if (isPlainObject(it))
+                    items.push(it);
+        }
+        else if (isPlainObject(content)) {
+            items.push(content);
+        }
+    }
+    return items;
+}
+export function* iterJsonl(p, errors) {
+    let text;
+    try {
+        text = fs.readFileSync(p, "utf-8");
+    }
+    catch (exc) {
+        errors.push(`${p}: cannot read: ${exc.message}`);
+        return;
+    }
+    const lines = text.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+        const stripped = lines[i].trim();
+        if (!stripped)
+            continue;
+        let item;
+        try {
+            item = JSON.parse(stripped);
+        }
+        catch (exc) {
+            errors.push(`${p}:${i + 1}: invalid jsonl: ${exc.message}`);
+            continue;
+        }
+        if (isPlainObject(item))
+            yield item;
+    }
+}
+export function signalType(text) {
+    if (!text || !DECISION_RE.test(text))
+        return null;
+    if (CORRECTION_RE.test(text))
+        return "correction";
+    if (QUESTION_RE.test(text))
+        return "question";
+    return "decision";
+}
+export function toolCallRecordFromItem(args) {
+    const { item, event } = args;
+    const kind = eventKind(event);
+    const itemType = item.type;
+    if (!["tool_call", "function_call"].includes(kind) &&
+        !["tool_call", "function_call", "tool_use"].includes(itemType)) {
+        return null;
+    }
+    const toolName = item.tool_name || item.name || item.tool;
+    if (typeof toolName !== "string" || !toolName)
+        return null;
+    let argumentsVal = item.arguments || item.input || item.args || {};
+    if (typeof argumentsVal === "string") {
+        try {
+            argumentsVal = JSON.parse(argumentsVal);
+        }
+        catch {
+            argumentsVal = { raw: argumentsVal };
+        }
+    }
+    if (!isPlainObject(argumentsVal))
+        argumentsVal = { value: argumentsVal };
+    return record({
+        sourceKind: "tool_call",
+        timestamp: eventTimestamp(event, args.fallbackTimestamp),
+        projectPath: args.projectPath,
+        runtime: args.runtime,
+        sourceClass: args.sourceClass,
+        sourceProduct: args.sourceProduct,
+        activeRuntime: args.activeRuntime,
+        sourceParts: [resolvePath(args.sourcePath), args.index, "tool", toolName],
+        sessionId: args.sessionId,
+        data: { tool_name: toolName, arguments: argumentsVal }, // cast: parsed JSON IO boundary
+    });
+}
+export function toolCallRecord(args) {
+    return toolCallRecordFromItem({ ...args, item: payloadItem(args.event) });
+}
+export { isPlainObject, rglob, isDir, isFilePath };

package/dist/analytics/extractCorpus/corpus.js

@@ -0,0 +1,167 @@
+import fs from "node:fs";
+import { ADAPTER_VERSION, FAMILIES, isoNow, runtimeStatus, discoverRuntimeStore, COPILOT_SPARSE_REMEDIATION, } from "./core.js";
+import { isPlainObject } from "./core.js";
+import { extractInstructionDocuments, extractProjectConfigSignals } from "./filesystemSources.js";
+import { extractCodexSessions, extractClaudeProjectSessions } from "./jsonlSessions.js";
+import { extractOpencodeSessions, PermissionDeniedError } from "./sqliteSessions.js";
+import { extractCopilotSessions } from "./copilotSessions.js";
+import { extractCursorSessions, extractCursorAgentSessions, } from "./cursorSessions.js";
+import { resolvePath } from "../../core/paths.js";
+import { applyTruncationToStatus } from "./sqliteCaps.js";
+export class ExtractionNotImplementedError extends Error {
+}
+function extractRuntimeStore(runtime, storePath, errors, extractor, sqliteCaps, provenance = {
+    sourceProduct: runtime ?? "unknown",
+    sourceClass: "active_runtime",
+    activeRuntime: true,
+}) {
+    const discovery = discoverRuntimeStore(runtime, storePath, provenance);
+    if (discovery.status !== "available")
+        return [[], discovery];
+    const errorStart = errors.length;
+    const ctx = { sqliteCaps };
+    let records;
+    try {
+        records = extractor(storePath, errors, ctx);
+    }
+    catch (exc) {
+        const fc = (discovery.file_count ?? null); // cast: discovery payload IO boundary
+        if (exc instanceof ExtractionNotImplementedError) {
+            return [[], runtimeStatus(runtime, { status: "degraded", reason: "extractor_unimplemented", storePath, fileCount: fc, recordCount: 0, errorCount: 0, ...provenance })];
+        }
+        if (exc instanceof PermissionDeniedError) {
+            return [[], runtimeStatus(runtime, { status: "degraded", reason: "store_locked", storePath, fileCount: fc, ...provenance })];
+        }
+        return [[], runtimeStatus(runtime, { status: "degraded", reason: "store_unreadable", storePath, fileCount: fc, ...provenance })];
+    }
+    const fc = (discovery.file_count ?? null); // cast: discovery payload IO boundary
+    const errorCount = errors.length - errorStart;
+    if (errorCount) {
+        return [records, runtimeStatus(runtime, { status: "degraded", reason: "schema_divergent", storePath, fileCount: fc, recordCount: records.length, errorCount, ...provenance })];
+    }
+    if (records.length === 0) {
+        return [records, runtimeStatus(runtime, {
+                status: "sparse",
+                reason: "no_matching_records",
+                storePath,
+                fileCount: fc,
+                recordCount: 0,
+                remediationLabels: runtime === "copilot" ? [COPILOT_SPARSE_REMEDIATION] : null,
+                ...provenance,
+            })];
+    }
+    return [
+        records,
+        applyTruncationToStatus(runtimeStatus(runtime, { status: "ok", reason: "records_extracted", storePath, fileCount: fc, recordCount: records.length, errorCount: 0, ...provenance }), ctx.truncation),
+    ];
+}
+export function dedupeRecords(records) {
+    const byId = new Map();
+    for (const item of records)
+        byId.set(item.source_id, item);
+    const actorOrder = (item) => {
+        const actor = isPlainObject(item.data) ? item.data.actor : null;
+        return actor === "user" ? 0 : actor === "assistant" ? 1 : 2;
+    };
+    return Array.from(byId.values()).sort((a, b) => {
+        const at = (a.timestamp ?? "");
+        const bt = (b.timestamp ?? "");
+        if (at !== bt)
+            return at < bt ? -1 : 1;
+        const ak = (a.source_kind ?? "");
+        const bk = (b.source_kind ?? "");
+        if (ak !== bk)
+            return ak < bk ? -1 : 1;
+        const ao = actorOrder(a);
+        const bo = actorOrder(b);
+        if (ao !== bo)
+            return ao - bo;
+        const ai = (a.source_id ?? "");
+        const bi = (b.source_id ?? "");
+        return ai < bi ? -1 : ai > bi ? 1 : 0;
+    });
+}
+export function buildMetadata(records, errors, runtimeStatuses, coverage) {
+    const counts = new Map();
+    for (const item of records) {
+        const sk = item.source_kind;
+        if (FAMILIES.includes(sk))
+            counts.set(sk, (counts.get(sk) ?? 0) + 1);
+    }
+    const families = {};
+    for (const family of FAMILIES) {
+        const count = counts.get(family) ?? 0;
+        families[family] = { count, status: count ? "ok" : "missing" };
+        if (count === 0)
+            families[family].error = "no records extracted for this family";
+    }
+    const runtimes = Array.from(new Set(records.filter((item) => item.active_runtime === true && item.runtime).map((item) => String(item.runtime)))).sort();
+    const sourceClasses = Array.from(new Set(records.map((item) => String(item.source_class)))).sort();
+    const sourceProducts = Array.from(new Set(records.map((item) => String(item.source_product)))).sort();
+    const coverageFields = coverage ?? {
+        available_runtimes: [],
+        selected_runtimes: [],
+        available_but_not_selected: [],
+    };
+    return {
+        extracted_at: isoNow(),
+        runtimes,
+        adapter_version: ADAPTER_VERSION,
+        families,
+        runtime_statuses: runtimeStatuses,
+        available_runtimes: coverageFields.available_runtimes,
+        selected_runtimes: coverageFields.selected_runtimes,
+        available_but_not_selected: coverageFields.available_but_not_selected,
+        total_records: records.length,
+        errors,
+        source_classes: sourceClasses,
+        source_products: sourceProducts,
+    };
+}
+export function buildCorpus(opts) {
+    const errors = [];
+    const normalizedRoots = [];
+    for (const root of opts.projectRoots) {
+        if (fs.existsSync(root))
+            normalizedRoots.push(resolvePath(root));
+        else
+            errors.push(`${root}: project root does not exist`);
+    }
+    const records = [];
+    records.push(...extractInstructionDocuments(normalizedRoots, errors));
+    records.push(...extractProjectConfigSignals(normalizedRoots, errors));
+    const runtimeStatuses = [];
+    const runtimes = [
+        ["codex", "codex", "active_runtime", true, opts.codexSessionsDir, extractCodexSessions, undefined],
+        ["cursor", "cursor", "active_runtime", true, opts.cursorProjectsDir ?? null, (sp, err) => extractCursorSessions(sp, err, normalizedRoots), undefined],
+        [
+            "cursor",
+            "cursor-agent",
+            "active_runtime",
+            true,
+            opts.cursorChatsDir ?? null,
+            (sp, err) => extractCursorAgentSessions(sp, err, normalizedRoots, opts.cursorProjectsDir ?? null),
+            "store.db",
+        ],
+        ["opencode", "opencode", "active_runtime", true, opts.opencodeConversationsDir ?? null, extractOpencodeSessions, undefined],
+        ["copilot", "github-copilot", "active_runtime", true, opts.copilotConversationsDir ?? null, extractCopilotSessions, undefined],
+    ];
+    if (opts.claudeProjectsDir !== null) {
+        runtimes.push([
+            null,
+            "claude-code",
+            "historical_import",
+            false,
+            opts.claudeProjectsDir,
+            extractClaudeProjectSessions,
+            "*.jsonl",
+        ]);
+    }
+    for (const [runtime, sourceProduct, sourceClass, activeRuntime, storePath, extractor, pattern] of runtimes) {
+        const [runtimeRecords, status] = extractRuntimeStore(runtime, storePath, errors, extractor, opts.sqliteCaps, { sourceProduct, sourceClass, activeRuntime, pattern });
+        records.push(...runtimeRecords);
+        runtimeStatuses.push(status);
+    }
+    const deduped = dedupeRecords(records);
+    return { metadata: buildMetadata(deduped, errors, runtimeStatuses, opts.coverage), records: deduped };
+}