@aixle/insights 0.1.0

package/dist/readers/claude.d.ts

@@ -0,0 +1,83 @@
+import type { IngestPayload } from "../lib/index.js";
+import { type PricingTable } from "../pricing.js";
+import { type RiskLevel } from "../risk-scanner.js";
+/** True when prompt text alone matches known local-command injection markers. */
+export declare function isClaudeLocalCommandNoisePrompt(promptText: string): boolean;
+/**
+ * Returns true for transcript turns that are Claude Code local-command noise
+ * (caveat, /exit, stdout) with no model usage — safe to omit from ingest.
+ */
+export declare function isClaudeNoiseTranscriptTurn(turn: ClaudeTranscriptTurn): boolean;
+/**
+ * True for turns that look mid-flight: a user prompt is present but the
+ * assistant hasn't responded yet (no text, no model, no output tokens).
+ *
+ * Skipping these is safe: the turn is NOT checkpointed (we never push it to
+ * finalizedTurns), so the next sync cycle re-parses the JSONL and persists
+ * the now-complete turn once the assistant has finished writing.
+ *
+ * Genuinely interrupted turns where the assistant never writes anything are
+ * also dropped by this guard — that's correct. The classic example is the
+ * "[Request interrupted by user for tool use]" placeholder Claude Code
+ * injects after a tool-use interruption; it's a system marker, not real
+ * user activity, so losing it produces cleaner Events table rows.
+ */
+export declare function isIncompleteTranscriptTurn(turn: ClaudeTranscriptTurn): boolean;
+export interface ClaudeTranscriptTurn {
+    sessionId: string;
+    turnId: string;
+    promptId?: string;
+    filePath: string;
+    fileSize: number;
+    cwd?: string;
+    model: string | null;
+    tokensIn: number;
+    tokensOut: number;
+    cacheWriteTokens: number;
+    cacheReadTokens: number;
+    occurredAt: string;
+    promptText: string;
+    assistantText: string;
+    riskLevel: RiskLevel;
+    riskScore: number;
+    riskCategories: string[];
+}
+/** Payload shape expected by the db90 ingest API. */
+export interface Db90Payload extends IngestPayload {
+    tool_name: "claude_code";
+    event_type: "chat";
+    model?: string;
+    tokens_in?: number;
+    tokens_out?: number;
+    tokens_total?: number;
+    cost_usd: number | null;
+    occurred_at: string;
+    project_id?: string;
+    metadata: {
+        session_id: string;
+        claude_session_id: string;
+        transcript_source: "claude_jsonl";
+        model: string | null;
+        base_input_tokens: number;
+        output_tokens: number;
+        cache_write_tokens: number;
+        cache_read_tokens: number;
+        risk_level: RiskLevel;
+        risk_categories: string[];
+        risk_score: number;
+        prompt_text?: string;
+        assistant_text?: string;
+        scannable: true;
+    };
+}
+/** Options for mapTranscriptTurn. */
+export interface ToDb90PayloadOptions {
+    projectId?: string | null;
+    pricing?: PricingTable;
+}
+/** Finds all *.jsonl transcript files across both Claude project directory roots. */
+export declare function findTranscriptFiles(baseDirs?: string[]): string[];
+/** Streams a JSONL file and splits Claude transcripts into individual turns. */
+export declare function parseTranscriptFile(filePath: string, verbose?: boolean): Promise<ClaudeTranscriptTurn[]>;
+/** Converts a Claude transcript turn to a db90 ingest payload. */
+export declare function mapTranscriptTurn(turn: ClaudeTranscriptTurn, options?: ToDb90PayloadOptions): Db90Payload;

package/dist/readers/claude.js

@@ -0,0 +1,317 @@
+import { createReadStream, statSync } from "node:fs";
+import { finished } from "node:stream/promises";
+import { createInterface } from "node:readline";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { glob } from "glob";
+import { calculateCost } from "../pricing.js";
+import { scanText } from "../risk-scanner.js";
+/** Prompt substrings emitted for local IDE commands — not real user prompts. */
+const LOCAL_COMMAND_NOISE_PROMPT_PATTERNS = [
+    /<local-command-caveat\b/i,
+    /<local-command-stdout\b/i,
+    /<command-name>/i,
+];
+function hasZeroTokenUsage(turn) {
+    return (turn.tokensIn === 0 &&
+        turn.tokensOut === 0 &&
+        turn.cacheWriteTokens === 0 &&
+        turn.cacheReadTokens === 0);
+}
+/** True when prompt text alone matches known local-command injection markers. */
+export function isClaudeLocalCommandNoisePrompt(promptText) {
+    const prompt = promptText.trim();
+    if (!prompt)
+        return false;
+    return LOCAL_COMMAND_NOISE_PROMPT_PATTERNS.some((pattern) => pattern.test(prompt));
+}
+/**
+ * Returns true for transcript turns that are Claude Code local-command noise
+ * (caveat, /exit, stdout) with no model usage — safe to omit from ingest.
+ */
+export function isClaudeNoiseTranscriptTurn(turn) {
+    if (!hasZeroTokenUsage(turn))
+        return false;
+    if (turn.model !== null)
+        return false;
+    if (turn.assistantText.trim().length > 0)
+        return false;
+    return isClaudeLocalCommandNoisePrompt(turn.promptText);
+}
+/**
+ * True for turns that look mid-flight: a user prompt is present but the
+ * assistant hasn't responded yet (no text, no model, no output tokens).
+ *
+ * Skipping these is safe: the turn is NOT checkpointed (we never push it to
+ * finalizedTurns), so the next sync cycle re-parses the JSONL and persists
+ * the now-complete turn once the assistant has finished writing.
+ *
+ * Genuinely interrupted turns where the assistant never writes anything are
+ * also dropped by this guard — that's correct. The classic example is the
+ * "[Request interrupted by user for tool use]" placeholder Claude Code
+ * injects after a tool-use interruption; it's a system marker, not real
+ * user activity, so losing it produces cleaner Events table rows.
+ */
+export function isIncompleteTranscriptTurn(turn) {
+    return (turn.model === null &&
+        turn.assistantText.trim().length === 0 &&
+        turn.tokensOut === 0);
+}
+/** Returns the two candidate Claude project directories (v1.0.30+ and legacy). */
+function claudeProjectDirs() {
+    const home = homedir();
+    return [
+        join(home, ".config", "claude", "projects"),
+        join(home, ".claude", "projects"),
+    ];
+}
+/** Finds all *.jsonl transcript files across both Claude project directory roots. */
+export function findTranscriptFiles(baseDirs) {
+    const dirs = baseDirs ?? claudeProjectDirs();
+    const files = [];
+    for (const dir of dirs) {
+        try {
+            const matches = glob.sync("**/*.jsonl", { cwd: dir, absolute: true });
+            files.push(...matches);
+        }
+        catch {
+            // directory does not exist — skip
+        }
+    }
+    // De-duplicate in case both paths resolve to the same files (symlinks, etc.)
+    return [...new Set(files)];
+}
+/** Extracts text strings from a content field (block array or plain string). */
+function extractContentText(content) {
+    if (typeof content === "string")
+        return [content];
+    if (!Array.isArray(content))
+        return [];
+    return content.flatMap((block) => {
+        if (typeof block !== "object" || block === null)
+            return [];
+        const { type, text } = block;
+        return type === "text" && typeof text === "string" ? [text] : [];
+    });
+}
+function hasTextContent(content) {
+    if (typeof content === "string")
+        return content.trim().length > 0;
+    if (!Array.isArray(content))
+        return false;
+    return content.some((block) => {
+        if (typeof block !== "object" || block === null)
+            return false;
+        const { type, text } = block;
+        return type === "text" && typeof text === "string" && text.trim().length > 0;
+    });
+}
+function newTurn(sessionId, turnIndex, filePath, fileSize, occurredAt, promptId) {
+    return {
+        sessionId,
+        turnId: `${sessionId}:${turnIndex}`,
+        promptId,
+        filePath,
+        fileSize,
+        cwd: undefined,
+        model: null,
+        tokensIn: 0,
+        tokensOut: 0,
+        cacheWriteTokens: 0,
+        cacheReadTokens: 0,
+        occurredAt,
+        promptText: "",
+        assistantText: "",
+        riskLevel: "low",
+        riskScore: 0,
+        riskCategories: [],
+        persisted: false,
+    };
+}
+function appendText(existing, addition) {
+    if (!addition.trim())
+        return existing;
+    return existing ? `${existing}\n\n${addition}` : addition;
+}
+function enrichTurnRisk(turn) {
+    if (!turn.promptText.trim())
+        return;
+    const result = scanText(turn.promptText);
+    turn.riskLevel = result.risk_level;
+    turn.riskScore = result.risk_score;
+    turn.riskCategories = result.risk_categories;
+}
+/** Streams a JSONL file and splits Claude transcripts into individual turns. */
+export async function parseTranscriptFile(filePath, verbose = false) {
+    const turns = [];
+    let fileSize = 0;
+    try {
+        fileSize = statSync(filePath).size;
+    }
+    catch {
+        return turns;
+    }
+    const stream = createReadStream(filePath, { encoding: "utf-8" });
+    const rl = createInterface({
+        input: stream,
+        crlfDelay: Infinity,
+    });
+    let lineNumber = 0;
+    let currentTurn = null;
+    let currentTurnIndex = 0;
+    const turnsByPromptId = new Map();
+    const finalizedTurns = [];
+    const flushCurrentTurn = () => {
+        if (!currentTurn)
+            return;
+        if (!currentTurn.persisted) {
+            enrichTurnRisk(currentTurn);
+            const hasContent = currentTurn.promptText.trim().length > 0 || currentTurn.assistantText.trim().length > 0;
+            if (hasContent &&
+                !isClaudeNoiseTranscriptTurn(currentTurn) &&
+                !isIncompleteTranscriptTurn(currentTurn)) {
+                currentTurn.persisted = true;
+                finalizedTurns.push(currentTurn);
+            }
+        }
+        currentTurn = null;
+    };
+    try {
+        for await (const line of rl) {
+            lineNumber++;
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            let entry;
+            try {
+                entry = JSON.parse(trimmed);
+            }
+            catch {
+                if (verbose) {
+                    console.warn(`[warn] ${filePath}:${lineNumber} — invalid JSON, skipping`);
+                }
+                continue;
+            }
+            const sessionId = entry.sessionId;
+            const promptId = entry.promptId;
+            const timestamp = entry.timestamp ?? new Date().toISOString();
+            const cwd = typeof entry.cwd === "string" && entry.cwd.trim().length > 0 ? entry.cwd : undefined;
+            if (entry.type === "user") {
+                if (!sessionId || !entry.message?.content)
+                    continue;
+                const text = extractContentText(entry.message.content).join("\n\n").trim();
+                // Claude emits tool_result-only user entries after assistant tool_use.
+                // Those are part of the active turn, not a new user prompt.
+                if (hasTextContent(entry.message.content)) {
+                    if (entry.isMeta === true || isClaudeLocalCommandNoisePrompt(text)) {
+                        if (verbose) {
+                            console.log("[verbose] Skipping Claude local-command/meta user line");
+                        }
+                        continue;
+                    }
+                    flushCurrentTurn();
+                    currentTurnIndex += 1;
+                    currentTurn = newTurn(sessionId, currentTurnIndex, filePath, fileSize, timestamp, promptId);
+                    currentTurn.cwd = cwd ?? currentTurn.cwd;
+                    currentTurn.promptText = appendText(currentTurn.promptText, text);
+                    currentTurn.occurredAt = timestamp;
+                    if (promptId)
+                        turnsByPromptId.set(promptId, currentTurn);
+                }
+                else if (promptId && turnsByPromptId.has(promptId)) {
+                    currentTurn = turnsByPromptId.get(promptId) ?? null;
+                    if (currentTurn) {
+                        currentTurn.cwd = cwd ?? currentTurn.cwd;
+                        currentTurn.occurredAt = timestamp > currentTurn.occurredAt ? timestamp : currentTurn.occurredAt;
+                    }
+                }
+                else if (currentTurn && currentTurn.sessionId === sessionId) {
+                    currentTurn.cwd = cwd ?? currentTurn.cwd;
+                    currentTurn.occurredAt = timestamp > currentTurn.occurredAt ? timestamp : currentTurn.occurredAt;
+                }
+                continue;
+            }
+            if (entry.type === "assistant") {
+                if (!sessionId || !entry.message)
+                    continue;
+                if (!currentTurn || currentTurn.sessionId !== sessionId) {
+                    currentTurnIndex += 1;
+                    currentTurn = newTurn(sessionId, currentTurnIndex, filePath, fileSize, timestamp);
+                }
+                currentTurn.cwd = cwd ?? currentTurn.cwd;
+                const usage = entry.message.usage;
+                if (usage) {
+                    currentTurn.tokensIn +=
+                        (usage.input_tokens ?? 0) +
+                            (usage.cache_creation_input_tokens ?? 0) +
+                            (usage.cache_read_input_tokens ?? 0);
+                    currentTurn.tokensOut += usage.output_tokens ?? 0;
+                    currentTurn.cacheWriteTokens += usage.cache_creation_input_tokens ?? 0;
+                    currentTurn.cacheReadTokens += usage.cache_read_input_tokens ?? 0;
+                }
+                else if (verbose) {
+                    console.warn(`[warn] ${filePath}:${lineNumber} — assistant message has no usage`);
+                }
+                if (entry.message.model)
+                    currentTurn.model = entry.message.model;
+                currentTurn.occurredAt = timestamp > currentTurn.occurredAt ? timestamp : currentTurn.occurredAt;
+                const text = extractContentText(entry.message.content).join("\n\n").trim();
+                currentTurn.assistantText = appendText(currentTurn.assistantText, text);
+            }
+        }
+    }
+    catch (err) {
+        if (verbose) {
+            const message = err instanceof Error ? err.message : String(err);
+            console.warn(`[warn] ${filePath} — stream error, skipping file: ${message}`);
+        }
+        rl.close();
+        stream.destroy();
+        await finished(stream).catch(() => undefined);
+        return turns;
+    }
+    flushCurrentTurn();
+    return finalizedTurns.map(({ persisted: _persisted, ...turn }) => turn);
+}
+/** Converts a Claude transcript turn to a db90 ingest payload. */
+export function mapTranscriptTurn(turn, options) {
+    const { projectId, pricing } = options ?? {};
+    const baseInputTokens = Math.max(0, turn.tokensIn - turn.cacheWriteTokens - turn.cacheReadTokens);
+    const cost = pricing
+        ? calculateCost(turn.model, baseInputTokens, turn.tokensOut, turn.cacheWriteTokens, turn.cacheReadTokens, pricing)
+        : null;
+    const payload = {
+        tool_name: "claude_code",
+        event_type: "chat",
+        cost_usd: cost,
+        occurred_at: turn.occurredAt,
+        metadata: {
+            session_id: turn.turnId,
+            claude_session_id: turn.sessionId,
+            transcript_source: "claude_jsonl",
+            model: turn.model,
+            base_input_tokens: baseInputTokens,
+            output_tokens: turn.tokensOut,
+            cache_write_tokens: turn.cacheWriteTokens,
+            cache_read_tokens: turn.cacheReadTokens,
+            risk_level: turn.riskLevel,
+            risk_categories: turn.riskCategories,
+            risk_score: turn.riskScore,
+            prompt_text: turn.promptText || undefined,
+            assistant_text: turn.assistantText || undefined,
+            scannable: true,
+        },
+    };
+    if (turn.model)
+        payload.model = turn.model;
+    if (turn.tokensIn > 0)
+        payload.tokens_in = turn.tokensIn;
+    if (turn.tokensOut > 0)
+        payload.tokens_out = turn.tokensOut;
+    if (turn.tokensIn > 0 || turn.tokensOut > 0) {
+        payload.tokens_total = turn.tokensIn + turn.tokensOut;
+    }
+    if (projectId)
+        payload.project_id = projectId;
+    return payload;
+}

package/dist/readers/cursor.d.ts

@@ -0,0 +1,134 @@
+import type { IngestPayload } from "../lib/index.js";
+import { type RiskLevel } from "../risk-scanner.js";
+export declare function cursorUserDir(): string;
+/** Smoke-test better-sqlite3 against the global Cursor state DB (CUR-V02 / verify scripts). */
+export declare function probeCursorGlobalStateDb(verbose?: boolean): boolean;
+export declare function findCursorDbs(baseDir?: string): string[];
+export interface CursorRow {
+    requestId?: string | null;
+    timestamp?: number | string | null;
+    model?: string | null;
+    promptTokens?: number | null;
+    generatedTokens?: number | null;
+    type?: number | null;
+    sessionId?: string | null;
+    [key: string]: unknown;
+}
+export declare function readLegacyEvents(since: Date | null, baseDir?: string, verbose?: boolean): Array<{
+    row: CursorRow;
+    workspacePath: string;
+}>;
+export interface DailyStatsEntry {
+    date: string;
+    value: unknown;
+    dbPath: string;
+}
+export declare function isGlobalStateDbPath(dbPath: string): boolean;
+type WorkspaceScope = "global" | "workspace";
+export declare function dedupeDailyStatsEntries(entries: DailyStatsEntry[]): DailyStatsEntry[];
+export declare function findStateVscDbs(baseDir?: string): string[];
+export interface DailyStatsReadResult {
+    raw: DailyStatsEntry[];
+    deduped: DailyStatsEntry[];
+}
+export declare function readDailyStatsWithDedupe(since: Date | null, baseDir?: string, verbose?: boolean): DailyStatsReadResult;
+export declare function readDailyStats(since: Date | null, baseDir?: string, verbose?: boolean): DailyStatsEntry[];
+export interface RecentCommitSnapshot {
+    value: Record<string, unknown>;
+    dbPath: string;
+}
+export declare function readRecentCommitSnapshots(since: Date | null, baseDir?: string, verbose?: boolean): RecentCommitSnapshot[];
+export declare function readEvents(since: Date | null, baseDir?: string, verbose?: boolean): Array<{
+    row: CursorRow;
+    workspacePath: string;
+}>;
+interface CursorComposerHeader {
+    composerId: string;
+    name: string | null;
+    workspacePath: string | null;
+    lastUpdatedAt: string | null;
+}
+export interface CursorTranscriptTurn {
+    turnId: string;
+    sessionId: string;
+    filePath: string;
+    fileSize: number;
+    /** SHA-256 prefix (32 hex chars) of the JSONL content — preferred over fileSize for change detection. */
+    contentHash?: string;
+    workspacePath: string | null;
+    composerName: string | null;
+    occurredAt: string;
+    promptText: string;
+    assistantText: string;
+    tokensIn: number;
+    tokensOut: number;
+    riskLevel: RiskLevel;
+    riskScore: number;
+    riskCategories: string[];
+}
+export declare function findCursorTranscriptFiles(projectDirs?: string[]): string[];
+export declare function parseCursorTranscriptFile(filePath: string, composerHeaders: Map<string, CursorComposerHeader>, verbose?: boolean): Promise<CursorTranscriptTurn[]>;
+export declare function readCursorTranscriptSessions(cursorUserBaseDir?: string, transcriptProjectDirs?: string[], verbose?: boolean): Promise<CursorTranscriptTurn[]>;
+declare const LINE_COST_MODEL: "estimated_line_count";
+declare const TOKEN_COST_MODEL: "token_count";
+declare const TRANSCRIPT_COST_MODEL: "estimated_transcript_text";
+export declare const HOOK_COST_MODEL: "cursor_hook";
+type CursorLineCostModel = typeof LINE_COST_MODEL;
+type CursorTokenCostModel = typeof TOKEN_COST_MODEL;
+export type CursorHookCostModel = typeof HOOK_COST_MODEL;
+export interface PricingConfig {
+    tokens_per_line: number;
+    completion_output_per_mtok: number;
+    chat_input_per_mtok: number;
+    chat_output_per_mtok: number;
+}
+export declare const DEFAULT_CURSOR_PRICING: PricingConfig;
+export type Db90CursorPayloadMetadata = {
+    session_id?: string;
+    cursor_session_id: string | null;
+    workspace: string;
+    workspace_scope?: WorkspaceScope;
+    workspace_folder?: string;
+    cost_model: CursorLineCostModel | CursorTokenCostModel | typeof TRANSCRIPT_COST_MODEL | CursorHookCostModel;
+    scannable: boolean;
+    risk_level: RiskLevel | "none";
+    source?: "recent_commit";
+    transcript_source?: "agent_transcript";
+    composer_name?: string;
+    prompt_text?: string;
+    assistant_text?: string;
+    risk_categories?: string[];
+    risk_score?: number;
+    commit_hash?: string;
+    commit_message?: string;
+    repo_name?: string;
+    branch_name?: string;
+    ai_percentage?: number;
+    ingest_source?: "cursor_hook";
+    hook_event_name?: string;
+    generation_id?: string;
+    hook_tool_name?: string;
+    duration_ms?: number;
+};
+export interface CursorDb90Payload extends IngestPayload {
+    tool_name: "cursor";
+    event_type: "completion" | "chat" | "commit";
+    model: string;
+    tokens_in: number;
+    tokens_out: number;
+    cost_usd: number;
+    occurred_at: string;
+    project_id?: string;
+    metadata: Db90CursorPayloadMetadata;
+}
+export declare function toEpochMs(timestamp: number | string | null | undefined): number | null;
+export declare function mapDailyStats(entry: DailyStatsEntry, projectId?: string, pricing?: PricingConfig, model?: string): CursorDb90Payload[];
+/**
+ * Maps Cursor’s latest-commit snapshot (`aiCodeTracking.recentCommit`) to a single commit-classified event.
+ * Cursor only keeps one recent commit row (overwritten on each new commit).
+ * Line-cost math still follows the chat-style line proxy (`computeLineCost("chat", …)`); only `event_type` differs.
+ */
+export declare function mapRecentCommit(entry: RecentCommitSnapshot, projectId?: string, pricing?: PricingConfig, model?: string): CursorDb90Payload | null;
+export declare function mapEvent(row: CursorRow, workspacePath: string, projectId?: string, pricing?: PricingConfig): CursorDb90Payload | null;
+export declare function mapTranscriptTurn(turn: CursorTranscriptTurn, projectId?: string, pricing?: PricingConfig, model?: string): CursorDb90Payload;
+export {};