@specship/specship 0.11.0

package/dist/server/ingest/impact-query.js

@@ -0,0 +1,343 @@
+/**
+ * computeSpecshipImpact — aggregate SpecShip token-impact metrics from
+ * claude_tool_calls + claude_sessions.
+ *
+ * Pure function of (db, opts) — no Fastify dependency, unit-testable.
+ * Called by GET /api/claude/specship-impact in routes/claude.ts.
+ *
+ * Algorithm (SQL fetch + JS reduction — no GROUP BY prompt_id/file SQL):
+ *   1. Fetch scoped rows joining claude_tool_calls ↔ claude_sessions.
+ *   2. Spend: ceil(Σ result_length / CHARS_PER_TOKEN) for is_specship=1.
+ *   3. Saved: per-prompt dedup — union displaced file paths across a prompt's
+ *      resolved calls, sum each distinct file's size ONCE, subtract the
+ *      prompt's specship spend chars, floor at 0.
+ *   4. Overhead: ceil(SPECSHIP_TOOLDEF_CHARS / CHARS_PER_TOKEN) ×
+ *      distinct session count with ≥1 specship call.
+ *   5. Cost: price spend/saved chars at input-token rate, grouped by model.
+ */
+import { normalizeModelId, resolvePricing } from './pricing.js';
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Characters treated as one token (rough average for Claude). */
+const CHARS_PER_TOKEN = 4;
+/**
+ * JSON.stringify length of the SpecShip MCP tool definitions array.
+ * Measured: JSON.stringify(tools.filter(t => t.name.startsWith('specship_'))).length
+ * at 2026-06-24 — 12 tools in the registry at that time.
+ * See src/mcp/tools.ts; recheck after adding/removing tools.
+ */
+const SPECSHIP_TOOLDEF_CHARS = 10047;
+/** Overhead tokens charged per session (tool-definition context cost). */
+const OVERHEAD_TOKENS_PER_SESSION = Math.ceil(SPECSHIP_TOOLDEF_CHARS / CHARS_PER_TOKEN);
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+function parseDisplacedFiles(raw) {
+    if (!raw)
+        return [];
+    try {
+        const parsed = JSON.parse(raw);
+        if (!Array.isArray(parsed))
+            return [];
+        return parsed;
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Load pricing rows from the DB (returns [] if the table is empty or absent).
+ */
+function loadPricing(db) {
+    try {
+        return db
+            .prepare('SELECT model, input_per_mtok, output_per_mtok, cache_creation_per_mtok, cache_read_per_mtok FROM claude_pricing')
+            .all();
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Price a char count (already summed) as input tokens at the input rate for
+ * the given model. Returns 0 when model is unknown or pricing is unavailable.
+ */
+function priceChars(chars, model, pricingRows) {
+    if (!model || pricingRows.length === 0)
+        return 0;
+    const pricing = resolvePricing(model, pricingRows);
+    if (!pricing)
+        return 0;
+    const tokens = chars / CHARS_PER_TOKEN;
+    return (tokens * pricing.input_per_mtok) / 1_000_000;
+}
+/**
+ * Per-prompt dedup logic. Given the rows for ONE prompt:
+ *   - union displaced file paths from resolved rows (same file counted once)
+ *   - promptSavedChars = max(0, unionSize - promptSpendChars)
+ *
+ * Returns { savedChars, spendChars }.
+ */
+function computePromptSaved(rows) {
+    const spendChars = rows
+        .filter(r => r.is_specship === 1)
+        .reduce((s, r) => s + (r.result_length ?? 0), 0);
+    // Union displaced files across ALL resolved calls in this prompt.
+    const fileMap = new Map(); // path → size
+    for (const row of rows) {
+        if (row.is_specship !== 1 || row.resolution !== 'resolved')
+            continue;
+        for (const [path, size] of parseDisplacedFiles(row.displaced_files)) {
+            if (!fileMap.has(path)) {
+                fileMap.set(path, size);
+            }
+        }
+    }
+    const readEquivChars = Array.from(fileMap.values()).reduce((s, n) => s + n, 0);
+    const savedChars = Math.max(0, readEquivChars - spendChars);
+    return { savedChars, spendChars };
+}
+// ---------------------------------------------------------------------------
+// Main export
+// ---------------------------------------------------------------------------
+export function computeSpecshipImpact(db, opts) {
+    const { since, project, sessionId } = opts;
+    // 1. Fetch scoped rows.
+    const queryParts = [
+        `SELECT tc.prompt_id, tc.tool_name, tc.is_specship, tc.result_length,
+            tc.displaced_files, tc.resolution, tc.ts,
+            s.project_path, s.last_model
+     FROM claude_tool_calls tc
+     JOIN claude_sessions s ON s.id = tc.session_id
+     WHERE tc.ts >= ?`,
+    ];
+    const params = [since];
+    if (project) {
+        queryParts.push('AND s.project_path = ?');
+        params.push(project);
+    }
+    if (sessionId) {
+        queryParts.push('AND tc.session_id = ?');
+        params.push(sessionId);
+    }
+    const rows = db.prepare(queryParts.join(' ')).all(...params);
+    if (rows.length === 0) {
+        return {
+            spendTokens: 0,
+            spendCostUsd: 0,
+            savedTokens: 0,
+            savedCostUsd: 0,
+            overheadTokens: 0,
+            netTokens: 0,
+            netCostUsd: 0,
+            unresolvedCalls: 0,
+            totalSpecshipCalls: 0,
+            byTool: [],
+            byProject: [],
+            trend: [],
+        };
+    }
+    const pricingRows = loadPricing(db);
+    // ---------------------------------------------------------------------------
+    // 2. Group rows by prompt_id for per-prompt dedup.
+    // ---------------------------------------------------------------------------
+    const byPrompt = new Map();
+    for (const row of rows) {
+        const key = row.prompt_id ?? `__no_prompt_${row.ts}`;
+        const arr = byPrompt.get(key);
+        if (arr)
+            arr.push(row);
+        else
+            byPrompt.set(key, [row]);
+    }
+    // ---------------------------------------------------------------------------
+    // 3. Global spend / saved (with per-prompt dedup) + cost by model.
+    // ---------------------------------------------------------------------------
+    let totalSpendChars = 0;
+    let totalSavedChars = 0;
+    let totalSpecshipCalls = 0;
+    let unresolvedCalls = 0;
+    // For cost we need to track chars by model.
+    const spendCharsByModel = new Map();
+    const savedCharsByModel = new Map();
+    // Per-tool accumulators: spend is exact; saved is a per-tool APPROXIMATION.
+    // Exact per-tool saved attribution is ambiguous when two tools in the same
+    // prompt share a file. We approximate: a tool's saved chars = its own
+    // resolved displaced files (deduped within the tool, not globally within
+    // the prompt). This overestimates when tools share files but keeps
+    // individual tool savings intuitive. Spend is always exact.
+    const toolSpendChars = new Map();
+    const toolSavedChars = new Map();
+    const toolCalls = new Map();
+    // Per-project (only when no project filter).
+    const projectSpendChars = new Map();
+    const projectSavedChars = new Map();
+    // Trend: daily buckets.
+    const dayMs = 24 * 60 * 60 * 1000;
+    const trendSpend = new Map(); // dayBucket → chars
+    const trendSaved = new Map();
+    // Pass 1: global spend/saved via per-prompt dedup.
+    for (const [, promptRows] of byPrompt) {
+        const { savedChars, spendChars } = computePromptSaved(promptRows);
+        totalSpendChars += spendChars;
+        totalSavedChars += savedChars;
+        // Attribute spend/saved chars to the session's model.
+        // All rows in a prompt share the same session (same last_model).
+        const model = promptRows[0]?.last_model ?? null;
+        const normModel = model ? normalizeModelId(model) : null;
+        if (normModel) {
+            spendCharsByModel.set(normModel, (spendCharsByModel.get(normModel) ?? 0) + spendChars);
+            savedCharsByModel.set(normModel, (savedCharsByModel.get(normModel) ?? 0) + savedChars);
+        }
+        // Track per-day trend for this prompt's is_specship rows.
+        for (const row of promptRows) {
+            if (row.is_specship !== 1)
+                continue;
+            totalSpecshipCalls++;
+            if (row.resolution === 'unresolved')
+                unresolvedCalls++;
+            // Session tracking (for overhead).
+            // We need the session_id — it's not in our fetch. Derive from project+model
+            // won't work. We'll collect it via a separate map after this loop using rows.
+        }
+    }
+    // We need session_ids for overhead. Since we joined sessions but didn't select
+    // session_id, do a lightweight distinct-session count via a separate query.
+    const sessionCountQParts = [
+        `SELECT COUNT(DISTINCT tc.session_id) as cnt
+     FROM claude_tool_calls tc
+     JOIN claude_sessions s ON s.id = tc.session_id
+     WHERE tc.ts >= ? AND tc.is_specship = 1`,
+    ];
+    const sessionCountParams = [since];
+    if (project) {
+        sessionCountQParts.push('AND s.project_path = ?');
+        sessionCountParams.push(project);
+    }
+    if (sessionId) {
+        sessionCountQParts.push('AND tc.session_id = ?');
+        sessionCountParams.push(sessionId);
+    }
+    const sessionCountRow = db
+        .prepare(sessionCountQParts.join(' '))
+        .get(...sessionCountParams);
+    const specshipSessionCount = sessionCountRow?.cnt ?? 0;
+    // ---------------------------------------------------------------------------
+    // Pass 2: per-tool and trend (row-level, NOT requiring per-prompt dedup).
+    // ---------------------------------------------------------------------------
+    // Per-tool saved approximation: for each is_specship row with resolution=resolved,
+    // compute its own per-call "saved" as max(0, sum(displaced_file_sizes) - result_length).
+    // This is approximate because it ignores cross-call file sharing within a prompt,
+    // but it gives an intuitive per-tool estimate and spend is always exact.
+    for (const row of rows) {
+        const dayBucket = Math.floor(row.ts / dayMs) * dayMs;
+        if (row.is_specship !== 1)
+            continue;
+        const toolName = row.tool_name ?? 'unknown';
+        toolCalls.set(toolName, (toolCalls.get(toolName) ?? 0) + 1);
+        const spend = row.result_length ?? 0;
+        toolSpendChars.set(toolName, (toolSpendChars.get(toolName) ?? 0) + spend);
+        // Per-tool saved approximation: own displaced files minus own spend, ≥0.
+        const displaced = parseDisplacedFiles(row.displaced_files);
+        if (row.resolution === 'resolved' && displaced.length > 0) {
+            const fileUnion = new Map();
+            for (const [p, sz] of displaced)
+                if (!fileUnion.has(p))
+                    fileUnion.set(p, sz);
+            const readEquiv = Array.from(fileUnion.values()).reduce((s, n) => s + n, 0);
+            const toolSaved = Math.max(0, readEquiv - spend);
+            toolSavedChars.set(toolName, (toolSavedChars.get(toolName) ?? 0) + toolSaved);
+        }
+        // Trend.
+        trendSpend.set(dayBucket, (trendSpend.get(dayBucket) ?? 0) + spend);
+    }
+    // Per-prompt saved for trend and byProject (both need per-prompt dedup).
+    for (const [, promptRows] of byPrompt) {
+        const { savedChars } = computePromptSaved(promptRows);
+        const project_path = promptRows[0]?.project_path ?? '';
+        const dayBucket = Math.floor((promptRows[0]?.ts ?? 0) / dayMs) * dayMs;
+        // Trend saved.
+        if (savedChars > 0) {
+            trendSaved.set(dayBucket, (trendSaved.get(dayBucket) ?? 0) + savedChars);
+        }
+        // byProject accumulators.
+        if (!project) {
+            const spendChars = promptRows
+                .filter(r => r.is_specship === 1)
+                .reduce((s, r) => s + (r.result_length ?? 0), 0);
+            projectSpendChars.set(project_path, (projectSpendChars.get(project_path) ?? 0) + spendChars);
+            projectSavedChars.set(project_path, (projectSavedChars.get(project_path) ?? 0) + savedChars);
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // 4. Compute tokens + cost.
+    // ---------------------------------------------------------------------------
+    const spendTokens = Math.ceil(totalSpendChars / CHARS_PER_TOKEN);
+    const savedTokens = Math.ceil(totalSavedChars / CHARS_PER_TOKEN);
+    const overheadTokens = OVERHEAD_TOKENS_PER_SESSION * specshipSessionCount;
+    let spendCostUsd = 0;
+    let savedCostUsd = 0;
+    for (const [model, chars] of spendCharsByModel) {
+        spendCostUsd += priceChars(chars, model, pricingRows);
+    }
+    for (const [model, chars] of savedCharsByModel) {
+        savedCostUsd += priceChars(chars, model, pricingRows);
+    }
+    const netTokens = savedTokens - spendTokens - overheadTokens;
+    const netCostUsd = savedCostUsd - spendCostUsd;
+    // ---------------------------------------------------------------------------
+    // 5. byTool.
+    // ---------------------------------------------------------------------------
+    const byTool = Array.from(toolCalls.entries()).map(([tool, calls]) => ({
+        tool,
+        calls,
+        spendTokens: Math.ceil((toolSpendChars.get(tool) ?? 0) / CHARS_PER_TOKEN),
+        savedTokens: Math.ceil((toolSavedChars.get(tool) ?? 0) / CHARS_PER_TOKEN),
+    })).sort((a, b) => b.calls - a.calls);
+    // ---------------------------------------------------------------------------
+    // 6. byProject (only when no project filter).
+    // ---------------------------------------------------------------------------
+    // NOTE: per-project netTokens = saved − spend and intentionally does NOT
+    // subtract the tool-definition overhead. Overhead is a per-session constant
+    // and isn't attributed to individual projects, so the byProject rows will
+    // not sum to the headline netTokens (which does include overhead). The page
+    // methodology note discloses this.
+    const byProject = project
+        ? []
+        : Array.from(projectSpendChars.keys()).map((proj) => {
+            const spend = Math.ceil((projectSpendChars.get(proj) ?? 0) / CHARS_PER_TOKEN);
+            const saved = Math.ceil((projectSavedChars.get(proj) ?? 0) / CHARS_PER_TOKEN);
+            return {
+                project: proj,
+                spendTokens: spend,
+                savedTokens: saved,
+                netTokens: saved - spend, // overhead not allocated per-project — see note above
+            };
+        }).sort((a, b) => b.spendTokens - a.spendTokens);
+    // ---------------------------------------------------------------------------
+    // 7. trend.
+    // ---------------------------------------------------------------------------
+    const allDayBuckets = new Set([...trendSpend.keys(), ...trendSaved.keys()]);
+    const trend = Array.from(allDayBuckets)
+        .sort((a, b) => a - b)
+        .map((ts) => ({
+        ts,
+        spendTokens: Math.ceil((trendSpend.get(ts) ?? 0) / CHARS_PER_TOKEN),
+        savedTokens: Math.ceil((trendSaved.get(ts) ?? 0) / CHARS_PER_TOKEN),
+    }));
+    return {
+        spendTokens,
+        spendCostUsd,
+        savedTokens,
+        savedCostUsd,
+        overheadTokens,
+        netTokens,
+        netCostUsd,
+        unresolvedCalls,
+        totalSpecshipCalls,
+        byTool,
+        byProject,
+        trend,
+    };
+}

package/dist/server/ingest/index.js

@@ -0,0 +1,19 @@
+/**
+ * Server-local JSONL ingest. Lives inside @selvakumaresra/specship-server
+ * so the HTTP API has a single deploy unit — no separate package, no
+ * dependency wiring.
+ *
+ * Walks Claude Code's JSONL transcripts under `~/.claude/projects/` and
+ * writes parsed entries into specship's SQLite database so the analytics
+ * surfaces (Sessions, Heatmap, Costs, Compare, Tips) run sub-millisecond
+ * queries instead of re-parsing megabytes of JSONL per page load.
+ *
+ * Two entry points:
+ *   - `ingestAll(db)`     — one-shot pass, useful for CI / scripts.
+ *   - `startWatcher(db)`  — live tail; runs ingestAll once + on each FS event.
+ */
+export { ingestAll, listTranscriptFiles, decodeProjectSlug, primaryProjectMatcher } from './ingestor.js';
+export { startWatcher } from './watcher.js';
+export { resolvePricing, computeCost, normalizeModelId } from './pricing.js';
+export { parseLine, summarizeToolInput, extractUserPrompt, toolResultLength } from './parser.js';
+export { backfillDisplaced } from './impact-backfill.js';