log-llm-config 1.5.2 → 1.5.8

package/dist/log_config_files/collection/opencode_token_usage_collector.js

@@ -0,0 +1,175 @@
+import { execFileSync } from 'node:child_process';
+import { existsSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import { resolveSqlite3Binary } from '../runtime/sqlite_binary.js';
+/**
+ * Token-usage / cost / session / version / identity aggregator for OpenCode.
+ *
+ * Unlike Claude/Copilot (JSONL transcripts), OpenCode stores everything in a SQLite DB at
+ * ~/.local/share/opencode/opencode.db. Each assistant row in the `message` table has a JSON
+ * `data` blob carrying `modelID`, `providerID`, `tokens` ({input,output,reasoning,cache:{read,write}})
+ * and `cost`; the `account` table carries the login `email`/`url`. We read those via the bundled
+ * `sqlite3` binary (same resolver the remediation path uses) and emit only the aggregated numbers
+ * (plus the login email) — never message content. The DB is opened read-only.
+ *
+ * Bounded by DB-file mtime recency so a stale install isn't re-read every hook run.
+ */
+const TOKEN_USAGE_RECENCY_DAYS = 30;
+const TOKEN_USAGE_FILE_TYPE = 'opencode_token_usage';
+// ASCII record / unit separators: never appear in JSON or emails, so they delimit SELECTed
+// rows/fields unambiguously. In SQL these are char(30) / char(31).
+const ROW_SEP = '\x1e';
+const FIELD_SEP = '\x1f';
+function emptyTotals() {
+    return {
+        input_tokens: 0,
+        output_tokens: 0,
+        cache_read_input_tokens: 0,
+        cache_creation_input_tokens: 0,
+    };
+}
+function num(value) {
+    return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+function asObject(value) {
+    return value && typeof value === 'object' ? value : {};
+}
+/** Run a read-only query and return raw stdout, or null when sqlite3/DB is unavailable. */
+function querySqlite(sqlite3, dbPath, sql) {
+    try {
+        return execFileSync(sqlite3, ['-readonly', '-noheader', dbPath], {
+            input: `.timeout 5000\n${sql}\n`,
+            encoding: 'utf8',
+            stdio: ['pipe', 'pipe', 'pipe'],
+        });
+    }
+    catch {
+        return null;
+    }
+}
+/** OpenCode CLI version from ~/.config/opencode/package.json (@opencode-ai/plugin pin), best-effort. */
+function readOpencodeVersion(home) {
+    for (const rel of ['.config/opencode/package.json', '.cache/opencode/package.json']) {
+        try {
+            const pkg = JSON.parse(readFileSync(join(home, rel), 'utf8'));
+            const deps = asObject(pkg.dependencies);
+            const v = deps['@opencode-ai/plugin'] ?? deps['opencode'] ?? pkg.version;
+            if (typeof v === 'string' && v)
+                return v.replace(/^[\^~]/, '');
+        }
+        catch {
+            /* missing/unparseable — try next */
+        }
+    }
+    return '';
+}
+/**
+ * Aggregate OpenCode token usage / cost / sessions / models from the SQLite DB.
+ * Returns a single-element array (one per-machine aggregate) or [] when there is no usable DB.
+ */
+function collectOpencodeTokenUsage(home = homedir()) {
+    const dbPath = join(home, '.local', 'share', 'opencode', 'opencode.db');
+    if (!existsSync(dbPath))
+        return [];
+    const cutoffMs = Date.now() - TOKEN_USAGE_RECENCY_DAYS * 24 * 60 * 60 * 1000;
+    try {
+        if (statSync(dbPath).mtimeMs < cutoffMs)
+            return [];
+    }
+    catch {
+        return [];
+    }
+    const sqlite3 = resolveSqlite3Binary();
+    if (!sqlite3)
+        return [];
+    // Window aggregation to the recency horizon (message.time_created is epoch ms) so a touched-but-
+    // -stale DB (VACUUM, backup, one new session) doesn't re-sum all-time history — matches the
+    // 30-day window the Claude/Copilot collectors apply per transcript file. cutoffMs is a numeric
+    // constant we control, so inlining it into the SQL is injection-safe.
+    const cutoffClause = `WHERE time_created >= ${cutoffMs}`;
+    // Assistant message JSON blobs, one per row (char(30)-delimited so embedded newlines in the
+    // JSON don't split rows).
+    const messagesRaw = querySqlite(sqlite3, dbPath, `SELECT data || char(30) FROM message ${cutoffClause};`);
+    if (messagesRaw === null)
+        return [];
+    const totals = emptyTotals();
+    const models = new Set();
+    const providers = new Set();
+    const modelTokenSplit = {};
+    let cost = 0;
+    let sawData = false;
+    for (const blob of messagesRaw.split(ROW_SEP)) {
+        const trimmed = blob.trim();
+        if (!trimmed)
+            continue;
+        let msg;
+        try {
+            msg = JSON.parse(trimmed);
+        }
+        catch {
+            continue;
+        }
+        if (msg.role !== 'assistant')
+            continue;
+        sawData = true;
+        const model = typeof msg.modelID === 'string' ? msg.modelID : '';
+        if (model)
+            models.add(model);
+        if (typeof msg.providerID === 'string' && msg.providerID)
+            providers.add(msg.providerID);
+        cost += num(msg.cost);
+        const tokens = asObject(msg.tokens);
+        const cache = asObject(tokens.cache);
+        const input = num(tokens.input);
+        const output = num(tokens.output) + num(tokens.reasoning);
+        const cacheRead = num(cache.read);
+        const cacheWrite = num(cache.write);
+        totals.input_tokens += input;
+        totals.output_tokens += output;
+        totals.cache_read_input_tokens += cacheRead;
+        totals.cache_creation_input_tokens += cacheWrite;
+        if (model) {
+            const sum = input + output + cacheRead + cacheWrite;
+            if (sum > 0)
+                modelTokenSplit[model] = (modelTokenSplit[model] ?? 0) + sum;
+        }
+    }
+    // Session count = distinct sessions with activity in the window (the `session` table has no
+    // timestamp, so count distinct session_id from recent messages — consistent with the windowed
+    // token totals above).
+    const sessionCountRaw = querySqlite(sqlite3, dbPath, `SELECT count(DISTINCT session_id) FROM message ${cutoffClause};`);
+    const sessionCount = sessionCountRaw ? num(parseInt(sessionCountRaw.trim(), 10)) : 0;
+    const accountRaw = querySqlite(sqlite3, dbPath, "SELECT COALESCE(email, '') || char(31) || COALESCE(url, '') FROM account ORDER BY time_updated DESC LIMIT 1;");
+    let email = '';
+    let host = '';
+    if (accountRaw && accountRaw.trim()) {
+        const [rawEmail, rawUrl] = accountRaw.trim().split(FIELD_SEP);
+        email = (rawEmail ?? '').trim();
+        host = (rawUrl ?? '')
+            .trim()
+            .replace(/^https?:\/\//, '')
+            .replace(/\/$/, '');
+    }
+    if (!sawData && !email)
+        return [];
+    return [
+        {
+            file_type: TOKEN_USAGE_FILE_TYPE,
+            file_path: dbPath, // stable synthetic path → one aggregate row per machine
+            raw_content: {
+                version: readOpencodeVersion(home),
+                session_count: sessionCount,
+                models: [...models].sort(),
+                providers: [...providers].sort(),
+                model_token_split: modelTokenSplit,
+                cost,
+                email,
+                host,
+                window_days: TOKEN_USAGE_RECENCY_DAYS,
+                totals,
+            },
+        },
+    ];
+}
+export { collectOpencodeTokenUsage, TOKEN_USAGE_RECENCY_DAYS, TOKEN_USAGE_FILE_TYPE };

package/dist/log_config_files/collection/pi_token_usage_collector.js

@@ -0,0 +1,180 @@
+import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+/**
+ * Session / token usage / model / provider aggregator for Pi (earendil-works, pi.dev).
+ *
+ * Pi writes one JSONL transcript per session under
+ * ~/.pi/agent/sessions/<sanitized-cwd>/<timestamp>_<uuid>.jsonl. The first line is a
+ * `type: "session"` header carrying the session `id`; assistant turns are `type: "message"`
+ * lines whose `message.usage` block carries `{input, output, cacheRead, cacheWrite,
+ * totalTokens, reasoning?}` alongside `message.model` / `message.provider`. We aggregate the
+ * *numbers* (and model/provider ids) on the endpoint and emit only those — never
+ * `message.content` (the actual prompt/response text) — so the backend gets token usage,
+ * session count, and models/providers used without ingesting transcript content.
+ *
+ * Configured defaults come from a scoped read of settings.json's `defaultProvider` /
+ * `defaultModel` / `lastChangelogVersion` keys only — never any other key. auth.json is never
+ * touched at all: it holds literal per-provider `{type, key}` credential blobs
+ * (`{"anthropic": {"type": ..., "key": ...}}`), the same risk class as Hermes's/Codex's
+ * auth.json.
+ *
+ * `lastChangelogVersion` (confirmed against a real install: "0.80.3", matching `pi --version`
+ * exactly) is a best-effort CLI version signal, not a guaranteed-current one — it records the
+ * version Pi last showed its changelog for, which in practice tracks the installed version but
+ * could theoretically lag by one release in a narrow upgrade window. The session header's own
+ * `version` field is a JSONL schema version (small int, e.g. `3`), not the CLI version, and is
+ * not used here.
+ *
+ * Bounded by mtime recency so a large session history is not fully re-parsed on every run.
+ */
+const TOKEN_USAGE_RECENCY_DAYS = 30;
+const TOKEN_USAGE_FILE_TYPE = 'pi_token_usage';
+function emptyTotals() {
+    return { input_tokens: 0, output_tokens: 0, cache_read_input_tokens: 0 };
+}
+function num(value) {
+    return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+function str(value) {
+    return typeof value === 'string' ? value.trim() : '';
+}
+function asObject(value) {
+    return value && typeof value === 'object' ? value : {};
+}
+/** Recursively collect *.jsonl files under a dir whose mtime is within the recency window. */
+function recentJsonlFiles(dir, cutoffMs) {
+    const out = [];
+    let entries;
+    try {
+        entries = readdirSync(dir, { withFileTypes: true });
+    }
+    catch {
+        return out;
+    }
+    for (const entry of entries) {
+        const full = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            out.push(...recentJsonlFiles(full, cutoffMs));
+        }
+        else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
+            try {
+                if (statSync(full).mtimeMs >= cutoffMs)
+                    out.push(full);
+            }
+            catch {
+                /* unreadable file — skip */
+            }
+        }
+    }
+    return out;
+}
+/** Scoped read of settings.json's `defaultProvider` / `defaultModel` / `lastChangelogVersion` keys only. */
+function readDefaults(home) {
+    try {
+        const raw = JSON.parse(readFileSync(join(home, '.pi', 'agent', 'settings.json'), 'utf8'));
+        return {
+            provider: str(raw.defaultProvider),
+            model: str(raw.defaultModel),
+            version: str(raw.lastChangelogVersion),
+        };
+    }
+    catch {
+        return { provider: '', model: '', version: '' };
+    }
+}
+/**
+ * Aggregate Pi session count / token usage / models / providers across recent session
+ * transcripts. Returns a single-element array (one per-machine aggregate) or [] when there is
+ * no recent data.
+ */
+function collectPiTokenUsage(home = homedir()) {
+    const sessionsDir = join(home, '.pi', 'agent', 'sessions');
+    if (!existsSync(sessionsDir))
+        return [];
+    const cutoffMs = Date.now() - TOKEN_USAGE_RECENCY_DAYS * 24 * 60 * 60 * 1000;
+    const files = recentJsonlFiles(sessionsDir, cutoffMs);
+    if (files.length === 0)
+        return [];
+    const totals = emptyTotals();
+    const sessions = new Set();
+    const models = new Set();
+    const providers = new Set();
+    const modelTokenSplit = {};
+    let sawData = false;
+    for (const file of files) {
+        let content;
+        try {
+            content = readFileSync(file, 'utf8');
+        }
+        catch {
+            continue;
+        }
+        for (const line of content.split('\n')) {
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            let entry;
+            try {
+                entry = JSON.parse(trimmed);
+            }
+            catch {
+                continue;
+            }
+            if (entry.type === 'session') {
+                const sessionId = str(entry.id);
+                if (sessionId)
+                    sessions.add(sessionId);
+                sawData = true;
+                continue;
+            }
+            if (entry.type !== 'message')
+                continue;
+            const message = asObject(entry.message);
+            const usage = asObject(message.usage);
+            if (Object.keys(usage).length === 0)
+                continue;
+            sawData = true;
+            const model = str(message.model);
+            const provider = str(message.provider);
+            if (model)
+                models.add(model);
+            if (provider)
+                providers.add(provider);
+            const input = num(usage.input);
+            // reasoning tokens fold into output_tokens — TokenTotals has no separate reasoning
+            // field (matches Codex's/Hermes's reasoning fold-in convention).
+            const output = num(usage.output) + num(usage.reasoning);
+            const cacheRead = num(usage.cacheRead);
+            totals.input_tokens += input;
+            totals.output_tokens += output;
+            totals.cache_read_input_tokens += cacheRead;
+            const sum = input + output + cacheRead;
+            if (model && sum > 0) {
+                modelTokenSplit[model] = (modelTokenSplit[model] ?? 0) + sum;
+            }
+        }
+    }
+    if (!sawData)
+        return [];
+    const defaults = readDefaults(home);
+    return [
+        {
+            file_type: TOKEN_USAGE_FILE_TYPE,
+            // Stable synthetic path → one aggregate row per machine.
+            file_path: sessionsDir,
+            raw_content: {
+                version: defaults.version,
+                session_count: sessions.size,
+                models: [...models].sort(),
+                providers: [...providers].sort(),
+                model_token_split: modelTokenSplit,
+                configured_model: defaults.model,
+                configured_provider: defaults.provider,
+                window_days: TOKEN_USAGE_RECENCY_DAYS,
+                totals,
+            },
+        },
+    ];
+}
+export { collectPiTokenUsage, TOKEN_USAGE_RECENCY_DAYS, TOKEN_USAGE_FILE_TYPE };

package/dist/log_config_files/runtime/main_runner.js

@@ -14,7 +14,7 @@ import { ensureAuthentication } from '../auth/auth_flow.js';
 import { readJSONFile, readMarkdownFile } from '../readers/file_readers.js';
 import { isVscdbVirtualPath, tryReadVscdbVirtualFile, summarizeComposerPayloadForDiagnostics, } from '../readers/vscdb_config_builder.js';
 import { persistVscdbComposerContractFromPatternsResponse } from '../readers/vscdb_reader.js';
-import { collectConfigFilesFromPatterns, collectMcpToolFiles, collectConfigFilesFromInstalledPlugins, collectPluginCacheMcpFiles, collectMcpFromClaudeJsonProjects, collectClaudeDesktopExtensionManifests, collectClaudeDesktopExtensionSettingsFiles, enrichClaudeDesktopExtensionsInstallationsUpload, determineFileTypeFromPath, } from '../collection/config_collector.js';
+import { collectConfigFilesFromPatterns, collectMcpToolFiles, collectConfigFilesFromInstalledPlugins, collectPluginCacheMcpFiles, collectMcpFromClaudeJsonProjects, collectClaudeTokenUsage, collectCopilotTokenUsage, collectOpencodeTokenUsage, collectCodexTokenUsage, collectCursorTokenUsage, collectHermesTokenUsage, collectOpenclawTokenUsage, collectPiTokenUsage, collectClaudeDesktopExtensionManifests, collectClaudeDesktopExtensionSettingsFiles, enrichClaudeDesktopExtensionsInstallationsUpload, determineFileTypeFromPath, } from '../collection/config_collector.js';
 import { ensureCursorUserSettingsSnapshotInBatch } from '../collection/ensure_cursor_user_settings_snapshot.js';
 import { collectSkillsCliInstalled } from '../collection/skills_cli_collector.js';
 import { collectWorkspaceVscdbs } from '../collection/mcp_tool_collector.js';
@@ -98,6 +98,70 @@ async function collectAllConfigFiles(endpointBase) {
             configFiles.push(entry);
         }
     }
+    hookRunLog(`aggregating Claude token usage from ~/.claude/projects transcripts`);
+    for (const entry of collectClaudeTokenUsage(HOME_DIR)) {
+        const key = `${entry.file_type}\t${entry.file_path}`;
+        if (!existingPaths.has(key)) {
+            existingPaths.add(key);
+            configFiles.push(entry);
+        }
+    }
+    hookRunLog(`aggregating Copilot token usage from ~/.copilot/session-state transcripts`);
+    for (const entry of collectCopilotTokenUsage(HOME_DIR)) {
+        const key = `${entry.file_type}\t${entry.file_path}`;
+        if (!existingPaths.has(key)) {
+            existingPaths.add(key);
+            configFiles.push(entry);
+        }
+    }
+    hookRunLog(`aggregating OpenCode token usage from ~/.local/share/opencode/opencode.db`);
+    for (const entry of collectOpencodeTokenUsage(HOME_DIR)) {
+        const key = `${entry.file_type}\t${entry.file_path}`;
+        if (!existingPaths.has(key)) {
+            existingPaths.add(key);
+            configFiles.push(entry);
+        }
+    }
+    hookRunLog(`aggregating Codex token usage from ~/.codex/sessions rollouts`);
+    for (const entry of collectCodexTokenUsage(HOME_DIR)) {
+        const key = `${entry.file_type}\t${entry.file_path}`;
+        if (!existingPaths.has(key)) {
+            existingPaths.add(key);
+            configFiles.push(entry);
+        }
+    }
+    hookRunLog(`aggregating Cursor token usage from state.vscdb`);
+    for (const entry of collectCursorTokenUsage(HOME_DIR)) {
+        const key = `${entry.file_type}\t${entry.file_path}`;
+        if (!existingPaths.has(key)) {
+            existingPaths.add(key);
+            configFiles.push(entry);
+        }
+    }
+    hookRunLog(`aggregating Hermes token usage from ~/.hermes/state.db`);
+    for (const entry of collectHermesTokenUsage(HOME_DIR)) {
+        const key = `${entry.file_type}\t${entry.file_path}`;
+        if (!existingPaths.has(key)) {
+            existingPaths.add(key);
+            configFiles.push(entry);
+        }
+    }
+    hookRunLog(`aggregating OpenClaw token usage from ~/.openclaw/agents/*/sessions/sessions.json`);
+    for (const entry of collectOpenclawTokenUsage(HOME_DIR)) {
+        const key = `${entry.file_type}\t${entry.file_path}`;
+        if (!existingPaths.has(key)) {
+            existingPaths.add(key);
+            configFiles.push(entry);
+        }
+    }
+    hookRunLog(`aggregating Pi token usage from ~/.pi/agent/sessions transcripts`);
+    for (const entry of collectPiTokenUsage(HOME_DIR)) {
+        const key = `${entry.file_type}\t${entry.file_path}`;
+        if (!existingPaths.has(key)) {
+            existingPaths.add(key);
+            configFiles.push(entry);
+        }
+    }
     hookRunLog(`merging disk-only Claude Desktop extensions into extensions-installations.json`);
     const diskExtMerge = enrichClaudeDesktopExtensionsInstallationsUpload(configFiles, HOME_DIR);
     hookRunLog(`claude desktop extensions: merged=${diskExtMerge.mergedCount} had_registry_upload=${diskExtMerge.hadRegistryUpload}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "log-llm-config",
-  "version": "1.5.2",
+  "version": "1.5.8",
   "description": "CLI helpers for logging hardware UUIDs and posting startup payloads to Optimus Security.",
   "type": "module",
   "bin": {