knodin 0.5.0

package/dist/src/output-telemetry.js

@@ -0,0 +1,359 @@
+import crypto from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { encode } from "gpt-tokenizer/encoding/o200k_base";
+export const TELEMETRY_TOKENIZER = "gpt-tokenizer@3.4.0:o200k_base";
+export const countOutputTokens = (value) => encode(value).length;
+export const DEFAULT_TELEMETRY_RETENTION_DAYS = 30;
+const DEFAULT_INPUT = ".reckon-telemetry.jsonl";
+function lineCount(value) {
+    return value.length === 0 ? 0 : value.split(/\r?\n/).length;
+}
+function repositoryIdentifier(repo) {
+    return `repo_${crypto.createHash("sha256").update(fs.realpathSync(repo)).digest("hex").slice(0, 16)}`;
+}
+function privatePath(repoPath, requested, purpose) {
+    const repo = fs.realpathSync(repoPath);
+    const target = path.resolve(repo, requested);
+    if (target !== repo && !target.startsWith(`${repo}${path.sep}`))
+        throw new Error(`knodin telemetry ${purpose}: path must stay inside the repository`);
+    const relative = path.relative(repo, target);
+    let cursor = repo;
+    for (const segment of relative === "" ? [] : relative.split(path.sep)) {
+        cursor = path.join(cursor, segment);
+        let stat;
+        try {
+            stat = fs.lstatSync(cursor);
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                break;
+            throw error;
+        }
+        if (stat.isSymbolicLink())
+            throw new Error(purpose === "persist"
+                ? "knodin telemetry: refusing symlinked persistence path"
+                : `knodin telemetry ${purpose}: refusing symlinked path`);
+    }
+    return { repo, target };
+}
+function atomicPrivateWrite(target, value) {
+    fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 });
+    const temporary = `${target}.${process.pid}.${Date.now()}.tmp`;
+    fs.writeFileSync(temporary, value, { mode: 0o600, flag: "wx" });
+    fs.renameSync(temporary, target);
+}
+function retentionCutoff(days, now) {
+    if (!Number.isInteger(days) || days < 1 || days > 3650)
+        throw new Error("knodin telemetry: retention days must be an integer from 1 to 3650");
+    return now.getTime() - days * 86_400_000;
+}
+function percent(baseline, optimized) {
+    if (baseline === 0)
+        return optimized === 0 ? 0 : Number.NEGATIVE_INFINITY;
+    return Math.round(((baseline - optimized) / baseline) * 10_000) / 100;
+}
+function baselineFiles(repo, operation, output) {
+    if (!["query:file_summary", "query:batch_outline"].includes(operation))
+        return [];
+    const candidates = operation === "query:file_summary"
+        ? [output.target]
+        : Array.isArray(output.results)
+            ? output.results.map((row) => row.file)
+            : [];
+    const root = fs.realpathSync(repo);
+    const files = new Set();
+    for (const candidate of candidates) {
+        if (typeof candidate !== "string" || !candidate)
+            continue;
+        const resolved = path.resolve(root, candidate);
+        let real;
+        try {
+            real = fs.realpathSync(resolved);
+        }
+        catch {
+            continue;
+        }
+        if (real !== root && !real.startsWith(`${root}${path.sep}`))
+            continue;
+        if (fs.statSync(real).isFile())
+            files.add(real);
+    }
+    return [...files].sort();
+}
+export function measureOutput(options) {
+    const serialized = JSON.stringify(options.output);
+    const optimized = {
+        contract: "actual-operation-response",
+        bytes: Buffer.byteLength(serialized),
+        tokens: encode(serialized).length,
+        lines: lineCount(serialized),
+    };
+    const files = baselineFiles(options.repo, options.operation, options.output);
+    const baselineText = files.length > 0 ? files.map((file) => fs.readFileSync(file, "utf-8")).join("\n") : null;
+    const baseline = baselineText === null
+        ? null
+        : {
+            contract: "executed-full-file-read",
+            bytes: Buffer.byteLength(baselineText),
+            tokens: encode(baselineText).length,
+            fileCount: files.length,
+            lines: lineCount(baselineText),
+        };
+    const status = typeof options.output.status === "string" ? options.output.status : undefined;
+    const availability = options.output.availability;
+    const freshness = options.output.freshness;
+    const fidelity = options.output.fidelity;
+    const database = path.join(fs.realpathSync(options.repo), ".reckon", "db.sqlite");
+    const latencyMs = Math.round((performance.now() - options.startedAt) * 100) / 100;
+    const outcome = (status !== undefined &&
+        ["error", "failed", "unavailable", "blocked", "timeout"].includes(status)) ||
+        availability?.state === "unavailable"
+        ? "error"
+        : "success";
+    let compressionFidelity = null;
+    if (typeof fidelity === "number")
+        compressionFidelity = fidelity;
+    else if (typeof fidelity?.percent === "number")
+        compressionFidelity = fidelity.percent;
+    return {
+        schemaVersion: 2,
+        at: new Date().toISOString(),
+        repositoryId: repositoryIdentifier(options.repo),
+        operation: options.operation,
+        latencyMs,
+        rssBytes: process.memoryUsage().rss,
+        temperature: "cold",
+        outcome,
+        agentRoundTrips: 1,
+        indexStorageBytes: fs.existsSync(database) ? fs.statSync(database).size : null,
+        indexingMs: options.operation.startsWith("index") ? latencyMs : null,
+        freshnessFailure: options.output.staleness === "stale" ||
+            (typeof freshness?.state === "string" && freshness.state !== "fresh") ||
+            availability?.state === "unavailable",
+        compressionFidelity,
+        confidence: baseline ? "measured" : "unavailable",
+        truncated: options.truncated,
+        detailMode: options.detailMode,
+        tokenizer: TELEMETRY_TOKENIZER,
+        schemaTokens: options.schemaTokens,
+        optimized,
+        baseline,
+        savings: baseline
+            ? {
+                bytes: baseline.bytes - optimized.bytes,
+                tokens: baseline.tokens - optimized.tokens,
+                bytePercent: percent(baseline.bytes, optimized.bytes),
+                tokenPercent: percent(baseline.tokens, optimized.tokens),
+            }
+            : null,
+    };
+}
+function percentile(values, fraction) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((left, right) => left - right);
+    return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)] ?? 0;
+}
+function escapeHtml(value) {
+    return value
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;")
+        .replaceAll('"', "&quot;")
+        .replaceAll("'", "&#39;");
+}
+function telemetrySummary(records) {
+    const measured = records.filter(({ savings }) => savings !== null);
+    return {
+        calls: records.length,
+        successes: records.filter(({ outcome }) => outcome !== "error").length,
+        errors: records.filter(({ outcome }) => outcome === "error").length,
+        measuredBaselines: measured.length,
+        measuredTokensAvoided: measured.reduce((sum, record) => sum + (record.savings?.tokens ?? 0), 0),
+        latencyP50Ms: percentile(records.map(({ latencyMs }) => latencyMs), 0.5),
+        latencyP95Ms: percentile(records.map(({ latencyMs }) => latencyMs), 0.95),
+        freshnessFailures: records.filter(({ freshnessFailure }) => freshnessFailure === true).length,
+        compressionFidelitySamples: records.filter(({ compressionFidelity }) => typeof compressionFidelity === "number").length,
+        compressionFidelityP50: percentile(records.flatMap(({ compressionFidelity }) => typeof compressionFidelity === "number" ? [compressionFidelity] : []), 0.5),
+        indexStorageBytes: Math.max(0, ...records.map(({ indexStorageBytes }) => indexStorageBytes ?? 0)),
+        indexingMs: records.reduce((sum, record) => sum + (record.indexingMs ?? 0), 0),
+    };
+}
+export function writeTelemetryReport(repoPath, records, outputPath = ".reckon/telemetry-report.html") {
+    const { repo, target: output } = privatePath(repoPath, outputPath, "report");
+    const groups = new Map();
+    for (const record of records) {
+        const group = groups.get(record.operation) ?? [];
+        group.push(record);
+        groups.set(record.operation, group);
+    }
+    const rows = [...groups]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([operation, values]) => {
+        const measured = values.filter(({ savings }) => savings !== null);
+        const tokens = measured.reduce((sum, value) => sum + (value.savings?.tokens ?? 0), 0);
+        return `<tr><td>${escapeHtml(operation)}</td><td>${values.length}</td><td>${values.filter(({ outcome }) => outcome === "error").length}</td><td>${measured.length}</td><td>${tokens}</td><td>${percentile(values.map(({ latencyMs }) => latencyMs), 0.5)}</td><td>${percentile(values.map(({ latencyMs }) => latencyMs), 0.95)}</td></tr>`;
+    })
+        .join("");
+    const summary = telemetrySummary(records);
+    const timeSeries = new Map();
+    for (const record of records) {
+        const day = typeof record.at === "string" ? record.at.slice(0, 10) : "unknown";
+        const current = timeSeries.get(day) ?? { calls: 0, tokens: 0 };
+        current.calls += 1;
+        current.tokens += record.savings?.tokens ?? 0;
+        timeSeries.set(day, current);
+    }
+    const seriesRows = [...timeSeries]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([day, value]) => `<tr><td>${escapeHtml(day)}</td><td>${value.calls}</td><td>${value.tokens}</td></tr>`)
+        .join("");
+    const repositories = new Map();
+    for (const record of records)
+        if (record.repositoryId)
+            repositories.set(record.repositoryId, (repositories.get(record.repositoryId) ?? 0) + 1);
+    const repositoryRows = [...repositories]
+        .sort(([left], [right]) => left.localeCompare(right))
+        .map(([id, calls]) => `<tr><td>${escapeHtml(id)}</td><td>${calls}</td></tr>`)
+        .join("");
+    const html = `<!doctype html><meta charset="utf-8"><meta name="color-scheme" content="light dark"><title>knodin telemetry</title><style>:root{color-scheme:light dark;--line:#d0d7de;--card:#f6f8fa}body{font:14px system-ui;margin:2rem;max-width:76rem}section{background:var(--card);padding:1rem;margin:1rem 0;border-radius:.5rem}table{border-collapse:collapse;width:100%}th,td{padding:.55rem;border-bottom:1px solid var(--line);text-align:left}@media(prefers-color-scheme:dark){:root{--line:#30363d;--card:#161b22}}</style><h1>knodin local telemetry</h1><p>Tokenizer: ${TELEMETRY_TOKENIZER}. Local, opt-in, metadata-only; source, raw output, command text, usernames, and paths are not persisted.</p><section><h2>Summary</h2><p>Total measured tokens avoided: ${summary.measuredTokensAvoided}</p><p>Calls: ${summary.calls}; successes: ${summary.successes}; errors: ${summary.errors}; measured baselines: ${summary.measuredBaselines}</p><p>Latency p50 / p95: ${summary.latencyP50Ms} / ${summary.latencyP95Ms} ms</p><p>Freshness failures: ${summary.freshnessFailures}; compression fidelity samples / p50: ${summary.compressionFidelitySamples} / ${summary.compressionFidelityP50}; indexing time: ${summary.indexingMs} ms; index storage: ${summary.indexStorageBytes} bytes</p></section><section><h2>Operation breakdown</h2><table><thead><tr><th>Operation</th><th>Calls</th><th>Errors</th><th>Measured baselines</th><th>Net tokens saved</th><th>p50 ms</th><th>p95 ms</th></tr></thead><tbody>${rows}</tbody></table></section><section><h2>Time series</h2><table><thead><tr><th>Day</th><th>Calls</th><th>Tokens avoided</th></tr></thead><tbody>${seriesRows}</tbody></table></section><section><h2>Repository breakdown</h2><table><thead><tr><th>Private repository id</th><th>Calls</th></tr></thead><tbody>${repositoryRows}</tbody></table></section><p>Use <code>knodin telemetry export</code> for the machine-readable evidence bundle and <code>knodin telemetry clear</code> to delete persisted records.</p>`;
+    atomicPrivateWrite(output, html);
+    return {
+        outputPath: path.relative(repo, output),
+        records: records.length,
+        operations: groups.size,
+    };
+}
+function sanitizeTelemetryRecord(record) {
+    return {
+        schemaVersion: record.schemaVersion,
+        at: record.at,
+        repositoryId: record.repositoryId,
+        operation: record.operation,
+        latencyMs: record.latencyMs,
+        rssBytes: record.rssBytes,
+        temperature: record.temperature,
+        outcome: record.outcome,
+        agentRoundTrips: record.agentRoundTrips,
+        indexStorageBytes: record.indexStorageBytes,
+        indexingMs: record.indexingMs,
+        freshnessFailure: record.freshnessFailure,
+        compressionFidelity: record.compressionFidelity,
+        confidence: record.confidence,
+        truncated: record.truncated,
+        detailMode: record.detailMode,
+        tokenizer: record.tokenizer,
+        schemaTokens: record.schemaTokens,
+        optimized: {
+            contract: "actual-operation-response",
+            bytes: record.optimized.bytes,
+            tokens: record.optimized.tokens,
+            lines: record.optimized.lines,
+        },
+        baseline: record.baseline
+            ? {
+                contract: "executed-full-file-read",
+                bytes: record.baseline.bytes,
+                tokens: record.baseline.tokens,
+                fileCount: record.baseline.fileCount,
+                lines: record.baseline.lines,
+            }
+            : null,
+        savings: record.savings
+            ? {
+                bytes: record.savings.bytes,
+                tokens: record.savings.tokens,
+                bytePercent: record.savings.bytePercent,
+                tokenPercent: record.savings.tokenPercent,
+            }
+            : null,
+    };
+}
+export function readTelemetryRecords(repoPath, inputPath = DEFAULT_INPUT, retentionDays, now = new Date()) {
+    const { target: input } = privatePath(repoPath, inputPath, "read");
+    if (!fs.existsSync(input))
+        return [];
+    const records = fs
+        .readFileSync(input, "utf-8")
+        .split(/\r?\n/)
+        .filter(Boolean)
+        .map((line, index) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch {
+            throw new Error(`knodin telemetry read: malformed JSONL record at line ${index + 1}`);
+        }
+    })
+        .filter((record) => typeof record.operation === "string" &&
+        record.tokenizer === TELEMETRY_TOKENIZER &&
+        typeof record.optimized?.tokens === "number")
+        .map(sanitizeTelemetryRecord);
+    if (retentionDays === undefined)
+        return records;
+    const cutoff = retentionCutoff(retentionDays, now);
+    return records.filter(({ at }) => typeof at !== "string" || Date.parse(at) >= cutoff);
+}
+export function appendTelemetryRecord(repoPath, record, retentionDays = DEFAULT_TELEMETRY_RETENTION_DAYS, now = new Date(), inputPath = DEFAULT_INPUT) {
+    const { target } = privatePath(repoPath, inputPath, "persist");
+    const records = readTelemetryRecords(repoPath, inputPath);
+    records.push(sanitizeTelemetryRecord({ ...record, at: record.at ?? now.toISOString() }));
+    const cutoff = retentionCutoff(retentionDays, now);
+    const retained = records.filter(({ at }) => typeof at !== "string" || Date.parse(at) >= cutoff);
+    atomicPrivateWrite(target, retained.length === 0 ? "" : `${retained.map((value) => JSON.stringify(value)).join("\n")}\n`);
+    return { records: retained.length, retentionDays };
+}
+export function telemetryStatus(repoPath, inputPath = DEFAULT_INPUT, retentionDays = DEFAULT_TELEMETRY_RETENTION_DAYS, now = new Date()) {
+    const { repo, target } = privatePath(repoPath, inputPath, "status");
+    const persisted = fs.existsSync(target);
+    const records = readTelemetryRecords(repo, inputPath, retentionDays, now);
+    const dates = records
+        .map(({ at }) => (typeof at === "string" ? at : null))
+        .filter((value) => value !== null)
+        .sort((left, right) => left.localeCompare(right));
+    return {
+        schemaVersion: 1,
+        enabledByDefault: false,
+        localOnly: true,
+        persisted,
+        inputPath: path.relative(repo, target),
+        records: records.length,
+        bytes: persisted ? fs.statSync(target).size : 0,
+        retentionDays,
+        oldestAt: dates[0] ?? null,
+        newestAt: dates.at(-1) ?? null,
+        tokenizer: TELEMETRY_TOKENIZER,
+    };
+}
+export function exportTelemetry(repoPath, records, outputPath = ".reckon/telemetry-export.json") {
+    const { repo, target } = privatePath(repoPath, outputPath, "export");
+    const bundle = {
+        schemaVersion: 1,
+        generatedAt: new Date().toISOString(),
+        privacy: "local-metadata-only",
+        tokenizer: TELEMETRY_TOKENIZER,
+        methodology: {
+            measured: "actual operation response versus executed repository-contained full-file read",
+            modeled: null,
+            unknownBaselines: "null",
+        },
+        summary: telemetrySummary(records),
+        records,
+    };
+    atomicPrivateWrite(target, `${JSON.stringify(bundle, null, 2)}\n`);
+    return {
+        outputPath: path.relative(repo, target),
+        records: records.length,
+        summary: bundle.summary,
+    };
+}
+export function clearTelemetry(repoPath, inputPath = DEFAULT_INPUT) {
+    const { target } = privatePath(repoPath, inputPath, "clear");
+    if (!fs.existsSync(target))
+        return { removed: false, records: 0, bytes: 0 };
+    const records = readTelemetryRecords(repoPath, inputPath).length;
+    const bytes = fs.statSync(target).size;
+    fs.unlinkSync(target);
+    return { removed: true, records, bytes };
+}