nilometer 0.0.0-stage → 0.1.0

package/dist/core/ingest/classify.js

@@ -0,0 +1,382 @@
+/**
+ * @file Classifying raw log lines and extracting their fields (docs/development.md P4.4).
+ *
+ * Implements the rules in fixtures/README.md, which restate D-001 (dedup key), D-004 (limit hits
+ * from structured fields), D-007 (per-request timestamps), and D-019 (retry notices), plus the
+ * conversation links D-021 and D-022 read (not part of expected.json). Every function
+ * here is pure: bytes or a parsed object in, plain values out. Nothing touches the database, so the
+ * rules can be tested line by line, and the fidelity suite compares their combined output with an
+ * independent Python implementation of the same spec.
+ */
+/** The seven line classes, in the order they are tested. */
+export const LINE_CLASSES = [
+    "malformed",
+    "limit_hit",
+    "api_error",
+    "synthetic_other",
+    "request",
+    "retry_notice",
+    "model_fallback",
+    "ignored_type",
+];
+/** Label used wherever a required string is absent or not a string. */
+export const MISSING = "<missing>";
+/**
+ * Parses a raw line as a JSON object.
+ * @param bytes - The exact line bytes.
+ * @returns The object, or null when the line isn't valid JSON or isn't an object (arrays and
+ *   primitives included). An empty line is invalid JSON, so it's null too.
+ */
+export function parseLine(bytes) {
+    let value;
+    try {
+        value = JSON.parse(bytes.toString("utf8"));
+    }
+    catch {
+        return null;
+    }
+    return isObject(value) ? value : null;
+}
+/**
+ * Reports whether a value is a plain JSON object.
+ * @param value - Any parsed JSON value.
+ * @returns True for objects; false for arrays, null, and primitives.
+ */
+export function isObject(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/**
+ * Reads a nested object field.
+ * @param container - The value to read from.
+ * @param key - Field name.
+ * @returns The field if it's an object, else null.
+ */
+export function objectField(container, key) {
+    if (!isObject(container)) {
+        return null;
+    }
+    const value = container[key];
+    return isObject(value) ? value : null;
+}
+/**
+ * Reports whether a value is a JSON number. Booleans, strings, and null are not numbers here.
+ * @param value - Any parsed JSON value.
+ * @returns True only for numbers.
+ */
+export function isNumber(value) {
+    return typeof value === "number";
+}
+/**
+ * Returns a string field, or the missing label when it's absent or not a string.
+ * @param value - Any parsed JSON value.
+ * @returns The string itself, or {@link MISSING}.
+ */
+export function identityString(value) {
+    return typeof value === "string" ? value : MISSING;
+}
+/**
+ * Returns a string, or null when the value isn't a non-empty string.
+ * @param value - Any parsed JSON value.
+ * @returns The string, or null for absent, empty, or non-string values.
+ */
+export function nonEmptyString(value) {
+    return typeof value === "string" && value !== "" ? value : null;
+}
+/**
+ * Classifies one line. The first matching rule wins (fixtures/README.md, Classification).
+ * @param line - The parsed object, or null for a line that didn't parse.
+ * @returns The line's class.
+ */
+export function classifyLine(line) {
+    if (line === null) {
+        return "malformed";
+    }
+    // Error lines are tested before the request test: synthetic error lines carry a zero usage
+    // object and would otherwise look like requests (D-004).
+    if (line["isApiErrorMessage"] === true) {
+        return line["error"] === "rate_limit" ? "limit_hit" : "api_error";
+    }
+    const message = objectField(line, "message");
+    if (message?.["model"] === "<synthetic>") {
+        return "synthetic_other";
+    }
+    if (line["type"] === "assistant" && objectField(message, "usage") !== null) {
+        return "request";
+    }
+    if (line["type"] === "system" && line["subtype"] === "api_error") {
+        return "retry_notice";
+    }
+    // A request one model refused and another retried (D-080): the only record of the refusal.
+    if (line["type"] === "system" && line["subtype"] === "model_refusal_fallback") {
+        return "model_fallback";
+    }
+    return "ignored_type";
+}
+/**
+ * Builds the D-001 dedup key for a request line.
+ * @param line - A request line.
+ * @returns `"<session>/m/<message.id>"`, else `"<session>/r/<requestId>"`, else null (unkeyed).
+ */
+export function dedupKey(line) {
+    const session = identityString(line["sessionId"]);
+    const messageId = nonEmptyString(objectField(line, "message")?.["id"]);
+    if (messageId !== null) {
+        return `${session}/m/${messageId}`;
+    }
+    const requestId = nonEmptyString(line["requestId"]);
+    // No invented key when both IDs are missing: the line passes through and gets reported.
+    return requestId === null ? null : `${session}/r/${requestId}`;
+}
+/** The three token fields whose absence is reported, with their log names. */
+const REPORTED_TOKEN_FIELDS = [
+    ["inputTokens", "input_tokens"],
+    ["outputTokens", "output_tokens"],
+    ["cacheReadTokens", "cache_read_input_tokens"],
+];
+/**
+ * Extracts request fields from a request line.
+ * @param line - A line classified as `request`.
+ * @returns The fields, with missing numbers defaulted and listed.
+ */
+export function extractRequest(line) {
+    const message = objectField(line, "message");
+    const usage = objectField(message, "usage") ?? {};
+    const missingFields = [];
+    const counts = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0 };
+    for (const [key, logName] of REPORTED_TOKEN_FIELDS) {
+        const value = usage[logName];
+        if (isNumber(value)) {
+            counts[key] = value;
+        }
+        else {
+            missingFields.push(logName);
+        }
+    }
+    const split = objectField(usage, "cache_creation");
+    /**
+     * Reads a cache-write count, defaulting silently to 0 (only the three token fields are reported).
+     * @param value - The raw value.
+     * @returns The number, or 0.
+     */
+    const cacheCount = (value) => (isNumber(value) ? value : 0);
+    const iterations = Array.isArray(usage["iterations"]) ? usage["iterations"] : [];
+    return {
+        dedupKey: dedupKey(line),
+        messageId: typeof message?.["id"] === "string" ? message["id"] : null,
+        requestId: typeof line["requestId"] === "string" ? line["requestId"] : null,
+        model: identityString(message?.["model"]),
+        ...counts,
+        // The split object decides the form: split fields, or one unsplit total from older logs.
+        cacheWrite5mTokens: split === null ? null : cacheCount(split["ephemeral_5m_input_tokens"]),
+        cacheWrite1hTokens: split === null ? null : cacheCount(split["ephemeral_1h_input_tokens"]),
+        cacheWriteUnsplitTokens: split === null ? cacheCount(usage["cache_creation_input_tokens"]) : null,
+        speed: typeof usage["speed"] === "string" ? usage["speed"] : null,
+        serviceTier: typeof usage["service_tier"] === "string" ? usage["service_tier"] : null,
+        inferenceGeo: typeof usage["inference_geo"] === "string" ? usage["inference_geo"] : null,
+        missingFields,
+        nonMessageIterations: iterations
+            .filter((iteration) => isObject(iteration) && iteration["type"] !== "message")
+            .map((iteration) => identityString(iteration["type"])),
+    };
+}
+/** Strict ISO-8601 pattern from fixtures/README.md: seconds required, uppercase Z or ±HH:MM. */
+const TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?(Z|([+-])(\d{2}):(\d{2}))$/;
+/**
+ * Parses a timestamp strictly.
+ * @param raw - The raw `timestamp` value.
+ * @returns The UTC instant and day, or null when the value isn't a string in the required format
+ *   with every part in range (a real date, hour 00–23, minute and second 00–59, offset hour 00–23,
+ *   offset minute 00–59).
+ * @example
+ * parseTimestamp("2026-09-02T01:30:00.000+02:00"); // { utc: "2026-09-01T23:30:00.000Z", day: "2026-09-01" }
+ */
+export function parseTimestamp(raw) {
+    if (typeof raw !== "string") {
+        return null;
+    }
+    const match = TIMESTAMP.exec(raw);
+    if (match === null) {
+        return null;
+    }
+    const [year, month, day, hour, minute, second] = match.slice(1, 7).map(Number);
+    const fraction = match[7] ?? "";
+    const sign = match[9] === "-" ? -1 : 1;
+    const offsetHour = match[10] === undefined ? 0 : Number(match[10]);
+    const offsetMinute = match[11] === undefined ? 0 : Number(match[11]);
+    if (hour > 23 || minute > 59 || second > 59 || offsetHour > 23 || offsetMinute > 59) {
+        return null;
+    }
+    // Date.UTC silently rolls over invalid dates (Feb 30 → Mar 2); reading the parts back catches that.
+    const local = new Date(Date.UTC(year, month - 1, day, hour, minute, second));
+    if (local.getUTCFullYear() !== year ||
+        local.getUTCMonth() !== month - 1 ||
+        local.getUTCDate() !== day) {
+        return null;
+    }
+    // Milliseconds from the first three fraction digits; finer precision is kept in the raw string.
+    const millis = Number(fraction.padEnd(3, "0").slice(0, 3));
+    const utcMs = local.getTime() + millis - sign * (offsetHour * 60 + offsetMinute) * 60_000;
+    const utc = new Date(utcMs).toISOString();
+    return { utc, day: utc.slice(0, 10) };
+}
+/**
+ * Joins a message's text the way fixtures/README.md defines it.
+ * @param line - Any parsed line.
+ * @returns Concatenated `content[].text` strings for list content, the string itself for string
+ *   content, otherwise an empty string.
+ */
+export function messageText(line) {
+    const content = objectField(line, "message")?.["content"];
+    if (typeof content === "string") {
+        return content;
+    }
+    if (!Array.isArray(content)) {
+        return "";
+    }
+    return content
+        .map((part) => (isObject(part) && typeof part["text"] === "string" ? part["text"] : ""))
+        .join("");
+}
+/**
+ * Parses a limit message's window and reset time. Matching ignores case.
+ * @param text - The message text.
+ * @returns The window (session and 5-hour wording checked before weekly) and the reset text.
+ * @example
+ * parseLimitText("You've hit your session limit · resets 6:10am (UTC)");
+ * // { window: "five_hour", resetText: "6:10am (UTC)" }
+ */
+export function parseLimitText(text) {
+    const lower = text.toLowerCase();
+    const window = lower.includes("session limit") || lower.includes("5-hour limit")
+        ? "five_hour"
+        : lower.includes("weekly limit")
+            ? "seven_day"
+            : null;
+    const at = lower.indexOf("resets ");
+    const reset = at === -1 ? "" : text.slice(at + "resets ".length).trim();
+    return { window, resetText: reset === "" ? null : reset };
+}
+/** The largest `resetsAt` read: 9999-12-31T23:59:59Z, the last second a four-digit year can write. */
+export const MAX_RESETS_AT = 253_402_300_799;
+/**
+ * Reads the structured window and reset that Claude Code writes on limit hits from 2.1.281 on
+ * (observed 2026-09-24). Older lines have no `quotaLimits`, which is not a problem; a member that
+ * is present but can't be used is listed, so a change of shape is reported rather than guessed at.
+ * @param line - A limit-hit line.
+ * @returns The window, the reset time, and the unusable members.
+ * @example
+ * parseQuotaLimits({ quotaLimits: { rateLimitType: "five_hour", resetsAt: 1788329400 } });
+ * // { window: "five_hour", resetsAtUtc: "2026-09-02T06:10:00.000Z", unusable: [] }
+ */
+export function parseQuotaLimits(line) {
+    if (!Object.hasOwn(line, "quotaLimits")) {
+        return { window: null, resetsAtUtc: null, unusable: [] };
+    }
+    const quota = line["quotaLimits"];
+    if (!isObject(quota)) {
+        return { window: null, resetsAtUtc: null, unusable: ["quotaLimits"] };
+    }
+    const unusable = [];
+    const type = quota["rateLimitType"];
+    // Only the two windows the rest of the tool knows; anything else (a per-model window, say) is
+    // reported, not mapped onto one of them.
+    const window = type === "five_hour" || type === "seven_day" ? type : null;
+    if (Object.hasOwn(quota, "rateLimitType") && window === null) {
+        unusable.push("rateLimitType");
+    }
+    const resetsAt = quota["resetsAt"];
+    // Whole seconds only: the observed values are integers, and a fraction would mean a unit change.
+    const resetsAtUtc = isNumber(resetsAt) && Number.isInteger(resetsAt) && resetsAt > 0 && resetsAt <= MAX_RESETS_AT
+        ? new Date(resetsAt * 1000).toISOString()
+        : null;
+    if (Object.hasOwn(quota, "resetsAt") && resetsAtUtc === null) {
+        unusable.push("resetsAt");
+    }
+    return { window, resetsAtUtc, unusable };
+}
+/**
+ * Extracts event fields from a line of an event class.
+ * @param line - The parsed line.
+ * @param lineClass - Its class: `limit_hit`, `api_error`, `synthetic_other`, `retry_notice`, or
+ *   `model_fallback`.
+ * @returns The event fields.
+ */
+export function extractEvent(line, lineClass) {
+    const errorObject = objectField(line, "error");
+    const status = isNumber(line["apiErrorStatus"])
+        ? line["apiErrorStatus"]
+        : lineClass === "retry_notice" && isNumber(errorObject?.["status"])
+            ? errorObject["status"]
+            : null;
+    const limit = lineClass === "limit_hit"
+        ? parseLimitText(messageText(line))
+        : { window: null, resetText: null };
+    const error = typeof line["error"] === "string" ? line["error"] : null;
+    return {
+        error,
+        apiErrorStatus: status,
+        window: limit.window,
+        resetText: limit.resetText,
+        quota: lineClass === "limit_hit"
+            ? parseQuotaLimits(line)
+            : { window: null, resetsAtUtc: null, unusable: [] },
+        unknownErrorKey: lineClass === "api_error" && error !== "server_error" ? (error ?? MISSING) : null,
+        retryRateLimitsPresent: lineClass === "retry_notice" &&
+            errorObject !== null &&
+            Object.hasOwn(errorObject, "rateLimits") &&
+            errorObject["rateLimits"] !== null,
+        fallback: lineClass === "model_fallback"
+            ? {
+                originalModel: stringOrNull(line["originalModel"]),
+                fallbackModel: stringOrNull(line["fallbackModel"]),
+                requestId: stringOrNull(line["requestId"]),
+                refusalCategory: stringOrNull(line["apiRefusalCategory"]),
+            }
+            : { originalModel: null, fallbackModel: null, requestId: null, refusalCategory: null },
+    };
+}
+/**
+ * Keeps a value only when it's a string.
+ * @param value - Any JSON value.
+ * @returns The string, or null for anything else, including a missing member.
+ */
+function stringOrNull(value) {
+    return typeof value === "string" ? value : null;
+}
+/**
+ * Extracts a line's conversation links.
+ * @param line - Any parsed line.
+ * @returns The links; absent or mistyped fields are null (or false for `isMeta`).
+ */
+export function extractLinks(line) {
+    const content = objectField(line, "message")?.["content"];
+    let userContent = null;
+    if (line["type"] === "user") {
+        if (typeof content === "string") {
+            userContent = "prompt";
+        }
+        else if (Array.isArray(content)) {
+            userContent = content.some((block) => isObject(block) && block["type"] === "tool_result")
+                ? "tool_result"
+                : "prompt";
+        }
+    }
+    const originKind = objectField(line, "origin")?.["kind"];
+    return {
+        uuid: nonEmptyString(line["uuid"]),
+        parentUuid: nonEmptyString(line["parentUuid"]),
+        originKind: typeof originKind === "string" ? originKind : null,
+        userContent,
+        isMeta: line["isMeta"] === true,
+    };
+}
+/**
+ * Builds the ignored-type label for a line.
+ * @param line - A line classified as `ignored_type`.
+ * @returns `type`, or `type/subtype` when `subtype` is a string; a missing type is {@link MISSING}.
+ */
+export function ignoredTypeKey(line) {
+    const type = identityString(line["type"]);
+    return typeof line["subtype"] === "string" ? `${type}/${line["subtype"]}` : type;
+}
+//# sourceMappingURL=classify.js.map

package/dist/core/ingest/command.js

@@ -0,0 +1,95 @@
+/**
+ * @file The `ingest` operation behind the CLI command and `init`'s backfill (docs/development.md P4.8).
+ *
+ * One call: open the database in the data directory, read new session-log lines and spool lines,
+ * bring derived tables and repository attribution up to date, and summarize what's stored. The
+ * summary counts come from SQL over the stored data, so a second run reports the same totals.
+ */
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { openDatabase } from "../db/database.js";
+import { resolveDataDir } from "../install/locations.js";
+import { securePrivateDataDir } from "../install/private-files.js";
+import { loadPriceTable, syncPrices } from "../pricing/prices.js";
+import { resolveRepositories } from "./attribution.js";
+import { ensureDerived } from "./derive.js";
+import { resolveRoots } from "./discover.js";
+import { ingestLogs } from "./ingest.js";
+/** File name of the database inside the data directory. */
+export const DATABASE_FILE = "usage.db";
+/** File name of the hook's error log inside the data directory (hooks/statusline.sh). */
+export const HOOK_ERRORS_FILE = "hook-errors.log";
+/**
+ * Counts lines in the hook's error log.
+ * @param dataDir - The data directory.
+ * @returns Number of non-empty lines; 0 when the log doesn't exist.
+ */
+export function countHookErrors(dataDir) {
+    const path = join(dataDir, HOOK_ERRORS_FILE);
+    if (!existsSync(path)) {
+        return 0;
+    }
+    return readFileSync(path, "utf8")
+        .split("\n")
+        .filter((line) => line.trim() !== "").length;
+}
+/**
+ * Runs one ingest and summarizes the stored data.
+ * @param options - Locations, environment, mode, and clock.
+ * @returns Paths, this run's counts, and totals over everything stored.
+ * @throws {import("./discover.js").DiscoveryError} If `CLAUDE_CONFIG_DIR` names no usable directory.
+ * @throws {import("../pricing/prices.js").PriceTableError} If `core/pricing/prices.json` is invalid.
+ * @throws {Error} Whatever opening the database or reading a file throws.
+ */
+export function runIngestCommand(options) {
+    const dataDir = resolveDataDir({
+        home: options.home,
+        env: options.env,
+        override: options.dataDirOverride,
+    });
+    const roots = resolveRoots(options.env, options.home);
+    // Load prices before opening the database, so an invalid price file fails before anything is written.
+    const prices = loadPriceTable(join(options.packageRoot, "core", "pricing", "prices.json"));
+    // Before anything is read or written: the data directory holds copies of session content (D-043).
+    const permissionsTightened = securePrivateDataDir(dataDir);
+    const databasePath = join(dataDir, DATABASE_FILE);
+    const db = openDatabase(databasePath, join(options.packageRoot, "core", "schema"));
+    try {
+        const run = ingestLogs(db, {
+            roots,
+            mode: options.full ? "full" : "incremental",
+            now: options.now,
+            spoolDir: dataDir,
+        });
+        syncPrices(db, prices);
+        const derived = ensureDerived(db);
+        const repositoriesResolved = resolveRepositories(db, options.now);
+        const row = db
+            .prepare(`SELECT
+           (SELECT COUNT(*) FROM requests_dedup) AS requests,
+           (SELECT COUNT(*) FROM requests WHERE dedup_key IS NULL) AS unkeyedRequests,
+           (SELECT COUNT(*) FROM events WHERE class = 'limit_hit') AS limitHits,
+           (SELECT COUNT(*) FROM events WHERE class NOT IN ('limit_hit', 'model_fallback')) AS otherEvents,
+           (SELECT COUNT(*) FROM events WHERE class = 'model_fallback') AS modelFallbacks,
+           (SELECT COUNT(*) FROM parsed_lines WHERE class = 'malformed') AS malformedLines,
+           (SELECT COUNT(*) FROM status_readings WHERE status = 'ok') AS statusReadings,
+           (SELECT COUNT(*) FROM status_readings WHERE status <> 'ok') AS malformedReadings,
+           (SELECT COUNT(*) FROM rate_limit_windows WHERE validity <> 'valid') AS invalidWindows,
+           (SELECT MIN(timestamp_utc) FROM requests_dedup) AS firstRequestUtc,
+           (SELECT MAX(timestamp_utc) FROM requests_dedup) AS lastRequestUtc`)
+            .get();
+        return {
+            databasePath,
+            roots,
+            run,
+            derived: { lines: derived.derived, rebuilt: derived.rebuilt },
+            repositoriesResolved,
+            totals: { ...row, hookErrors: countHookErrors(dataDir) },
+            permissionsTightened,
+        };
+    }
+    finally {
+        db.close();
+    }
+}
+//# sourceMappingURL=command.js.map