@ferrule-io/ok-fine 0.3.12 → 0.3.14

package/dist/metrics/store.js

@@ -0,0 +1,298 @@
+import { rmSync } from "node:fs";
+import { performance } from "node:perf_hooks";
+import { BUCKET_COUNT, bucketIndex, quantile } from "./histogram.js";
+import { METRICS_LAYERS, } from "./types.js";
+export const METRICS_DB_FILE = "metrics.sqlite";
+/** Bump when the tables or LATENCY_BUCKETS_MS change; a mismatching database is rebuilt (metrics are disposable). */
+const SCHEMA_VERSION = 1;
+const FLUSH_INTERVAL_MS = 10_000;
+const MINUTE_RETENTION_SECONDS = 172_800;
+const PRUNE_INTERVAL_SECONDS = 3_600;
+const WINDOWS = {
+    "1h": { seconds: 3_600, table: "metrics_minute", stepSeconds: 60 },
+    "24h": { seconds: 86_400, table: "metrics_minute", stepSeconds: 900 },
+    "7d": { seconds: 604_800, table: "metrics_hour", stepSeconds: 7_200 },
+    "30d": { seconds: 2_592_000, table: "metrics_hour", stepSeconds: 21_600 },
+};
+const TABLES = ["metrics_minute", "metrics_hour"];
+const BUCKET_COLUMNS = Array.from({ length: BUCKET_COUNT }, (_, i) => `b${String(i).padStart(2, "0")}`);
+const VALUE_COLUMNS = ["count", "client_errors", "server_errors", "sum_ms", "min_ms", "max_ms", ...BUCKET_COLUMNS];
+const SUMS = [
+    "SUM(count) AS count",
+    "SUM(client_errors) AS client_errors",
+    "SUM(server_errors) AS server_errors",
+    "SUM(sum_ms) AS sum_ms",
+    "MIN(min_ms) AS min_ms",
+    "MAX(max_ms) AS max_ms",
+    ...BUCKET_COLUMNS.map((c) => `SUM(${c}) AS ${c}`),
+].join(", ");
+function createSchema(db) {
+    const columns = [
+        "bucket_start INTEGER NOT NULL",
+        "layer TEXT NOT NULL",
+        "op TEXT NOT NULL",
+        "count INTEGER NOT NULL",
+        "client_errors INTEGER NOT NULL",
+        "server_errors INTEGER NOT NULL",
+        "sum_ms REAL NOT NULL",
+        "min_ms REAL NOT NULL",
+        "max_ms REAL NOT NULL",
+        ...BUCKET_COLUMNS.map((c) => `${c} INTEGER NOT NULL`),
+        "PRIMARY KEY (bucket_start, layer, op)",
+    ].join(", ");
+    for (const table of TABLES) {
+        db.exec(`DROP TABLE IF EXISTS ${table}`);
+        db.exec(`CREATE TABLE ${table} (${columns}) WITHOUT ROWID`);
+    }
+    db.exec(`PRAGMA user_version = ${SCHEMA_VERSION}`);
+}
+function prepare(db) {
+    const perTable = (sql) => ({
+        metrics_minute: db.prepare(sql("metrics_minute")),
+        metrics_hour: db.prepare(sql("metrics_hour")),
+    });
+    const insertColumns = ["bucket_start", "layer", "op", ...VALUE_COLUMNS];
+    const updates = VALUE_COLUMNS.map((c) => c === "min_ms" || c === "max_ms" ? `${c} = ${c.slice(0, 3)}(${c}, excluded.${c})` : `${c} = ${c} + excluded.${c}`).join(", ");
+    return {
+        upsert: perTable((t) => `INSERT INTO ${t} (${insertColumns.join(", ")}) VALUES (${insertColumns.map(() => "?").join(", ")}) ` +
+            `ON CONFLICT (bucket_start, layer, op) DO UPDATE SET ${updates}`),
+        prune: perTable((t) => `DELETE FROM ${t} WHERE bucket_start < ?`),
+        operations: perTable((t) => `SELECT layer, op, ${SUMS} FROM ${t} WHERE bucket_start >= ? GROUP BY layer, op`),
+        // `bucket_start - (bucket_start % ?)`, not `/ ? * ?`: node:sqlite binds JS numbers as REAL, so `/` would not truncate.
+        series: perTable((t) => `SELECT bucket_start - (bucket_start % ?) AS t, layer, ${SUMS} FROM ${t} WHERE bucket_start >= ? ` +
+            "GROUP BY t, layer ORDER BY t, layer"),
+    };
+}
+function num(row, key) {
+    const v = row[key];
+    return typeof v === "number" ? v : typeof v === "bigint" ? Number(v) : 0;
+}
+function str(row, key) {
+    const v = row[key];
+    return typeof v === "string" ? v : "";
+}
+function totalsOf(row) {
+    return {
+        count: num(row, "count"),
+        clientErrors: num(row, "client_errors"),
+        serverErrors: num(row, "server_errors"),
+        sumMs: num(row, "sum_ms"),
+        minMs: num(row, "min_ms"),
+        maxMs: num(row, "max_ms"),
+        buckets: BUCKET_COLUMNS.map((c) => num(row, c)),
+    };
+}
+function merge(into, from) {
+    into.count += from.count;
+    into.clientErrors += from.clientErrors;
+    into.serverErrors += from.serverErrors;
+    into.sumMs += from.sumMs;
+    into.minMs = Math.min(into.minMs, from.minMs);
+    into.maxMs = Math.max(into.maxMs, from.maxMs);
+    for (let i = 0; i < BUCKET_COUNT; i++)
+        into.buckets[i] = (into.buckets[i] ?? 0) + (from.buckets[i] ?? 0);
+}
+const round3 = (ms) => Math.round(ms * 1000) / 1000;
+function statsOf(t) {
+    return {
+        count: t.count,
+        clientErrors: t.clientErrors,
+        serverErrors: t.serverErrors,
+        avgMs: round3(t.count === 0 ? 0 : t.sumMs / t.count),
+        maxMs: round3(t.maxMs),
+        p50Ms: round3(quantile(t.buckets, 0.5, t.minMs, t.maxMs)),
+        p95Ms: round3(quantile(t.buckets, 0.95, t.minMs, t.maxMs)),
+        p99Ms: round3(quantile(t.buckets, 0.99, t.minMs, t.maxMs)),
+    };
+}
+const iso = (sec) => new Date(sec * 1000).toISOString();
+/**
+ * Per-minute call aggregates (count, errors, latency histogram) in SQLite under DATA_DIR, rolled up hourly.
+ * Observations buffer in memory and flush every FLUSH_INTERVAL_MS; a crash loses at most one interval.
+ */
+export class MetricsStore {
+    db;
+    statements;
+    retentionDays;
+    log;
+    now;
+    pending = new Map();
+    lastPruneSec;
+    closed = false;
+    timer;
+    constructor(db, statements, retentionDays, log, now) {
+        this.db = db;
+        this.statements = statements;
+        this.retentionDays = retentionDays;
+        this.log = log;
+        this.now = now;
+        this.lastPruneSec = Math.floor(now() / 1000);
+        this.prune(this.lastPruneSec);
+        this.timer = setInterval(() => this.flush(), FLUSH_INTERVAL_MS);
+        this.timer.unref();
+    }
+    static async open(options) {
+        // Loaded lazily: node:sqlite prints an ExperimentalWarning on load, so only processes that open storage pay it,
+        // not every `ok-fine --help` or stdio-proxy start.
+        const { DatabaseSync } = await import("node:sqlite");
+        const connect = () => {
+            let db;
+            try {
+                db = new DatabaseSync(options.path);
+                // A filesystem without WAL support keeps rollback journaling; the result is ignored.
+                db.exec("PRAGMA journal_mode = WAL; PRAGMA synchronous = NORMAL; PRAGMA busy_timeout = 1000");
+                const version = db.prepare("PRAGMA user_version").get();
+                if (!version || num(version, "user_version") !== SCHEMA_VERSION)
+                    createSchema(db);
+                return { db, statements: prepare(db) };
+            }
+            catch (err) {
+                if (db?.isOpen)
+                    db.close();
+                throw err;
+            }
+        };
+        let opened;
+        try {
+            opened = connect();
+        }
+        catch (err) {
+            options.log.warn({ err, path: options.path }, "metrics database unusable; recreating it");
+            for (const suffix of ["", "-wal", "-shm"])
+                rmSync(`${options.path}${suffix}`, { force: true });
+            opened = connect();
+        }
+        return new MetricsStore(opened.db, opened.statements, options.retentionDays, options.log, options.now ?? Date.now);
+    }
+    record(layer, op, startedAt, outcome) {
+        this.observe(layer, op, performance.now() - startedAt, outcome);
+    }
+    observe(layer, op, durationMs, outcome) {
+        if (this.closed)
+            return;
+        const minuteStart = Math.floor(this.now() / 60_000) * 60;
+        const key = `${minuteStart}\u0000${layer}\u0000${op}`;
+        let p = this.pending.get(key);
+        if (!p) {
+            p = {
+                minuteStart,
+                layer,
+                op,
+                count: 0,
+                clientErrors: 0,
+                serverErrors: 0,
+                sumMs: 0,
+                minMs: durationMs,
+                maxMs: durationMs,
+                buckets: new Array(BUCKET_COUNT).fill(0),
+            };
+            this.pending.set(key, p);
+        }
+        p.count++;
+        if (outcome === "client_error")
+            p.clientErrors++;
+        if (outcome === "server_error")
+            p.serverErrors++;
+        p.sumMs += durationMs;
+        p.minMs = Math.min(p.minMs, durationMs);
+        p.maxMs = Math.max(p.maxMs, durationMs);
+        const i = bucketIndex(durationMs);
+        p.buckets[i] = (p.buckets[i] ?? 0) + 1;
+    }
+    /** Writes buffered observations (and prunes when due) in one transaction. Never throws. */
+    flush() {
+        if (this.closed)
+            return;
+        const batch = this.pending;
+        this.pending = new Map();
+        const nowSec = Math.floor(this.now() / 1000);
+        const pruneDue = nowSec - this.lastPruneSec >= PRUNE_INTERVAL_SECONDS;
+        if (batch.size === 0 && !pruneDue)
+            return;
+        try {
+            this.db.exec("BEGIN");
+            for (const p of batch.values()) {
+                const values = [
+                    p.layer,
+                    p.op,
+                    p.count,
+                    p.clientErrors,
+                    p.serverErrors,
+                    p.sumMs,
+                    p.minMs,
+                    p.maxMs,
+                    ...p.buckets,
+                ];
+                this.statements.upsert.metrics_minute.run(p.minuteStart, ...values);
+                this.statements.upsert.metrics_hour.run(p.minuteStart - (p.minuteStart % 3600), ...values);
+            }
+            if (pruneDue)
+                this.prune(nowSec);
+            this.db.exec("COMMIT");
+        }
+        catch (err) {
+            if (this.db.isTransaction)
+                this.db.exec("ROLLBACK");
+            this.log.warn({ err }, "metrics flush failed; dropped one batch");
+        }
+    }
+    prune(nowSec) {
+        this.statements.prune.metrics_minute.run(nowSec - MINUTE_RETENTION_SECONDS);
+        this.statements.prune.metrics_hour.run(nowSec - this.retentionDays * 86_400);
+        this.lastPruneSec = nowSec;
+    }
+    report(window) {
+        this.flush();
+        const spec = WINDOWS[window];
+        const nowSec = Math.floor(this.now() / 1000);
+        const start = nowSec - spec.seconds;
+        const fromSec = start - (start % spec.stepSeconds);
+        const layerTotals = new Map();
+        const operations = [];
+        for (const row of this.statements.operations[spec.table].all(fromSec)) {
+            const layer = METRICS_LAYERS.find((l) => l === str(row, "layer"));
+            if (!layer)
+                continue;
+            const totals = totalsOf(row);
+            operations.push({ layer, op: str(row, "op"), ...statsOf(totals) });
+            const sum = layerTotals.get(layer);
+            if (sum)
+                merge(sum, totals);
+            else
+                layerTotals.set(layer, { ...totals, buckets: [...totals.buckets] });
+        }
+        operations.sort((a, b) => METRICS_LAYERS.indexOf(a.layer) - METRICS_LAYERS.indexOf(b.layer) ||
+            b.count - a.count ||
+            (a.op < b.op ? -1 : a.op > b.op ? 1 : 0));
+        const layers = METRICS_LAYERS.flatMap((layer) => {
+            const totals = layerTotals.get(layer);
+            return totals ? [{ layer, ...statsOf(totals) }] : [];
+        });
+        const series = [];
+        for (const row of this.statements.series[spec.table].all(spec.stepSeconds, fromSec)) {
+            const layer = METRICS_LAYERS.find((l) => l === str(row, "layer"));
+            if (!layer)
+                continue;
+            const stats = statsOf(totalsOf(row));
+            series.push({
+                at: iso(num(row, "t")),
+                layer,
+                count: stats.count,
+                serverErrors: stats.serverErrors,
+                avgMs: stats.avgMs,
+                p95Ms: stats.p95Ms,
+            });
+        }
+        return { window, from: iso(fromSec), to: iso(nowSec), stepSeconds: spec.stepSeconds, layers, operations, series };
+    }
+    /** Idempotent: stops the timer, flushes, and closes the database. */
+    close() {
+        if (this.closed)
+            return;
+        clearInterval(this.timer);
+        this.flush();
+        this.closed = true;
+        this.db.close();
+    }
+}
+//# sourceMappingURL=store.js.map

package/dist/metrics/types.js

@@ -0,0 +1,4 @@
+// Wire types of GET /api/v1/metrics. No imports: the web UI imports this module.
+export const METRICS_LAYERS = ["service", "storage"];
+export const METRICS_WINDOWS = ["1h", "24h", "7d", "30d"];
+//# sourceMappingURL=types.js.map

package/dist/okf/concept.js

@@ -1,5 +1,5 @@
 import { parseFrontmatter, splitFrontmatter } from "./frontmatter.js";
-import { extractLinks } from "./markdown.js";
+import { extractCrossProjectLinks, extractLinks } from "./markdown.js";
 import { blobRevision } from "./paths.js";
 import { displayTitle, effectiveStatus, generatedAt, generatedBy, lastVerifiedAt, staleAfter, trustTier, } from "./semantics.js";
 export function parseConcept(project, id, file, buf) {
@@ -26,6 +26,7 @@ export function parseConcept(project, id, file, buf) {
             generatedBy: null,
             lastVerifiedAt: null,
             outbound: [],
+            crossProjectOutbound: [],
         };
     }
     const parsed = parseFrontmatter(split.yaml);
@@ -49,6 +50,7 @@ export function parseConcept(project, id, file, buf) {
             generatedBy: null,
             lastVerifiedAt: null,
             outbound: [],
+            crossProjectOutbound: [],
         };
     }
     const { data } = parsed;
@@ -72,6 +74,7 @@ export function parseConcept(project, id, file, buf) {
             generatedBy: generatedBy(data),
             lastVerifiedAt: lastVerifiedAt(data),
             outbound: extractLinks(split.body, id),
+            crossProjectOutbound: extractCrossProjectLinks(split.body),
         };
     }
     const tags = Array.isArray(data.tags) ? data.tags.filter((t) => typeof t === "string") : [];
@@ -94,6 +97,7 @@ export function parseConcept(project, id, file, buf) {
         generatedBy: generatedBy(data),
         lastVerifiedAt: lastVerifiedAt(data),
         outbound: extractLinks(split.body, id),
+        crossProjectOutbound: extractCrossProjectLinks(split.body),
     };
 }
 //# sourceMappingURL=concept.js.map

package/dist/okf/lint.js

@@ -1,6 +1,6 @@
 import { posix } from "node:path";
 import { parseFrontmatter, splitFrontmatter } from "./frontmatter.js";
-import { computationBlocks, extractFootnoteLabels, extractLinks, hasLegacyCitations } from "./markdown.js";
+import { computationBlocks, extractCrossProjectLinks, extractFootnoteLabels, extractLinks, hasLegacyCitations, } from "./markdown.js";
 import { ISO_DATETIME, isStale, parseActor } from "./semantics.js";
 const COMMIT_RE = /^[0-9a-f]{7,64}$/i;
 const SHELL_META_OR_WHITESPACE = /[\s`$;|&<>()\\"'`]/;
@@ -371,6 +371,20 @@ export function lintConceptFile(path, text, ctx) {
             });
         }
     }
+    // Warning: cross-project outbound links
+    if (ctx.crossProjectConceptExists) {
+        const crossLinks = extractCrossProjectLinks(body);
+        for (const target of crossLinks) {
+            if (!ctx.crossProjectConceptExists(target.project, target.id)) {
+                issues.push({
+                    severity: "warning",
+                    code: "broken_cross_link",
+                    path,
+                    message: `cross-project link to "okf://${target.project}/${target.id}" does not exist`,
+                });
+            }
+        }
+    }
     // Info: stale
     if (isStale(data, ctx.now)) {
         issues.push({

package/dist/okf/markdown.js

@@ -1,8 +1,90 @@
 import { posix } from "node:path";
 import { marked } from "marked";
-import { isReservedName } from "./paths.js";
+import { isReservedName, PROJECT_RE, SEGMENT_RE } from "./paths.js";
 const SCHEME_RE = /^[a-z][a-z0-9+.-]*:/i;
 const FOOTNOTE_RE = /\[\^([^\]\s]+)\](?!:)/g;
+export function parseCrossProjectTarget(rawHref) {
+    if (!rawHref) {
+        return null;
+    }
+    let href = rawHref;
+    try {
+        href = decodeURI(href);
+    }
+    catch {
+        // ignore URI decode failures
+    }
+    if (href.includes("\\")) {
+        return null;
+    }
+    const trimmed = href.trim();
+    if (!trimmed.toLowerCase().startsWith("okf://")) {
+        return null;
+    }
+    let withoutHash = trimmed;
+    const hashIdx = withoutHash.indexOf("#");
+    if (hashIdx !== -1) {
+        withoutHash = withoutHash.slice(0, hashIdx);
+    }
+    const queryIdx = withoutHash.indexOf("?");
+    if (queryIdx !== -1) {
+        withoutHash = withoutHash.slice(0, queryIdx);
+    }
+    const rest = withoutHash.slice(6);
+    const slashIdx = rest.indexOf("/");
+    if (slashIdx === -1) {
+        return null;
+    }
+    const project = rest.slice(0, slashIdx);
+    if (!PROJECT_RE.test(project)) {
+        return null;
+    }
+    let target = rest.slice(slashIdx + 1);
+    if (target.endsWith(".md")) {
+        target = target.slice(0, -3);
+    }
+    if (target.length === 0 || target.length > 512) {
+        return null;
+    }
+    const segments = target.split("/");
+    if (segments.length > 12) {
+        return null;
+    }
+    for (const seg of segments) {
+        if (!SEGMENT_RE.test(seg)) {
+            return null;
+        }
+    }
+    const lastSeg = segments[segments.length - 1];
+    if (!lastSeg) {
+        return null;
+    }
+    const lastLower = lastSeg.toLowerCase();
+    if (lastLower === "index" || lastLower === "log") {
+        return null;
+    }
+    return { project, id: segments.join("/") };
+}
+export function extractCrossProjectLinks(body) {
+    const tokens = marked.lexer(body);
+    const found = [];
+    const seen = new Set();
+    marked.walkTokens(tokens, (token) => {
+        if (token.type !== "link" || token.raw.startsWith("[^")) {
+            return;
+        }
+        const target = parseCrossProjectTarget(token.href);
+        if (!target) {
+            return;
+        }
+        const key = `${target.project}/${target.id}`;
+        if (!seen.has(key)) {
+            seen.add(key);
+            found.push(target);
+        }
+    });
+    return found;
+}
 export function extractLinks(body, conceptId) {
     const tokens = marked.lexer(body);
     const found = [];

package/dist/server.js

@@ -1,3 +1,4 @@
+import { join } from "node:path";
 import { setTimeout as sleep } from "node:timers/promises";
 import { buildOAuthProtectedResourceMetadata, createMcpHandler, getOAuthProtectedResourceMetadataUrl, } from "@modelcontextprotocol/server";
 import Fastify, {} from "fastify";
@@ -7,10 +8,13 @@ import { createAuthenticator } from "./auth/http-auth.js";
 import { JwtTokenVerifier, principalFromAuthInfo } from "./auth/verifier.js";
 import { OkfError } from "./errors.js";
 import { registerMcpRoute } from "./http/mcp-route.js";
+import { registerMetricsRoutes } from "./http/metrics.js";
 import { registerRestRoutes } from "./http/rest.js";
 import { DEFAULT_UI_DIR, endpointOrigins, registerUiRoutes } from "./http/ui.js";
 import { createLocalHost } from "./local-host.js";
 import { createMcpServer } from "./mcp/server.js";
+import { InstrumentedStorage, instrumentService } from "./metrics/instrument.js";
+import { METRICS_DB_FILE, MetricsStore } from "./metrics/store.js";
 import { KnowledgeService } from "./service/knowledge-service.js";
 import { Catalog } from "./store/catalog.js";
 import { GitBackend } from "./store/git-backend.js";
@@ -205,10 +209,8 @@ export async function startServer(config, options = {}) {
             },
         };
     }
-    const storage = await GitBackend.open(config, app.log);
-    const catalog = new Catalog();
-    const service = new KnowledgeService({ config, storage, catalog, log: app.log });
-    await service.initialize();
+    const knowledge = await openKnowledge(config, app.log);
+    const service = knowledge.service;
     const mcpHandler = createMcpHandler(({ authInfo }) => {
         if (!authInfo)
             throw new Error("MCP request reached the handler without authInfo");
@@ -270,6 +272,7 @@ export async function startServer(config, options = {}) {
         });
         registerMcpRoute(secured, (request, options) => mcpHandler.fetch(request, options), config.publicBaseUrl);
         registerRestRoutes(secured, service);
+        registerMetricsRoutes(secured, knowledge.metrics);
     });
     const stopSync = startPeriodicSync(config, service, app.log);
     let localHost;
@@ -283,7 +286,7 @@ export async function startServer(config, options = {}) {
     catch (err) {
         stopSync();
         await localHost?.close();
-        await storage.close();
+        await knowledge.close();
         await mcpHandler.close();
         throw err;
     }
@@ -294,11 +297,43 @@ export async function startServer(config, options = {}) {
             stopSync();
             await localHost?.close();
             await app.close();
-            await storage.close();
+            await knowledge.close();
             await mcpHandler.close();
         },
     };
 }
+/** Opens git storage, the metrics database under DATA_DIR, and the instrumented service on top of them. */
+async function openKnowledge(config, log) {
+    // Creates DATA_DIR, which the metrics database lives in.
+    const git = await GitBackend.open(config, log);
+    let metrics;
+    try {
+        metrics = await MetricsStore.open({
+            path: join(config.dataDir, METRICS_DB_FILE),
+            retentionDays: config.metricsRetentionDays,
+            log,
+        });
+    }
+    catch (err) {
+        await git.close();
+        throw err;
+    }
+    const storage = new InstrumentedStorage(git, metrics);
+    const close = async () => {
+        await storage.close();
+        metrics.close();
+    };
+    // Not instrumented yet: the startup catalog rebuild records storage calls but no service op.
+    const raw = new KnowledgeService({ config, storage, catalog: new Catalog(), log });
+    try {
+        await raw.initialize();
+    }
+    catch (err) {
+        await close();
+        throw err;
+    }
+    return { service: instrumentService(raw, metrics), metrics, close };
+}
 /** Runs `service.syncNow()` every GIT_SYNC_INTERVAL_SECONDS when a remote is configured; returns a stop function. */
 function startPeriodicSync(config, service, log) {
     if (!config.gitRemoteUrl || config.gitSyncIntervalSeconds <= 0)
@@ -326,10 +361,8 @@ function startPeriodicSync(config, service, log) {
 /** Starts the local endpoint host that listens on a domain socket / named pipe. */
 export async function startLocalHost(config, options) {
     const { log } = options;
-    const storage = await GitBackend.open(config, log);
-    const catalog = new Catalog();
-    const service = new KnowledgeService({ config, storage, catalog, log });
-    await service.initialize();
+    const knowledge = await openKnowledge(config, log);
+    const service = knowledge.service;
     const stopSync = startPeriodicSync(config, service, log);
     const host = await createLocalHost({ service, config, log });
     let closing;
@@ -340,7 +373,7 @@ export async function startLocalHost(config, options) {
                 stopSync();
                 await host.close();
                 // Waits for the storage mutex, so an in-flight write still commits.
-                await storage.close();
+                await knowledge.close();
             })();
             return closing;
         },