@elvisw/dsh-magic-context 0.0.0-stage → 0.13.0

package/lib/dsh/telemetry.js

@@ -0,0 +1,349 @@
+import { SCHEMA_VERSION } from "../core/stores.js";
+const DAY_MS = 24 * 60 * 60_000;
+/**
+ * Normalize the `?session=` query value.
+ * @param raw - the request URL's first `session` parameter.
+ * @returns the Session id, or undefined for absent/empty (whole-store scope).
+ */
+export function sessionOf(raw) {
+    if (raw === null || raw === undefined)
+        return undefined;
+    const value = raw.trim();
+    return value === '' ? undefined : value;
+}
+export function createTelemetryService(deps) {
+    const now = deps.now ?? Date.now;
+    const pollMs = deps.pollMs ?? 2_000;
+    const streams = new Set();
+    /**
+     * Browser halves that ever attached, and when the newest one did.
+     *
+     * `sseClients` in the summary is the only machine-readable proof that a
+     * browser half is actually connected and receiving journal rows — without it
+     * "is the UI showing anything?" can only be answered by looking at the screen.
+     */
+    let attachedTotal = 0;
+    let lastAttachedAt;
+    /** The newest journal row of a scope, or undefined. */
+    const newestOf = (sessionId) => {
+        try {
+            return deps.stores.listJournal({
+                limit: 1,
+                ...sessionId === undefined ? {} : { sessionId },
+            })[0];
+        }
+        catch {
+            return undefined;
+        }
+    };
+    /** The newest `memory-recall` row of a scope, or undefined. */
+    const latestRecall = (sessionId) => {
+        try {
+            return deps.stores.listJournal({
+                kinds: ['memory-recall'],
+                limit: 1,
+                ...sessionId === undefined ? {} : { sessionId },
+            })[0];
+        }
+        catch {
+            return undefined;
+        }
+    };
+    /**
+     * The project this scope is working on, read from the newest recall: the
+     * Session's own when one is named, the most recent one overall otherwise.
+     */
+    const project = (sessionId) => {
+        const recall = latestRecall(sessionId);
+        const identity = recall?.data?.identity;
+        const memories = recall?.data?.rows;
+        return {
+            ...typeof identity === 'string' ? { identity } : {},
+            ...typeof memories === 'number' ? { memories } : {},
+        };
+    };
+    const health = (sessionId) => {
+        const since = now() - DAY_MS;
+        // Errors reported by the RUNNING process, not by the last day. The pill is a
+        // live health badge: after a restart it must not keep flagging failures that
+        // the current build already fixed (the historian token-cap failures stayed
+        // red for a day until this existed).
+        const bootSince = deps.startedAt === undefined ? since : Math.max(deps.startedAt, since);
+        /** First-wins capture: journal rows iterate newest-first, so the newest
+         * reading for a field simply arrives first. */
+        const firstNum = (current, value) => current === undefined && typeof value === 'number' ? value : current;
+        // 24h counters: kind → aggregate field (single scan; error level handled inline).
+        const KIND_COUNTS = {
+            pass: 'passes24h',
+            'drop-landed': 'drops24h',
+            'compartment-published': 'compartments24h',
+            'emergency-reclaim': 'emergency24h',
+            'memory-promote': 'promote24h',
+        };
+        const h = { passes24h: 0, drops24h: 0, compartments24h: 0, emergency24h: 0, promote24h: 0 };
+        let errors24h = 0;
+        let errorsSinceBoot = 0;
+        let reclaimed24h = 0;
+        let lastPressure;
+        let lastTokens;
+        let lastContextWindow;
+        let surfaceTokens;
+        let surfaceWindow;
+        try {
+            for (const row of deps.stores.journalByKind(since, sessionId)) {
+                if (row.level === 'error')
+                    errors24h += row.count;
+                const field = KIND_COUNTS[row.kind];
+                if (field !== undefined)
+                    h[field] = (h[field] ?? 0) + row.count;
+            }
+            for (const row of deps.stores.journalByKind(bootSince, sessionId)) {
+                if (row.level === 'error')
+                    errorsSinceBoot += row.count;
+            }
+            // The pill's one number: the newest pressure reading, plus the 24h net
+            // reclaim the execute band actually produced (sum of negative deltas).
+            for (const entry of deps.stores.listJournal({ limit: 300, ...(sessionId === undefined ? {} : { sessionId }) })) {
+                if (entry.kind === 'context-pressure' && entry.data !== undefined) {
+                    // The native meter's own numbers (contextPressure projection): the
+                    // projected occupancy answers the NEXT request, while the pass
+                    // ledger below only moves when a management pass runs. Preferred
+                    // whenever a reading exists — newest row wins by iteration order.
+                    lastTokens = firstNum(lastTokens, entry.data.projectedTokens);
+                    lastContextWindow = firstNum(lastContextWindow, entry.data.contextWindow);
+                    if (lastPressure === undefined && typeof entry.data.projectedTokens === 'number' && typeof entry.data.contextWindow === 'number' && entry.data.contextWindow > 0) {
+                        lastPressure = entry.data.projectedTokens / entry.data.contextWindow * 100;
+                    }
+                    continue;
+                }
+                if (entry.kind !== 'pass' || entry.data === undefined)
+                    continue;
+                lastPressure = firstNum(lastPressure, entry.data.pressure);
+                // The post-compaction SURFACE account (execute passes only), separate
+                // from the projected occupancy above: different denominators of truth.
+                if (entry.data.decision === 'execute') {
+                    surfaceTokens = firstNum(surfaceTokens, entry.data.tokensBefore);
+                    surfaceWindow = firstNum(surfaceWindow, entry.data.contextWindow);
+                    const delta = entry.data.tokensDelta;
+                    if (typeof delta === 'number' && delta < 0)
+                        reclaimed24h += -delta;
+                }
+                lastTokens = firstNum(lastTokens, entry.data.tokensBefore);
+                lastContextWindow = firstNum(lastContextWindow, entry.data.contextWindow);
+            }
+        }
+        catch {
+            /* a locked or missing table must not break the UI */
+        }
+        // Bounds are global-store facts: read once, serve both the scoped and the
+        // whole-store fields (a failed read leaves them at zero — diagnostic only).
+        let boundsRows = 0;
+        let boundsNewest;
+        try {
+            const bounds = deps.stores.journalBounds();
+            boundsRows = bounds.rows;
+            boundsNewest = bounds.newest;
+        }
+        catch {
+            /* diagnostic fields degrade to zero */
+        }
+        const journalRows = sessionId === undefined
+            ? boundsRows
+            : (() => {
+                try {
+                    return deps.stores.journalByKind(undefined, sessionId).reduce((total, row) => total + row.count, 0);
+                }
+                catch {
+                    return 0;
+                }
+            })();
+        const lastEventAt = sessionId === undefined ? boundsNewest : newestOf(sessionId)?.at;
+        let memoryStats;
+        let maintenanceLast;
+        try {
+            // The KPI must count the SAME project the header reports, or a global
+            // summary shows one project's identity above another project's totals
+            // (measured: identity git:93eea8… with 101 rows that belonged to
+            // git:a81f6372…, because the fallback picked the first-seen identity of a
+            // process serving several sessions). `project()` resolves it from the
+            // newest recall of this scope — the same source as `project.identity`.
+            const scopeIdentity = project(sessionId).identity;
+            memoryStats = deps.memoryStats?.(sessionId, typeof scopeIdentity === 'string' ? scopeIdentity : undefined);
+            maintenanceLast = deps.maintenanceLast?.();
+        }
+        catch {
+            /* stats are panel decoration; never let them break the summary */
+        }
+        return {
+            journalRows,
+            journalRowsAll: boundsRows,
+            lastEventAt,
+            passes24h: h.passes24h,
+            errors24h,
+            errorsSinceBoot,
+            drops24h: h.drops24h,
+            compartments24h: h.compartments24h,
+            emergency24h: h.emergency24h,
+            promote24h: h.promote24h,
+            reclaimed24h,
+            lastPressure,
+            lastTokens,
+            lastContextWindow,
+            // Frozen summary shape (telemetry.spec asserts it): same value as
+            // project.memories, kept for JSON-consumer compatibility.
+            memoriesRecalled: typeof latestRecall(sessionId)?.data?.rows === 'number'
+                ? latestRecall(sessionId)?.data?.rows
+                : 0,
+            // The post-compaction SURFACE account (newest execute pass), separate
+            // from the projected occupancy: the panel labels the two explicitly.
+            ...(surfaceTokens !== undefined ? { surfaceLedger: { tokens: surfaceTokens, ...(surfaceWindow !== undefined ? { window: surfaceWindow } : {}) } } : {}),
+            ...(memoryStats !== undefined ? { memoryStats } : {}),
+            ...(maintenanceLast !== undefined ? { maintenanceLast } : {}),
+        };
+    };
+    const recent = (limit, sessionId) => {
+        try {
+            // Newest-first from the store; the UI wants chronological order.
+            return deps.stores.listJournal({
+                limit,
+                ...sessionId === undefined ? {} : { sessionId },
+            })
+                .map(entry => ({
+                at: entry.at,
+                level: entry.level,
+                kind: entry.kind,
+                summary: entry.summary,
+                ...entry.sessionId === undefined ? {} : { sessionId: entry.sessionId },
+                // The record model reads numbers off `data` (inserted/ms/pressure);
+                // without it the snapshot strip falls back to the raw English
+                // summary while the live stream renders properly.
+                ...entry.data === undefined ? {} : { data: entry.data },
+            }))
+                .reverse();
+        }
+        catch {
+            return [];
+        }
+    };
+    const summary = (sessionId) => ({
+        scope: sessionId === undefined ? 'global' : 'session',
+        ...sessionId === undefined ? {} : { sessionId },
+        adapter: {
+            schemaVersion: SCHEMA_VERSION,
+            uptimeMs: deps.startedAt === undefined ? undefined : now() - deps.startedAt,
+            sseClients: streams.size,
+            sseAttachedTotal: attachedTotal,
+            ...lastAttachedAt === undefined ? {} : { sseLastAttachedAt: lastAttachedAt },
+            ...deps.meta(),
+        },
+        project: project(sessionId),
+        health: health(sessionId),
+        recent: recent(20, sessionId),
+    });
+    const sendJson = (res, status, payload) => {
+        const body = JSON.stringify(payload);
+        res.writeHead(status, {
+            'content-type': 'application/json; charset=utf-8',
+            'cache-control': 'no-store',
+            'content-length': String(Buffer.byteLength(body)),
+        });
+        res.end(body);
+    };
+    const openStream = (res, sessionId) => {
+        res.writeHead(200, {
+            'content-type': 'text/event-stream; charset=utf-8',
+            'cache-control': 'no-store',
+            connection: 'keep-alive',
+            'x-accel-buffering': 'no',
+        });
+        const write = (chunk) => {
+            try {
+                res.write?.(chunk);
+            }
+            catch {
+                /* the client went away */
+            }
+        };
+        write(`event: snapshot\ndata: ${JSON.stringify(summary(sessionId))}\n\n`);
+        // Poll the journal rather than holding a query open: reads are cheap, the
+        // writer is a different connection, and a restart of the store is invisible.
+        // The cursor starts at this scope's newest row, so a Session stream never
+        // replays another Session's history.
+        let lastSeen = 0;
+        try {
+            lastSeen = Number((sessionId === undefined
+                ? deps.stores.journalBounds().newest
+                : newestOf(sessionId)?.at) ?? 0);
+        }
+        catch {
+            lastSeen = 0;
+        }
+        const timer = setInterval(() => {
+            try {
+                const fresh = deps.stores.listJournal({
+                    since: lastSeen + 1,
+                    limit: 20,
+                    ...sessionId === undefined ? {} : { sessionId },
+                });
+                for (const entry of [...fresh].reverse()) {
+                    lastSeen = Math.max(lastSeen, entry.at);
+                    write(`event: journal\ndata: ${JSON.stringify({
+                        at: entry.at,
+                        level: entry.level,
+                        kind: entry.kind,
+                        summary: entry.summary,
+                        ...entry.sessionId === undefined ? {} : { sessionId: entry.sessionId },
+                        data: entry.data,
+                    })}\n\n`);
+                }
+            }
+            catch {
+                /* keep the stream alive on a transient read failure */
+            }
+        }, pollMs);
+        const keepAlive = setInterval(() => write(': keep-alive\n\n'), 15_000);
+        const stop = () => {
+            clearInterval(timer);
+            clearInterval(keepAlive);
+            streams.delete(stream);
+            try {
+                res.end();
+            }
+            catch {
+                /* already closed */
+            }
+        };
+        const stream = { stop };
+        streams.add(stream);
+        attachedTotal += 1;
+        lastAttachedAt = now();
+        res.on?.('close', stop);
+    };
+    return {
+        get openStreams() {
+            return streams.size;
+        },
+        summary,
+        handle: (req, res) => {
+            const base = '/mc-telemetry';
+            // A throwaway base keeps `new URL` total for an origin-form request target.
+            const requestUrl = new URL(req.url ?? '/', 'http://telemetry.invalid');
+            const url = requestUrl.pathname;
+            const path = url.startsWith(base) ? url.slice(base.length) || '/' : url;
+            const sessionId = sessionOf(requestUrl.searchParams.get('session'));
+            if (path === '/summary' || path === '/' || path === '') {
+                sendJson(res, 200, summary(sessionId));
+                return;
+            }
+            if (path === '/events') {
+                openStream(res, sessionId);
+                return;
+            }
+            sendJson(res, 404, { error: 'unknown telemetry route', routes: ['/summary', '/events'] });
+        },
+        close: () => {
+            for (const stream of [...streams])
+                stream.stop();
+        },
+    };
+}