crewly 1.20.194 → 1.20.196

package/dist/backend/backend/src/services/trace/trace-store.js

@@ -0,0 +1,714 @@
+/**
+ * Run trace store: append-only `<CREWLY_HOME>/traces/<traceId>.jsonl` files
+ * plus a small `index.json` for listing and ref lookups.
+ *
+ * - Bookkeeping (index, refs, size cap) is synchronous and in memory; file
+ *   writes are chained on one promise and never awaited by callers.
+ * - A trace takes at most MAX_EVENTS_PER_TRACE events / MAX_BYTES_PER_TRACE
+ *   bytes; the event that would cross a limit becomes one `trace.truncated`
+ *   marker and later events are dropped.
+ * - Traces idle for RETENTION_MS are deleted by a sweep that runs on the first
+ *   write after boot and then at most once per SWEEP_INTERVAL_MS; the same
+ *   sweep (and the disk janitor) prunes the oldest traces when the folder is
+ *   over MAX_TOTAL_BYTES.
+ * - Nothing here throws to a caller: a failed write is logged and counted.
+ *
+ * specs/2026-10-03-run-traces.md
+ *
+ * @module services/trace/trace-store
+ */
+import { promises as fsp, readFileSync } from 'fs';
+import * as path from 'path';
+import { TRACE_CONSTANTS } from '../../constants.js';
+import { getCrewlyHomePath } from '../core/crewly-home.utils.js';
+import { LoggerService } from '../core/logger.service.js';
+import { isTraceId, traceRefKey, } from './trace.types.js';
+/** The real file system. */
+const NODE_FS = {
+    mkdir: (dir, opts) => fsp.mkdir(dir, opts),
+    appendFile: (file, data) => fsp.appendFile(file, data, 'utf8'),
+    writeFile: (file, data) => fsp.writeFile(file, data, 'utf8'),
+    rename: (from, to) => fsp.rename(from, to),
+    readFile: (file) => fsp.readFile(file, 'utf8'),
+    unlink: (file) => fsp.unlink(file),
+    readdir: (dir) => fsp.readdir(dir),
+    stat: (file) => fsp.stat(file),
+};
+/**
+ * Cap on the traces folder: `CREWLY_TRACES_MAX_TOTAL_MB`, else
+ * {@link TRACE_CONSTANTS.MAX_TOTAL_BYTES}.
+ *
+ * @returns Bytes
+ */
+export function defaultMaxTotalBytes() {
+    const mb = Number(process.env.CREWLY_TRACES_MAX_TOTAL_MB);
+    return Number.isFinite(mb) && mb > 0 ? Math.floor(mb * 1024 * 1024) : TRACE_CONSTANTS.MAX_TOTAL_BYTES;
+}
+/**
+ * Copy an index entry (root and tags are copied too).
+ *
+ * @param e - Entry
+ * @returns Copy
+ */
+function copyEntry(e) {
+    const out = { ...e, root: { ...e.root } };
+    if (e.tags) {
+        out.tags = {
+            ...(e.tags.autopilot ? { autopilot: { ...e.tags.autopilot } } : {}),
+            ...(e.tags.labels ? { labels: [...e.tags.labels] } : {}),
+        };
+    }
+    return out;
+}
+/**
+ * Whether an entry matches the tag filters of a list call.
+ *
+ * @param e - Entry
+ * @param filter - Filter
+ * @returns True when every given tag filter matches
+ */
+function matchesTags(e, filter) {
+    if (filter.autopilotProjectId && e.tags?.autopilot?.projectId !== filter.autopilotProjectId)
+        return false;
+    if (filter.day && e.tags?.autopilot?.day !== filter.day)
+        return false;
+    if (filter.label) {
+        const wanted = filter.label.toLowerCase();
+        if (!(e.tags?.labels ?? []).some((l) => l.toLowerCase() === wanted))
+            return false;
+    }
+    return true;
+}
+/**
+ * The trace store. One per process (see {@link getTraceStore}).
+ */
+export class TraceStore {
+    logger;
+    fs;
+    now;
+    maxEvents;
+    maxBytes;
+    retentionMs;
+    sweepIntervalMs;
+    indexFlushDelayMs;
+    maxTotalBytes;
+    explicitDir;
+    resolvedDir = null;
+    index = null;
+    chain = Promise.resolve();
+    indexTimer = null;
+    dirReady = false;
+    sweepScheduled = false;
+    sweeping = null;
+    failures = 0;
+    /**
+     * @param options - Folder, clock, limits, file ops
+     */
+    constructor(options = {}) {
+        this.logger = LoggerService.getInstance().createComponentLogger('TraceStore');
+        this.fs = options.fs ?? NODE_FS;
+        this.now = options.now ?? (() => new Date());
+        this.maxEvents = options.maxEvents ?? TRACE_CONSTANTS.MAX_EVENTS_PER_TRACE;
+        this.maxBytes = options.maxBytes ?? TRACE_CONSTANTS.MAX_BYTES_PER_TRACE;
+        this.retentionMs = options.retentionMs ?? TRACE_CONSTANTS.RETENTION_MS;
+        this.sweepIntervalMs = options.sweepIntervalMs ?? TRACE_CONSTANTS.SWEEP_INTERVAL_MS;
+        this.indexFlushDelayMs = options.indexFlushDelayMs ?? TRACE_CONSTANTS.INDEX_FLUSH_DELAY_MS;
+        this.maxTotalBytes = options.maxTotalBytes ?? defaultMaxTotalBytes();
+        this.explicitDir = options.dir;
+    }
+    /**
+     * The traces folder.
+     *
+     * @returns Absolute path
+     */
+    get dir() {
+        if (!this.resolvedDir)
+            this.resolvedDir = this.explicitDir ?? path.join(getCrewlyHomePath(), TRACE_CONSTANTS.DIR_NAME);
+        return this.resolvedDir;
+    }
+    /** Writes that failed since the store was created. */
+    get writeFailures() {
+        return this.failures;
+    }
+    // ---------------------------------------------------------------------------
+    // Writes (fire-and-forget)
+    // ---------------------------------------------------------------------------
+    /**
+     * Register a new trace and write its `trace.root` event.
+     *
+     * @param root - The root (its traceId must be new and well-formed)
+     * @returns True when registered
+     */
+    createRoot(root) {
+        try {
+            if (!isTraceId(root.traceId))
+                return false;
+            const index = this.loadIndex();
+            if (index.traces[root.traceId])
+                return false;
+            index.traces[root.traceId] = {
+                traceId: root.traceId,
+                root,
+                updatedAt: root.createdAt,
+                eventCount: 0,
+                bytes: 0,
+                truncated: false,
+            };
+            this.append({
+                ts: root.createdAt,
+                traceId: root.traceId,
+                type: 'trace.root',
+                actor: root.actor,
+                refs: root.refs,
+                summary: root.summary,
+                outcome: 'info',
+                data: { kind: root.kind },
+            });
+            return true;
+        }
+        catch (err) {
+            this.noteFailure('createRoot', err);
+            return false;
+        }
+    }
+    /**
+     * Append one event. Unknown traces and events past the cap are dropped.
+     *
+     * @param event - The event
+     * @returns True when it was queued for writing
+     */
+    append(event) {
+        try {
+            const index = this.loadIndex();
+            const entry = index.traces[event.traceId];
+            if (!entry || entry.truncated)
+                return false;
+            let line = `${JSON.stringify(event)}\n`;
+            let bytes = Buffer.byteLength(line);
+            if (entry.eventCount + 1 > this.maxEvents || entry.bytes + bytes > this.maxBytes) {
+                const marker = {
+                    ts: event.ts,
+                    traceId: event.traceId,
+                    type: 'trace.truncated',
+                    actor: { kind: 'system' },
+                    refs: {},
+                    summary: `Trace reached its size limit (${entry.eventCount} events, ${entry.bytes} bytes); later events are dropped`,
+                    outcome: 'info',
+                };
+                line = `${JSON.stringify(marker)}\n`;
+                bytes = Buffer.byteLength(line);
+                entry.truncated = true;
+            }
+            entry.eventCount += 1;
+            entry.bytes += bytes;
+            if (event.ts > entry.updatedAt)
+                entry.updatedAt = event.ts;
+            const file = this.fileOf(event.traceId);
+            this.enqueue(async () => {
+                await this.ensureDir();
+                await this.fs.appendFile(file, line);
+            }, 'append');
+            this.scheduleIndexFlush();
+            this.maybeSweep();
+            return !entry.truncated || event.type === 'trace.truncated';
+        }
+        catch (err) {
+            this.noteFailure('append', err);
+            return false;
+        }
+    }
+    /**
+     * Remember which trace an entity belongs to. The first link wins: an
+     * entity is never moved to another trace.
+     *
+     * @param kind - Entity kind
+     * @param id - Entity id
+     * @param traceId - Trace
+     */
+    linkRef(kind, id, traceId) {
+        try {
+            if (!id || !isTraceId(traceId))
+                return;
+            const index = this.loadIndex();
+            if (!index.traces[traceId])
+                return;
+            const key = traceRefKey(kind, id);
+            if (index.refs[key])
+                return;
+            index.refs[key] = traceId;
+            this.scheduleIndexFlush();
+        }
+        catch (err) {
+            this.noteFailure('linkRef', err);
+        }
+    }
+    /**
+     * Tag a trace (index only). The autopilot tag is set once (the first tag
+     * wins: a ticket's trace keeps the day its work started); labels are
+     * merged, case-insensitively, up to MAX_TAG_LABELS.
+     *
+     * @param traceId - Trace
+     * @param tags - Tags to add
+     * @returns True when the entry changed
+     */
+    tag(traceId, tags) {
+        try {
+            if (!isTraceId(traceId))
+                return false;
+            const entry = this.loadIndex().traces[traceId];
+            if (!entry)
+                return false;
+            let changed = false;
+            const next = { ...(entry.tags ?? {}) };
+            if (tags.autopilot && !next.autopilot && tags.autopilot.projectId && tags.autopilot.day) {
+                next.autopilot = { projectId: tags.autopilot.projectId, day: tags.autopilot.day };
+                changed = true;
+            }
+            const labels = [...(next.labels ?? [])];
+            for (const raw of tags.labels ?? []) {
+                const label = String(raw).trim().slice(0, TRACE_CONSTANTS.DATA_VALUE_MAX_CHARS);
+                if (!label || labels.length >= TRACE_CONSTANTS.MAX_TAG_LABELS)
+                    continue;
+                if (labels.some((l) => l.toLowerCase() === label.toLowerCase()))
+                    continue;
+                labels.push(label);
+                changed = true;
+            }
+            if (labels.length > 0)
+                next.labels = labels;
+            if (!changed)
+                return false;
+            entry.tags = next;
+            this.scheduleIndexFlush();
+            return true;
+        }
+        catch (err) {
+            this.noteFailure('tag', err);
+            return false;
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // Reads
+    // ---------------------------------------------------------------------------
+    /**
+     * The trace an entity belongs to.
+     *
+     * @param kind - Entity kind
+     * @param id - Entity id
+     * @returns Trace id, or null
+     */
+    traceByRef(kind, id) {
+        if (!id)
+            return null;
+        try {
+            const traceId = this.loadIndex().refs[traceRefKey(kind, id)];
+            return traceId && this.loadIndex().traces[traceId] ? traceId : null;
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Whether a trace exists.
+     *
+     * @param traceId - Trace id
+     * @returns True when indexed
+     */
+    has(traceId) {
+        try {
+            return isTraceId(traceId) && !!this.loadIndex().traces[traceId];
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * The index entry of a trace.
+     *
+     * @param traceId - Trace id
+     * @returns Copy of the entry, or null
+     */
+    getEntry(traceId) {
+        if (!this.has(traceId))
+            return null;
+        return copyEntry(this.loadIndex().traces[traceId]);
+    }
+    /**
+     * Traces, most recently active first.
+     *
+     * @param filter - Since / root kind / limit
+     * @returns Index entries (copies)
+     */
+    list(filter = {}) {
+        const since = filter.since ? filter.since.toISOString() : null;
+        const limit = Math.max(1, Math.min(filter.limit ?? TRACE_CONSTANTS.DEFAULT_LIST_LIMIT, TRACE_CONSTANTS.MAX_LIST_LIMIT));
+        let entries;
+        try {
+            entries = Object.values(this.loadIndex().traces);
+        }
+        catch {
+            return [];
+        }
+        return entries
+            .filter((e) => (!since || e.updatedAt >= since) && (!filter.rootKind || e.root.kind === filter.rootKind) && matchesTags(e, filter))
+            .sort((a, b) => (a.updatedAt < b.updatedAt ? 1 : a.updatedAt > b.updatedAt ? -1 : 0))
+            .slice(0, limit)
+            .map(copyEntry);
+    }
+    /**
+     * Every trace tagged with an autopilot project (no row limit: the stats
+     * read a whole window). Most recently active first.
+     *
+     * @param filter - `autopilotProjectId` (required), since / day / label
+     * @returns Index entries (copies); empty without a project
+     * @throws When the index cannot be read
+     */
+    listTagged(filter) {
+        if (!filter.autopilotProjectId)
+            return [];
+        const since = filter.since ? filter.since.toISOString() : null;
+        // An unreadable index throws here (unlike list): the stats must not
+        // mistake a read failure for a day without autopilot work.
+        const entries = Object.values(this.loadIndex().traces);
+        return entries
+            .filter((e) => (!since || e.updatedAt >= since) && (!filter.rootKind || e.root.kind === filter.rootKind) && matchesTags(e, filter))
+            .sort((a, b) => (a.updatedAt < b.updatedAt ? 1 : a.updatedAt > b.updatedAt ? -1 : 0))
+            .map(copyEntry);
+    }
+    /**
+     * One page of a trace's events, oldest first. Waits for queued writes.
+     *
+     * @param traceId - Trace id
+     * @param offset - Events to skip
+     * @param limit - Page size
+     * @returns The page, or null when the trace is unknown
+     */
+    async read(traceId, offset = 0, limit = TRACE_CONSTANTS.DEFAULT_PAGE_SIZE) {
+        const all = await this.readAll(traceId);
+        if (!all)
+            return null;
+        const start = Math.max(0, offset);
+        const size = Math.max(1, Math.min(limit, TRACE_CONSTANTS.MAX_PAGE_SIZE));
+        return {
+            root: all.root,
+            events: all.events.slice(start, start + size),
+            total: all.events.length,
+            offset: start,
+            limit: size,
+            truncated: all.truncated,
+        };
+    }
+    /**
+     * Every event of a trace in one file read and one parse (metrics, timeline,
+     * summary). A torn last line (crash mid-append) is skipped.
+     *
+     * @param traceId - Trace id (validated by the caller)
+     * @returns Root, all events and the truncated flag, or null when unknown
+     */
+    async readAll(traceId) {
+        const entry = this.getEntry(traceId);
+        if (!entry)
+            return null;
+        await this.flush();
+        let raw = '';
+        try {
+            raw = await this.fs.readFile(this.fileOf(traceId));
+        }
+        catch (err) {
+            this.logger.debug('Trace file could not be read', { traceId, error: errText(err) });
+        }
+        const events = [];
+        for (const line of raw.split('\n')) {
+            if (!line.trim())
+                continue;
+            try {
+                events.push(JSON.parse(line));
+            }
+            catch {
+                // A torn last line (crash mid-append) is skipped.
+            }
+        }
+        return { root: entry.root, events, truncated: entry.truncated };
+    }
+    // ---------------------------------------------------------------------------
+    // Retention + persistence
+    // ---------------------------------------------------------------------------
+    /**
+     * Delete traces whose last event is older than the retention, their refs,
+     * and orphan `.jsonl` files older than the retention.
+     *
+     * @returns What was removed
+     */
+    async sweep() {
+        const result = { removedTraces: 0, removedFiles: 0 };
+        try {
+            const index = this.loadIndex();
+            const nowMs = this.now().getTime();
+            const cutoff = new Date(nowMs - this.retentionMs).toISOString();
+            const expired = new Set(Object.values(index.traces).filter((e) => e.updatedAt < cutoff).map((e) => e.traceId));
+            for (const id of expired)
+                delete index.traces[id];
+            for (const [key, traceId] of Object.entries(index.refs)) {
+                if (expired.has(traceId) || !index.traces[traceId])
+                    delete index.refs[key];
+            }
+            index.lastSweepAt = new Date(nowMs).toISOString();
+            result.removedTraces = expired.size;
+            await this.flush();
+            let files = [];
+            try {
+                files = await this.fs.readdir(this.dir);
+            }
+            catch {
+                files = [];
+            }
+            for (const name of files) {
+                if (!name.endsWith('.jsonl'))
+                    continue;
+                const id = name.slice(0, -'.jsonl'.length);
+                const file = path.join(this.dir, name);
+                let remove = expired.has(id);
+                if (!remove && !index.traces[id]) {
+                    const stat = await this.fs.stat(file).catch(() => null);
+                    remove = !!stat && stat.mtimeMs < nowMs - this.retentionMs;
+                }
+                if (!remove)
+                    continue;
+                await this.fs.unlink(file).then(() => {
+                    result.removedFiles += 1;
+                }, (err) => this.logger.debug('Expired trace file could not be removed', { file, error: errText(err) }));
+            }
+            await this.writeIndex();
+            if (result.removedTraces || result.removedFiles)
+                this.logger.info('Trace retention sweep', { ...result });
+            await this.enforceTotalCap();
+        }
+        catch (err) {
+            this.noteFailure('sweep', err);
+        }
+        return result;
+    }
+    /**
+     * Size of the traces folder, and — unless `dryRun` — delete the least
+     * recently active traces until it is under the cap. Run by the daily sweep
+     * and the disk janitor. Never throws.
+     *
+     * @param opts.capBytes - Cap (default: the store's)
+     * @param opts.dryRun - Only report what would be pruned
+     * @returns Size and what was pruned
+     */
+    async enforceTotalCap(opts = {}) {
+        const capBytes = opts.capBytes ?? this.maxTotalBytes;
+        const result = { bytes: 0, files: 0, capBytes, prunedTraces: 0, freedBytes: 0 };
+        try {
+            await this.flush();
+            const index = this.loadIndex();
+            let names = [];
+            try {
+                names = await this.fs.readdir(this.dir);
+            }
+            catch {
+                return result;
+            }
+            const files = [];
+            for (const name of names) {
+                const file = path.join(this.dir, name);
+                const stat = await this.fs.stat(file).catch(() => null);
+                if (!stat)
+                    continue;
+                const size = stat.size ?? 0;
+                result.bytes += size;
+                if (!name.endsWith('.jsonl'))
+                    continue;
+                result.files += 1;
+                const id = name.slice(0, -'.jsonl'.length);
+                files.push({ id, file, size, at: index.traces[id]?.updatedAt ?? new Date(stat.mtimeMs).toISOString() });
+            }
+            if (result.bytes <= capBytes)
+                return result;
+            files.sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
+            let remaining = result.bytes;
+            for (const f of files) {
+                if (remaining <= capBytes)
+                    break;
+                remaining -= f.size;
+                result.prunedTraces += 1;
+                result.freedBytes += f.size;
+                if (opts.dryRun)
+                    continue;
+                delete index.traces[f.id];
+                await this.fs.unlink(f.file).catch((err) => this.noteFailure('prune', err));
+            }
+            if (!opts.dryRun && result.prunedTraces > 0) {
+                for (const [key, traceId] of Object.entries(index.refs)) {
+                    if (!index.traces[traceId])
+                        delete index.refs[key];
+                }
+                await this.writeIndex();
+                this.logger.info('Traces folder over its cap — oldest traces pruned', { ...result });
+            }
+        }
+        catch (err) {
+            this.noteFailure('enforceTotalCap', err);
+        }
+        return result;
+    }
+    /**
+     * Wait for queued writes and write the index now.
+     */
+    async flush() {
+        if (this.indexTimer) {
+            clearTimeout(this.indexTimer);
+            this.indexTimer = null;
+            this.enqueue(() => this.writeIndex(), 'index');
+        }
+        await this.chain;
+    }
+    /**
+     * Wait for queued writes and any automatic sweep in progress (tests / shutdown).
+     */
+    async idle() {
+        await this.flush();
+        if (this.sweeping)
+            await this.sweeping;
+        await this.flush();
+    }
+    /**
+     * Stop the index timer (tests / shutdown). Pending writes still finish.
+     */
+    dispose() {
+        if (this.indexTimer)
+            clearTimeout(this.indexTimer);
+        this.indexTimer = null;
+    }
+    // ---------------------------------------------------------------------------
+    // Internals
+    // ---------------------------------------------------------------------------
+    /**
+     * Path of a trace's event file.
+     *
+     * @param traceId - A well-formed id
+     * @returns Absolute path
+     */
+    fileOf(traceId) {
+        return path.join(this.dir, `${traceId}.jsonl`);
+    }
+    /**
+     * The in-memory index, read from disk on first use.
+     *
+     * @returns The index
+     */
+    loadIndex() {
+        if (this.index)
+            return this.index;
+        let loaded = null;
+        try {
+            const parsed = JSON.parse(readFileSync(path.join(this.dir, TRACE_CONSTANTS.INDEX_FILE), 'utf8'));
+            if (parsed && typeof parsed === 'object' && parsed.traces && parsed.refs) {
+                loaded = {
+                    version: TRACE_CONSTANTS.INDEX_VERSION,
+                    ...(parsed.lastSweepAt ? { lastSweepAt: parsed.lastSweepAt } : {}),
+                    traces: parsed.traces,
+                    refs: parsed.refs,
+                };
+            }
+        }
+        catch {
+            loaded = null;
+        }
+        this.index = loaded ?? { version: TRACE_CONSTANTS.INDEX_VERSION, traces: {}, refs: {} };
+        return this.index;
+    }
+    /** Create the folder once. */
+    async ensureDir() {
+        if (this.dirReady)
+            return;
+        await this.fs.mkdir(this.dir, { recursive: true });
+        this.dirReady = true;
+    }
+    /** Write the index atomically (temp file + rename). */
+    async writeIndex() {
+        const index = this.loadIndex();
+        await this.ensureDir();
+        const file = path.join(this.dir, TRACE_CONSTANTS.INDEX_FILE);
+        const tmp = `${file}.${process.pid}.tmp`;
+        await this.fs.writeFile(tmp, JSON.stringify(index));
+        await this.fs.rename(tmp, file);
+    }
+    /** Write the index after a short delay, once per burst of changes. */
+    scheduleIndexFlush() {
+        if (this.indexTimer)
+            return;
+        this.indexTimer = setTimeout(() => {
+            this.indexTimer = null;
+            this.enqueue(() => this.writeIndex(), 'index');
+        }, this.indexFlushDelayMs);
+        this.indexTimer.unref?.();
+    }
+    /** Run the retention sweep when it is due (first write after boot, then daily). */
+    maybeSweep() {
+        if (this.sweepScheduled)
+            return;
+        const last = this.loadIndex().lastSweepAt;
+        const due = !last || this.now().getTime() - Date.parse(last) >= this.sweepIntervalMs;
+        if (!due)
+            return;
+        this.sweepScheduled = true;
+        this.sweeping = this.chain
+            .then(() => this.sweep())
+            .then(() => undefined)
+            .finally(() => {
+            this.sweepScheduled = false;
+            this.sweeping = null;
+        });
+    }
+    /**
+     * Chain a write after the previous one; a failure is logged and counted.
+     *
+     * @param op - The write
+     * @param what - Label for the log
+     */
+    enqueue(op, what) {
+        this.chain = this.chain.then(op).catch((err) => this.noteFailure(what, err));
+    }
+    /**
+     * Count and log (debug) a failure.
+     *
+     * @param what - Operation
+     * @param err - Error
+     */
+    noteFailure(what, err) {
+        this.failures += 1;
+        this.logger.debug('Trace write failed (ignored)', { what, error: errText(err) });
+    }
+}
+/**
+ * Error text.
+ *
+ * @param err - Anything thrown
+ * @returns Message
+ */
+function errText(err) {
+    return err instanceof Error ? err.message : String(err);
+}
+let instance = null;
+/**
+ * The process-wide trace store.
+ *
+ * @returns The store (created on first use)
+ */
+export function getTraceStore() {
+    if (!instance)
+        instance = new TraceStore();
+    return instance;
+}
+/**
+ * Replace the process-wide store (tests).
+ *
+ * @param store - Store, or null to rebuild the default lazily
+ */
+export function setTraceStoreForTesting(store) {
+    instance?.dispose();
+    instance = store;
+}
+//# sourceMappingURL=trace-store.js.map