apm-optima 1.0.4 → 2.0.0

package/dist/core/storage/persistence/segment-index.js

@@ -0,0 +1,206 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SegmentIndex = void 0;
+const fsp = __importStar(require("fs/promises"));
+const path_1 = __importDefault(require("path"));
+const file_buffer_1 = require("../utility/file-buffer");
+const persistence_writer_1 = require("./persistence-writer");
+const logger_1 = __importDefault(require("../../telemetry/logger"));
+const bucket_metric_1 = require("../stores/bucket-metric");
+const CATEGORY = 'metric_buckets';
+const FILE_RE = /^(\d{4}-\d{2}-\d{2})\.ndjson(\.gz)?$/;
+const todayUtc = () => new Date().toISOString().slice(0, 10);
+function touch(entries, date, b) {
+    const startMs = Date.parse(b.startTime);
+    const endMs = Date.parse(b.endTime);
+    if (Number.isNaN(startMs) || Number.isNaN(endMs)) {
+        return;
+    }
+    const e = entries.get(b.sessionId);
+    if (!e) {
+        entries.set(b.sessionId, {
+            date, sessionId: b.sessionId,
+            minSequence: b.sequence, maxSequence: b.sequence,
+            minStartMs: startMs, maxEndMs: endMs,
+            count: 1
+        });
+        return;
+    }
+    e.minSequence = Math.min(e.minSequence, b.sequence);
+    e.maxSequence = Math.max(e.maxSequence, b.sequence);
+    e.minStartMs = Math.min(e.minStartMs, startMs);
+    e.maxEndMs = Math.max(e.maxEndMs, endMs);
+    e.count++;
+}
+class SegmentIndex {
+    baseDir;
+    dir;
+    byDate = new Map();
+    dirty = new Set();
+    constructor(baseDir) {
+        this.baseDir = baseDir;
+        this.dir = path_1.default.join(baseDir, CATEGORY);
+    }
+    async rebuild() {
+        this.byDate.clear();
+        this.dirty.clear();
+        const files = await fsp.readdir(this.dir).catch(() => []);
+        const dates = new Set();
+        for (const f of files) {
+            const m = FILE_RE.exec(f);
+            if (m)
+                dates.add(m[1]);
+        }
+        const today = todayUtc();
+        for (const date of [...dates].sort()) {
+            try {
+                await this.loadOrScan(date, date < today);
+            }
+            catch (err) {
+                logger_1.default.error(`SegmentIndex: failed to index ${date}: ${err}`);
+            }
+        }
+    }
+    noteWritten(bucket) {
+        const date = bucket.endTime.slice(0, 10);
+        let entries = this.byDate.get(date);
+        if (!entries) {
+            entries = new Map();
+            this.byDate.set(date, entries);
+        }
+        touch(entries, date, bucket);
+        this.dirty.add(date);
+    }
+    find(sessionId, fromMs, toMs) {
+        const dates = [];
+        for (const [date, entries] of this.byDate) {
+            const e = entries.get(sessionId);
+            if (e && e.minStartMs <= toMs && e.maxEndMs >= fromMs) {
+                dates.push(date);
+            }
+        }
+        return dates.sort();
+    }
+    async persistClosed() {
+        const today = todayUtc();
+        for (const date of [...this.dirty]) {
+            if (date >= today)
+                continue;
+            try {
+                await this.persist(date);
+                this.dirty.delete(date);
+            }
+            catch (err) {
+                logger_1.default.error(`SegmentIndex: failed to persist index for ${date}: ${err}`);
+            }
+        }
+    }
+    indexPath(date) {
+        return path_1.default.join(this.dir, `${date}.index.json`);
+    }
+    async resolveExistingFile(category, date) {
+        const dir = path_1.default.join(this.baseDir, category);
+        const plain = path_1.default.join(dir, `${date}.ndjson`);
+        const gz = `${plain}.gz`;
+        if (await fsp.access(plain).then(() => true).catch(() => false))
+            return plain;
+        if (await fsp.access(gz).then(() => true).catch(() => false))
+            return gz;
+        return null;
+    }
+    async loadOrScan(date, closed) {
+        const file = await this.resolveExistingFile(CATEGORY, date);
+        if (!file)
+            return;
+        const size = (await fsp.stat(file)).size;
+        if (closed) {
+            const cached = await this.tryLoad(date, path_1.default.basename(file), size);
+            if (cached) {
+                this.byDate.set(date, cached);
+                return;
+            }
+        }
+        const entries = await this.scan(file, date);
+        this.byDate.set(date, entries);
+        if (closed) {
+            await this.persist(date).catch(err => logger_1.default.error(`SegmentIndex: persist failed for ${date}: ${err}`));
+        }
+        else {
+            this.dirty.add(date);
+        }
+    }
+    async tryLoad(date, fileName, fileSize) {
+        try {
+            const raw = JSON.parse(await fsp.readFile(this.indexPath(date), 'utf-8'));
+            if (raw.version !== 1 || raw.date !== date || raw.fileName !== fileName || raw.fileSize !== fileSize) {
+                return null;
+            }
+            return new Map(raw.entries.map(e => [e.sessionId, e]));
+        }
+        catch {
+            return null;
+        }
+    }
+    async scan(file, date) {
+        const entries = new Map();
+        for await (const rec of (0, persistence_writer_1.dedupeRecords)((0, file_buffer_1.readRecords)(file))) {
+            const b = rec.payload;
+            if (!b || b.schemaVersion !== bucket_metric_1.BUCKET_SCHEMA_VERSION || typeof b.sessionId !== 'string')
+                continue;
+            touch(entries, date, b);
+        }
+        return entries;
+    }
+    async persist(date) {
+        const entries = this.byDate.get(date);
+        const file = await this.resolveExistingFile(CATEGORY, date);
+        if (!entries || !file)
+            return;
+        const size = (await fsp.stat(file)).size;
+        const data = {
+            version: 1,
+            date,
+            fileName: path_1.default.basename(file),
+            fileSize: size,
+            entries: [...entries.values()]
+        };
+        await (0, file_buffer_1.writeJSONAtomic)(this.indexPath(date), data);
+    }
+}
+exports.SegmentIndex = SegmentIndex;

package/dist/core/storage/persistence/session-registry.d.ts

@@ -0,0 +1,28 @@
+import type { SessionRecord } from '../../domain';
+import type { PersistenceWriter } from './persistence-writer';
+export interface SessionWindow {
+    start: Date;
+    end: Date;
+}
+export declare class SessionRegistry {
+    private readonly manifestPath;
+    private readonly writer;
+    private manifest;
+    private dirty;
+    private _current;
+    private _recoveredFromCrash;
+    constructor(manifestPath: string, writer: PersistenceWriter);
+    get current(): SessionRecord | null;
+    get all(): readonly SessionRecord[];
+    get recoveredFromCrash(): boolean;
+    start(sessionId?: string): Promise<SessionRecord>;
+    notePersisted(committedSequence?: number): void;
+    heartbeat(): Promise<void>;
+    complete(): Promise<void>;
+    static windowEnd(s: SessionRecord): string | null;
+    static window(s: SessionRecord): SessionWindow;
+    static enumerateDates({ start, end }: SessionWindow): string[];
+    static enumerateHours({ start, end }: SessionWindow): string[];
+    private save;
+    private migrate;
+}

package/dist/core/storage/persistence/session-registry.js

@@ -0,0 +1,127 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SessionRegistry = void 0;
+const crypto_1 = require("crypto");
+const index_1 = require("../utility/index");
+const emptyManifest = () => ({ version: 2, sessions: [] });
+class SessionRegistry {
+    manifestPath;
+    writer;
+    manifest = emptyManifest();
+    dirty = false;
+    _current = null;
+    _recoveredFromCrash = false;
+    constructor(manifestPath, writer) {
+        this.manifestPath = manifestPath;
+        this.writer = writer;
+    }
+    get current() { return this._current; }
+    get all() { return this.manifest.sessions; }
+    get recoveredFromCrash() { return this._recoveredFromCrash; }
+    async start(sessionId = (0, crypto_1.randomUUID)()) {
+        const raw = await (0, index_1.tryOrDefault)(() => (0, index_1.readJSON)(this.manifestPath), null);
+        this.manifest = this.migrate(raw);
+        for (const s of this.manifest.sessions) {
+            if (s.status === 'RUNNING') {
+                s.status = 'INTERRUPTED';
+                s.endedAt = s.lastPersistedAt ?? s.startedAt;
+                this._recoveredFromCrash = true;
+            }
+        }
+        const number = this.manifest.sessions.reduce((m, s) => Math.max(m, s.sessionNumber), 0) + 1;
+        this._current = {
+            sessionId: sessionId,
+            sessionNumber: number,
+            startedAt: new Date().toISOString(),
+            endedAt: null,
+            lastPersistedAt: null,
+            lastCommittedSequence: 0,
+            status: 'RUNNING',
+        };
+        this.manifest.sessions.push(this._current);
+        await this.save('manifest: session start');
+        return this._current;
+    }
+    notePersisted(committedSequence) {
+        if (!this._current)
+            return;
+        this._current.lastPersistedAt = new Date().toISOString();
+        if (committedSequence !== undefined) {
+            this._current.lastCommittedSequence = Math.max(this._current.lastCommittedSequence, committedSequence);
+        }
+        this.dirty = true;
+    }
+    async heartbeat() {
+        if (!this.dirty)
+            return;
+        this.dirty = false;
+        await this.save('manifest: heartbeat').catch(() => { this.dirty = true; });
+    }
+    async complete() {
+        if (!this._current)
+            return;
+        await this.writer.whenIdle();
+        const now = new Date().toISOString();
+        this._current.status = 'COMPLETED';
+        this._current.endedAt = now;
+        this._current.lastPersistedAt = now;
+        this.dirty = false;
+        await this.save('manifest: session complete');
+    }
+    static windowEnd(s) {
+        return s.status === 'RUNNING' ? null : (s.endedAt ?? s.lastPersistedAt);
+    }
+    static window(s) {
+        const end = SessionRegistry.windowEnd(s);
+        return {
+            start: new Date(s.startedAt),
+            end: end ? new Date(end) : new Date(),
+        };
+    }
+    static enumerateDates({ start, end }) {
+        const dates = [];
+        const cursor = new Date(Date.UTC(start.getUTCFullYear(), start.getUTCMonth(), start.getUTCDate()));
+        const last = new Date(Date.UTC(end.getUTCFullYear(), end.getUTCMonth(), end.getUTCDate()));
+        while (cursor <= last) {
+            dates.push(cursor.toISOString().slice(0, 10));
+            cursor.setUTCDate(cursor.getUTCDate() + 1);
+        }
+        return dates;
+    }
+    static enumerateHours({ start, end }) {
+        const hours = [];
+        const cursor = new Date(Date.UTC(start.getUTCFullYear(), start.getUTCMonth(), start.getUTCDate(), start.getUTCHours()));
+        const last = new Date(Date.UTC(end.getUTCFullYear(), end.getUTCMonth(), end.getUTCDate(), end.getUTCHours()));
+        while (cursor <= last) {
+            hours.push(cursor.toISOString());
+            cursor.setUTCHours(cursor.getUTCHours() + 1);
+        }
+        return hours;
+    }
+    save(description) {
+        const snapshot = structuredClone(this.manifest);
+        return this.writer.enqueueTask(description, () => (0, index_1.writeJSONAtomic)(this.manifestPath, snapshot));
+    }
+    migrate(raw) {
+        if (raw?.version === 2 && Array.isArray(raw.sessions)) {
+            for (const s of raw.sessions) {
+                if (typeof s.lastCommittedSequence !== 'number') {
+                    s.lastCommittedSequence = typeof s.lastCommitedSequence === 'number' ? s.lastCommitedSequence : 0;
+                }
+                delete s.lastCommitedSequence;
+            }
+            return raw;
+        }
+        const sessions = (raw?.sessionHistory ?? []).map((h) => ({
+            sessionId: `legacy-${h.sessionNumber}`,
+            sessionNumber: h.sessionNumber,
+            startedAt: h.startedAt,
+            endedAt: h.endedAt ?? null,
+            lastPersistedAt: h.endedAt ?? null,
+            lastCommittedSequence: 0,
+            status: h.endedAt ? 'COMPLETED' : 'INTERRUPTED',
+        }));
+        return { version: 2, sessions };
+    }
+}
+exports.SessionRegistry = SessionRegistry;

package/dist/core/storage/persistence/session-summary.d.ts

@@ -0,0 +1,3 @@
+import type { Incident, SessionRecord, SessionSummary } from "../../domain";
+import type { TelemetryQueryService } from "../../telemetry/telemetry-query.service";
+export declare function buildSessionSummary(queries: TelemetryQueryService, session: SessionRecord, windowHours: number, slowLatencyMs: number, incidents?: Incident[]): Promise<SessionSummary>;

package/dist/core/storage/persistence/session-summary.js

@@ -0,0 +1,112 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildSessionSummary = buildSessionSummary;
+const session_registry_1 = require("./session-registry");
+const bucket_metric_1 = require("../stores/bucket-metric");
+const HOUR_MS = 60 * 60 * 1000;
+const TARGET_POINTS = 60;
+const TOP_ROUTES = 5;
+const RESOLUTIONS_MS = [
+    1_000, 5_000, 10_000, 30_000,
+    60_000, 5 * 60_000, 10 * 60_000, 15 * 60_000, 30 * 60_000,
+    HOUR_MS, 3 * HOUR_MS, 6 * HOUR_MS, 12 * HOUR_MS, 24 * HOUR_MS,
+];
+function pickResolution(spanMs) {
+    return RESOLUTIONS_MS.find(r => spanMs / r <= TARGET_POINTS) ?? RESOLUTIONS_MS[RESOLUTIONS_MS.length - 1];
+}
+const rate = (part, total) => total > 0 ? part / total : 0;
+async function buildSessionSummary(queries, session, windowHours, slowLatencyMs, incidents = []) {
+    const window = session_registry_1.SessionRegistry.window(session);
+    const startCandidate = new Date(window.end.getTime() - windowHours * HOUR_MS);
+    if (startCandidate > window.start)
+        window.start = startCandidate;
+    const resolutionMs = pickResolution(window.end.getTime() - window.start.getTime());
+    const result = await queries.query({
+        sessionId: session.sessionId,
+        from: window.start.toISOString(),
+        to: window.end.toISOString(),
+        resolutionMs,
+        includeEndpoints: true,
+        consistency: session.status === 'RUNNING' ? 'live' : 'committed',
+    });
+    const routes = new Map();
+    const points = result.points.map(p => {
+        for (const e of p.endpoints ?? []) {
+            const key = `${e.method}:${e.route}`;
+            let r = routes.get(key);
+            if (!r) {
+                r = { method: e.method, route: e.route, agg: (0, bucket_metric_1.createAggregate)() };
+                routes.set(key, r);
+            }
+            (0, bucket_metric_1.mergeInto)(r.agg, e);
+        }
+        const m = (0, bucket_metric_1.deriveMetrics)(p.requests, p.durationMs);
+        return {
+            startTime: p.startTime,
+            durationMs: p.durationMs,
+            requestCount: m.requestCount,
+            rps: m.rps,
+            maxRps: p.maxRps,
+            p95: m.p95,
+            serverErrorRate: rate(m.serverErrorCount, m.requestCount),
+            clientErrorCount: m.clientErrorCount,
+            serverErrorCount: m.serverErrorCount,
+        };
+    });
+    const topRoutes = [...routes.values()]
+        .map(({ method, route, agg }) => ({
+        method,
+        route,
+        requestCount: agg.requestCount,
+        impactedRequests: agg.serverErrorCount + (0, bucket_metric_1.countSlowerThan)(agg, slowLatencyMs),
+        p95: (0, bucket_metric_1.deriveMetrics)(agg, 1000).p95,
+        serverErrorRate: rate(agg.serverErrorCount, agg.requestCount),
+    }))
+        .filter(r => r.impactedRequests > 0)
+        .sort((a, b) => b.impactedRequests - a.impactedRequests || b.requestCount - a.requestCount)
+        .slice(0, TOP_ROUTES);
+    let missingIntervals = 0;
+    let conflicts = 0;
+    for (const issue of result.issues) {
+        if (issue.type === 'GAP')
+            missingIntervals += issue.toSequence - issue.fromSequence + 1;
+        else if (issue.type === 'CONFLICT')
+            conflicts++;
+    }
+    const t = result.total.derived;
+    return {
+        sessionId: session.sessionId,
+        sessionNumber: session.sessionNumber,
+        status: session.status,
+        startedAt: session.startedAt,
+        endedAt: session.endedAt,
+        windowStart: window.start.toISOString(),
+        windowEnd: window.end.toISOString(),
+        measuredMs: result.total.durationMs,
+        traffic: {
+            requestCount: t.requestCount,
+            avgRps: t.rps,
+            peakRps: result.total.maxRps,
+        },
+        latency: {
+            p50: t.p50,
+            p95: t.p95,
+            p99: t.p99,
+            max: t.maxLatency,
+        },
+        errors: {
+            serverErrorCount: t.serverErrorCount,
+            serverErrorRate: rate(t.serverErrorCount, t.requestCount),
+            clientErrorCount: t.clientErrorCount,
+        },
+        incidents,
+        topRoutes,
+        series: { resolutionMs, points },
+        quality: {
+            bucketCount: result.total.bucketCount,
+            missingIntervals,
+            conflicts,
+            complete: result.complete,
+        },
+    };
+}

package/dist/core/storage/persistence.layer.d.ts

@@ -0,0 +1,69 @@
+import { type ReadonlyConfig } from "../../config";
+import type { ApplicationEvent, SessionSummary, SessionRecord, SessionManifest } from "../domain";
+import { PersistenceWriter, type StorageStatus } from './persistence/persistence-writer';
+import { type MetricBucket } from "./stores/bucket-metric";
+import type { CommittedBucketSource, MetricsQuery, QueryIssue, TelemetryQueryService } from "../telemetry/telemetry-query.service";
+import type { CorrelationFinding } from "../telemetry/utility/correlation-finding";
+type HttpMetricRecord = {
+    method: string;
+    path: string;
+    statusCode: number;
+    durationMs: number;
+    timestamp: string;
+};
+export declare class PersistenceRepository {
+    private readonly conf;
+    private httpBuffer;
+    private readonly enabled;
+    private readonly maxBufferSize;
+    private readonly baseDir;
+    private readonly persistRawRequests;
+    private readonly heartbeatIntervalMs;
+    private readonly writer;
+    private readonly registry;
+    private readonly ready;
+    private heartbeatTimer;
+    private shutdownPromise;
+    private closing;
+    private readonly inFlight;
+    readonly sessionId: `${string}-${string}-${string}-${string}-${string}`;
+    private readonly segments;
+    private queries;
+    constructor(conf: ReadonlyConfig);
+    attachQueryService(q: TelemetryQueryService): void;
+    get isEnabled(): boolean;
+    get bucketSource(): CommittedBucketSource;
+    private initSession;
+    private track;
+    private drainInFlight;
+    private enqueue;
+    onHttpRequest(metric: HttpMetricRecord): void;
+    flushHttpBuffer(): Promise<void>;
+    onBucketClosed(bucket: MetricBucket): Promise<boolean>;
+    saveCorrelationFinding(finding: CorrelationFinding): Promise<boolean>;
+    onApplicationEvent(event: ApplicationEvent): Promise<void>;
+    getStorageStatus(): {
+        status: StorageStatus;
+        lostRecords: number;
+        pending: number;
+        lastError: PersistenceWriter['lastError'];
+    };
+    getSessionManifest(): Promise<SessionManifest>;
+    getSessionRecord(): SessionRecord | null;
+    findSession(sessionNumber: number): Promise<SessionRecord | null>;
+    private resolveExistingFile;
+    private requireQueries;
+    sessionQuery(session: SessionRecord): MetricsQuery;
+    getSessionSummary(sessionNumber: number): Promise<SessionSummary | null>;
+    private sessionIncidents;
+    findCorrelationFinding(findingId: string): Promise<CorrelationFinding | null>;
+    readBuckets(sessionId: string, fromMs: number, toMs: number, onIssue: (i: QueryIssue) => void): AsyncGenerator<MetricBucket>;
+    shutdown(): Promise<void>;
+    private archiveOldCategory;
+    archiveAllCategories(): void;
+    private writeChunk;
+    private streamJsonArray;
+    private sessionRecords;
+    streamSessionExport(out: NodeJS.WritableStream, sessionNumber: number): Promise<boolean>;
+}
+export {};