apm-optima 1.0.4 → 2.0.0

package/dist/core/domain/telemetry.types.d.ts

@@ -5,4 +5,5 @@ export interface TelemetryRequest {
     statusCode: number;
     responseTime: number;
     clientIp?: string;
+    origin?: 'real' | 'synthetic';
 }

package/dist/core/organizers/event.manager.d.ts

@@ -0,0 +1,19 @@
+import { PersistenceRepository } from '../storage/persistence.layer';
+import { ApplicationEvent } from '../domain';
+type AppPartialEvent = Omit<ApplicationEvent, 'timestamp' | 'applicationVersion'>;
+type AsyncListener = (event: ApplicationEvent) => void | Promise<void>;
+type ApplicationEventManagerInstance = Omit<ApplicationEventManager, 'start' | 'stop'>;
+export declare class ApplicationEventManager {
+    private persistence;
+    private applicationVersion;
+    static instance: ApplicationEventManagerInstance | null;
+    private listeners;
+    private initilized;
+    constructor(persistence: PersistenceRepository, applicationVersion?: string);
+    start(): void;
+    stop(): void;
+    on(listener: AsyncListener): void;
+    off(listener: AsyncListener): void;
+    emit(event: AppPartialEvent): Promise<void>;
+}
+export {};

package/dist/core/organizers/event.manager.js

@@ -0,0 +1,73 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ApplicationEventManager = void 0;
+const logger_1 = __importDefault(require("../telemetry/logger"));
+class ApplicationEventManager {
+    persistence;
+    applicationVersion;
+    static instance = null;
+    listeners = [];
+    initilized = false;
+    constructor(persistence, applicationVersion = '') {
+        this.persistence = persistence;
+        this.applicationVersion = applicationVersion;
+        if (ApplicationEventManager.instance) {
+            throw new Error('ApplicationEventManager is a singleton class. Use ApplicationEventManager.getInstance() to access the instance.');
+        }
+        ApplicationEventManager.instance = this;
+    }
+    start() {
+        if (this.initilized)
+            return;
+        this.initilized = true;
+        this.on((event) => this.persistence.onApplicationEvent(event));
+    }
+    stop() {
+        if (!this.initilized)
+            return;
+        for (const listener of this.listeners) {
+            this.off(listener);
+        }
+        this.listeners = [];
+        this.initilized = false;
+    }
+    on(listener) {
+        this.listeners.push(listener);
+    }
+    off(listener) {
+        this.listeners = this.listeners.filter(l => l !== listener);
+    }
+    async emit(event) {
+        if (!this.initilized)
+            return;
+        const fullEvent = {
+            timestamp: new Date().toISOString(),
+            type: event.type,
+            applicationVersion: this.applicationVersion,
+            reason: event.reason || '',
+            ...(event.details ? { details: event.details } : {}),
+        };
+        // Using Promise.allSettled to ensure all listeners are called, even if some fail
+        // But this is causing strange race conditions, so we will call them sequentially for now
+        // const results = await Promise.allSettled(
+        //     this.listeners.map(listener => Promise.resolve(listener(fullEvent)))
+        // );
+        // for (const result of results) {
+        //     if (result.status === 'rejected') {
+        //         Logger.error('Error in event listener:', result.reason);
+        //     }
+        // }
+        for (const listener of this.listeners) {
+            try {
+                await listener(fullEvent);
+            }
+            catch (error) {
+                logger_1.default.error('Error in event listener:', error);
+            }
+        }
+    }
+}
+exports.ApplicationEventManager = ApplicationEventManager;

package/dist/core/organizers/index.d.ts

@@ -0,0 +1,2 @@
+export * from './interval.manager';
+export * from './event.manager';

package/dist/core/organizers/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./interval.manager"), exports);
+__exportStar(require("./event.manager"), exports);

package/dist/core/organizers/interval.manager.d.ts

@@ -0,0 +1,18 @@
+import type { ReadonlyConfig } from "../../config";
+interface TickCallback {
+    (): void;
+}
+interface TickCallbackAsync {
+    (): Promise<void>;
+}
+type IntervalType = 'tick' | 'publisher' | 'persistence';
+export declare class IntervalManager {
+    private readonly config;
+    private intervals;
+    private initialized;
+    private started;
+    constructor(callbacks: Record<IntervalType, TickCallback | TickCallbackAsync | null>, config: ReadonlyConfig);
+    startIntervals(): void;
+    stopIntervals(): void;
+}
+export {};

package/dist/core/organizers/interval.manager.js

@@ -0,0 +1,54 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IntervalManager = void 0;
+class IntervalManager {
+    config;
+    intervals = { tick: null, publisher: null, persistence: null, };
+    initialized = false;
+    started = false;
+    constructor(callbacks, config) {
+        this.config = config;
+        this.intervals.tick = callbacks.tick ? {
+            id: null,
+            callback: callbacks.tick,
+            intervalMs: config.tickIntervalMs,
+        } : null;
+        this.intervals.publisher = callbacks.publisher ? {
+            id: null,
+            callback: callbacks.publisher,
+            intervalMs: config.publisher.intervalMs,
+        } : null;
+        if (config.persistence !== false) {
+            this.intervals.persistence = callbacks.persistence ? {
+                id: null,
+                callback: callbacks.persistence,
+                intervalMs: config.persistence.archiveIntervalMs,
+            } : null;
+        }
+        this.initialized = true;
+    }
+    startIntervals() {
+        if (!this.initialized || this.started)
+            return;
+        for (const intervalType in this.intervals) {
+            const interval = this.intervals[intervalType];
+            if (interval && interval.callback) {
+                interval.id = setInterval(interval.callback, interval.intervalMs);
+            }
+        }
+        this.started = true;
+    }
+    stopIntervals() {
+        if (!this.initialized || !this.started)
+            return;
+        for (const intervalType in this.intervals) {
+            const interval = this.intervals[intervalType];
+            if (interval && interval.id) {
+                clearInterval(interval.id);
+                interval.id = null;
+            }
+        }
+        this.started = false;
+    }
+}
+exports.IntervalManager = IntervalManager;

package/dist/core/storage/index.d.ts

@@ -1,3 +1,4 @@
 export * from './stores';
 export * from './utility';
 export * from './local.repository';
+export * from './persistence.layer';

package/dist/core/storage/index.js

@@ -17,3 +17,4 @@ Object.defineProperty(exports, "__esModule", { value: true });
 __exportStar(require("./stores"), exports);
 __exportStar(require("./utility"), exports);
 __exportStar(require("./local.repository"), exports);
+__exportStar(require("./persistence.layer"), exports);

package/dist/core/storage/local.repository.d.ts

@@ -1,17 +1,17 @@
 import type { TelemetryRequest } from '../domain';
-import { BucketStore, DashboardStore, AnalyticsStore, HealthStore, SystemStore, EndpointStore, AlertStore, MetricsStore } from './stores';
-declare class LocalRepository {
-    readonly endpoint: EndpointStore;
-    readonly alerts: AlertStore;
-    readonly metrics: MetricsStore;
-    readonly health: HealthStore;
+import type { ReadonlyConfig } from '../../config';
+import { BucketStore } from './stores';
+import { PersistenceRepository } from './persistence.layer';
+import { RuntimeStore } from './stores/runtime.store';
+export declare class LocalRepository {
+    private readonly persistence;
+    private readonly conf;
     readonly bucket: BucketStore;
-    readonly dashboard: DashboardStore;
-    readonly analytics: AnalyticsStore;
-    readonly system: SystemStore;
-    constructor();
+    readonly runtime: RuntimeStore;
+    constructor(persistence: PersistenceRepository, conf: ReadonlyConfig);
     record(request: TelemetryRequest): void;
     tick(): void;
+    closeLastInterval(): Promise<void>;
+    dispose(): void;
+    private closeInterval;
 }
-export declare const storage: LocalRepository;
-export {};

package/dist/core/storage/local.repository.js

@@ -1,52 +1,52 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.storage = void 0;
-const config_1 = require("../../config");
+exports.LocalRepository = void 0;
+const crypto_1 = require("crypto");
 const stores_1 = require("./stores");
+const runtime_store_1 = require("./stores/runtime.store");
 class LocalRepository {
-    // Standard Stores
-    endpoint;
-    alerts;
-    metrics;
-    // Historics Store
-    health;
+    persistence;
+    conf;
     bucket;
-    // Core Stores
-    dashboard;
-    analytics;
-    system;
-    constructor() {
-        const config = (0, config_1.getConfig)();
-        // Standard 
-        this.endpoint = new stores_1.EndpointStore();
-        this.alerts = new stores_1.AlertStore(config.alertBufferSize);
-        this.metrics = new stores_1.MetricsStore();
-        // Historic
-        this.health = new stores_1.HealthStore(config.ringBufferSize, config.publisher.eventLoopLagThresholdMs);
-        this.bucket = new stores_1.BucketStore(this.endpoint, this.health, this.metrics, config.ringBufferSize);
-        this.dashboard = new stores_1.DashboardStore(this.bucket, this.health, this.endpoint, this.alerts);
-        this.analytics = new stores_1.AnalyticsStore(this.bucket, this.endpoint, config.publisher.slowLatencyThresholdMs);
-        this.system = new stores_1.SystemStore();
+    runtime;
+    constructor(persistence, conf) {
+        this.persistence = persistence;
+        this.conf = conf;
+        this.runtime = new runtime_store_1.RuntimeStore(conf);
+        this.bucket = new stores_1.BucketStore(this.runtime, {
+            sessionId: persistence.sessionId,
+            instanceId: (0, crypto_1.randomUUID)(),
+            intervalMs: conf.collection.bucketIntervalMs,
+            historySize: conf.cache.liveBuckets,
+            maxEndpoints: conf.collection.maxEndpointsPerBucket,
+        });
     }
     record(request) {
-        const data = {
+        this.bucket.record({
             method: request.method,
             route: request.endpoint,
             duration: request.responseTime,
-            statusCode: request.statusCode
-        };
-        this.endpoint.record(data);
-        const status = this.metrics.record({
-            duration: request.responseTime,
-            statusCode: request.statusCode
+            statusCode: request.statusCode,
+            origin: request.origin
+        });
+        this.persistence.onHttpRequest({
+            method: request.method,
+            path: request.endpoint,
+            statusCode: request.statusCode,
+            durationMs: request.responseTime,
+            timestamp: new Date(request.timestamp).toISOString()
         });
-        if (status === 'ANOMALY') {
-            this.alerts.warning(`Anomaly detected: ${request.method} ${request.endpoint} took ${request.responseTime}ms`);
-        }
-        this.bucket.recordRequest(request.method, request.endpoint);
     }
-    tick() {
-        this.bucket.flush();
+    tick() { void this.closeInterval(false); }
+    closeLastInterval() { return this.closeInterval(true); }
+    dispose() { this.runtime.dispose(); }
+    async closeInterval(final) {
+        const bucket = this.bucket.flush({ force: final, seal: final });
+        if (!bucket)
+            return;
+        const ok = await this.persistence.onBucketClosed(bucket);
+        if (ok)
+            this.bucket.markCommitted(bucket.sequence);
     }
 }
-exports.storage = new LocalRepository();
+exports.LocalRepository = LocalRepository;

package/dist/core/storage/persistence/persistence-writer.d.ts

@@ -0,0 +1,48 @@
+import { type StoredRecord } from '../utility/index';
+export type PersistenceCategory = 'http_requests' | 'events' | 'correlation' | 'metric_buckets';
+export type StorageStatus = 'OK' | 'DEGRADED';
+export interface WriterOptions {
+    maxQueue: number;
+    maxRetries: number;
+    retryDelayMs: number;
+    onStatusChange?: (status: StorageStatus) => void;
+}
+export interface RecordInput<T> {
+    recordId: string;
+    payload: T;
+}
+export declare class PersistenceWriter {
+    private readonly queue;
+    private runner;
+    private accepting;
+    private readonly maxQueue;
+    private readonly maxRetries;
+    private readonly retryDelayMs;
+    private readonly onStatusChange?;
+    private _status;
+    private _lostRecords;
+    private _completedJobs;
+    private _lastError;
+    constructor(opts: WriterOptions);
+    get status(): StorageStatus;
+    get lostRecords(): number;
+    get completedJobs(): number;
+    get lastError(): {
+        description: string;
+        message: string;
+        at: string;
+    } | null;
+    get isAccepting(): boolean;
+    get pending(): number;
+    enqueueAppend<T>(baseDir: string, category: PersistenceCategory, items: RecordInput<T>[], opts?: {
+        date?: Date;
+    }): Promise<number>;
+    enqueueTask(description: string, task: () => Promise<void>): Promise<number>;
+    whenIdle(): Promise<void>;
+    close(): Promise<void>;
+    private push;
+    private kick;
+    private drain;
+    private markLost;
+}
+export declare function dedupeRecords<T>(source: AsyncIterable<StoredRecord<T>>, onConflict?: (id: string) => void): AsyncGenerator<StoredRecord<T>>;

package/dist/core/storage/persistence/persistence-writer.js

@@ -0,0 +1,191 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PersistenceWriter = void 0;
+exports.dedupeRecords = dedupeRecords;
+const fsp = __importStar(require("fs/promises"));
+const logger_1 = __importDefault(require("../../telemetry/logger"));
+const index_1 = require("../utility/index");
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+class PersistenceWriter {
+    queue = [];
+    runner = null;
+    accepting = true;
+    maxQueue;
+    maxRetries;
+    retryDelayMs;
+    onStatusChange;
+    _status = 'OK';
+    _lostRecords = 0;
+    _completedJobs = 0;
+    _lastError = null;
+    constructor(opts) {
+        this.maxQueue = opts.maxQueue;
+        this.maxRetries = opts.maxRetries;
+        this.retryDelayMs = opts.retryDelayMs;
+        this.onStatusChange = opts.onStatusChange;
+    }
+    get status() { return this._status; }
+    get lostRecords() { return this._lostRecords; }
+    get completedJobs() { return this._completedJobs; }
+    get lastError() { return this._lastError; }
+    get isAccepting() { return this.accepting; }
+    get pending() { return this.queue.length; }
+    enqueueAppend(baseDir, category, items, opts = {}) {
+        if (items.length === 0)
+            return Promise.resolve(this._completedJobs);
+        const createdAt = new Date().toISOString();
+        const records = items.map(i => ({
+            id: i.recordId,
+            createdAt,
+            schemaVersion: 1,
+            category,
+            payload: i.payload
+        }));
+        let chunk;
+        try {
+            chunk = records.map(r => JSON.stringify(r)).join('\n') + '\n';
+        }
+        catch (err) {
+            this.markLost(items.length);
+            return Promise.reject(new Error(`Serialization failed for ${category}: ${err}`));
+        }
+        const filePath = (0, index_1.constructFilePath)({ baseDir, category, date: opts.date });
+        return this.push({
+            description: `Append ${records.length} records to ${filePath}`,
+            recordCount: items.length,
+            run: async (attempt) => {
+                await fsp.appendFile(filePath, attempt > 0 ? '\n' + chunk : chunk, 'utf-8');
+            },
+        });
+    }
+    enqueueTask(description, task) {
+        return this.push({ description, recordCount: 0, run: () => task() });
+    }
+    async whenIdle() {
+        while (this.runner)
+            await this.runner;
+    }
+    async close() {
+        this.accepting = false;
+        await this.whenIdle();
+    }
+    push(partial) {
+        if (!this.accepting) {
+            this.markLost(partial.recordCount);
+            return Promise.reject(new Error(`Writer closed; rejected: ${partial.description}`));
+        }
+        if (this.queue.length >= this.maxQueue) {
+            this.markLost(partial.recordCount);
+            logger_1.default.error(`PersistenceWriter: queue full (${this.maxQueue}); rejected: ${partial.description}`);
+            return Promise.reject(new Error('Persistence queue full'));
+        }
+        return new Promise((resolve, reject) => {
+            this.queue.push({ ...partial, resolve, reject });
+            this.kick();
+        });
+    }
+    kick() {
+        if (this.runner)
+            return;
+        this.runner = this.drain()
+            .catch(err => logger_1.default.error('PersistenceWriter: unexpected drain error', err))
+            .finally(() => {
+            this.runner = null;
+            if (this.queue.length > 0)
+                this.kick();
+        });
+    }
+    async drain() {
+        while (this.queue.length > 0) {
+            const job = this.queue[0];
+            let lastErr = null;
+            let ok = false;
+            for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
+                try {
+                    await job.run(attempt);
+                    ok = true;
+                    break;
+                }
+                catch (err) {
+                    lastErr = err;
+                    if (attempt < this.maxRetries)
+                        await sleep(this.retryDelayMs * 2 ** attempt);
+                }
+            }
+            this.queue.shift();
+            if (ok) {
+                this._completedJobs++;
+                job.resolve(this._completedJobs);
+            }
+            else {
+                this._lastError = {
+                    description: job.description,
+                    message: lastErr instanceof Error ? lastErr.message : String(lastErr),
+                    at: new Date().toISOString()
+                };
+                this.markLost(job.recordCount);
+                logger_1.default.error(`PersistenceWriter: giving up on "${job.description}":`, lastErr);
+                job.reject(lastErr instanceof Error ? lastErr : new Error(String(lastErr)));
+            }
+        }
+    }
+    markLost(count) {
+        this._lostRecords += count;
+        if (this._status !== 'DEGRADED') {
+            this._status = 'DEGRADED';
+            this.onStatusChange?.(this._status);
+        }
+    }
+}
+exports.PersistenceWriter = PersistenceWriter;
+async function* dedupeRecords(source, onConflict) {
+    const seen = new Map();
+    for await (const record of source) {
+        const fingerprint = JSON.stringify(record.payload);
+        const prev = seen.get(record.id);
+        if (prev === undefined) {
+            seen.set(record.id, fingerprint);
+            yield record;
+        }
+        else if (prev !== fingerprint) {
+            logger_1.default.error(`Record ID conflict: ${record.id}`);
+            onConflict?.(record.id);
+        }
+    }
+}

package/dist/core/storage/persistence/segment-index.d.ts

@@ -0,0 +1,27 @@
+import { type MetricBucket } from '../stores/bucket-metric';
+export interface SegmentEntry {
+    date: string;
+    sessionId: string;
+    minSequence: number;
+    maxSequence: number;
+    minStartMs: number;
+    maxEndMs: number;
+    count: number;
+}
+export declare class SegmentIndex {
+    private readonly baseDir;
+    private readonly dir;
+    private readonly byDate;
+    private readonly dirty;
+    constructor(baseDir: string);
+    rebuild(): Promise<void>;
+    noteWritten(bucket: MetricBucket): void;
+    find(sessionId: string, fromMs: number, toMs: number): string[];
+    persistClosed(): Promise<void>;
+    private indexPath;
+    private resolveExistingFile;
+    private loadOrScan;
+    private tryLoad;
+    private scan;
+    private persist;
+}