@oas-tools/oas-telemetry 0.7.1 → 0.8.0-alpha.0

package/dist/esm/docs/openapi.yaml

@@ -31,8 +31,8 @@ servers:
       port:
         default: "3000"
       basePath:
-        default: telemetry
-        description: Base path for all endpoints. Can be customized, e.g. "myCustomTelemetryPath"
+        default: oas-telemetry
+        description: Base path for all endpoints. Configured via OASTLM_BOOT_BASE_URL environment variable
 
 tags:
   - name: traces
@@ -326,6 +326,44 @@ paths:
                     type: string
                     example: Invalid retention time. Must be a positive number.
 
+  /traces/export:
+    get:
+      tags:
+        - traces
+      summary: Export collected traces
+      description: Export all collected traces in NDJSON format
+      responses:
+        '200':
+          description: Traces exported successfully
+          content:
+            application/x-ndjson:
+              schema:
+                type: string
+
+  /traces/import:
+    post:
+      tags:
+        - traces
+      summary: Import traces
+      description: Import traces from NDJSON format
+      requestBody:
+        required: true
+        content:
+          application/x-ndjson:
+            schema:
+              type: string
+      responses:
+        '200':
+          description: Traces imported successfully
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  message:
+                    type: string
+                    example: Traces imported successfully
+
   /metrics:
     get:
       summary: Get collected metrics
@@ -548,6 +586,69 @@ paths:
                     type: string
                     example: Invalid retention time. Must be a positive number.
 
+  /metrics/export:
+    get:
+      tags:
+        - metrics
+      summary: Export collected metrics
+      description: Export all collected metrics in NDJSON format
+      responses:
+        '200':
+          description: Metrics exported successfully
+          content:
+            application/x-ndjson:
+              schema:
+                type: string
+
+  /metrics/import:
+    post:
+      tags:
+        - metrics
+      summary: Import metrics
+      description: Import metrics from NDJSON format
+      requestBody:
+        required: true
+        content:
+          application/x-ndjson:
+            schema:
+              type: string
+      responses:
+        '200':
+          description: Metrics imported successfully
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  message:
+                    type: string
+                    example: Metrics imported successfully
+
+  /metrics/stats:
+    get:
+      tags:
+        - metrics
+      summary: Get metrics statistics
+      description: Get statistics about collected metrics
+      responses:
+        '200':
+          description: Metrics statistics retrieved successfully
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  totalMetrics:
+                    type: number
+                    example: 42
+                  activeMetrics:
+                    type: number
+                    example: 35
+                  retentionTime:
+                    type: number
+                    description: Retention time in seconds
+                    example: 3600
+
   /logs:
     get:
       summary: Get collected logs
@@ -683,6 +784,44 @@ paths:
                     type: string
                     example: Invalid regex in query
 
+  /logs/export:
+    get:
+      tags:
+        - logs
+      summary: Export collected logs
+      description: Export all collected logs in NDJSON format
+      responses:
+        '200':
+          description: Logs exported successfully
+          content:
+            application/x-ndjson:
+              schema:
+                type: string
+
+  /logs/import:
+    post:
+      tags:
+        - logs
+      summary: Import logs
+      description: Import logs from NDJSON format
+      requestBody:
+        required: true
+        content:
+          application/x-ndjson:
+            schema:
+              type: string
+      responses:
+        '200':
+          description: Logs imported successfully
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  message:
+                    type: string
+                    example: Logs imported successfully
+
   /utils/spec:
     get:
       summary: Get OpenAPI spec in JSON
@@ -842,14 +981,14 @@ paths:
                     type: string
                     example: Started generating mock logs
 
-  /utils/generate-wait/{seconds}:
+  /utils/generate-wait:
     get:
       summary: Wait for the specified seconds
       tags:
         - utils
       parameters:
         - name: seconds
-          in: path
+          in: query
           required: false
           schema:
             type: integer
@@ -867,6 +1006,21 @@ paths:
                     type: number
                     example: 2
 
+  /utils/oas-telemetry-spec:
+    get:
+      summary: Get OAS-Telemetry specification
+      tags:
+        - utils
+      description: Retrieve the OpenAPI specification for OAS-Telemetry
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                type: object
+                description: OpenAPI specification
+
   /health:
     get:
       summary: Health check

package/dist/esm/index.js

@@ -5,25 +5,26 @@ import { getConfig } from './config/config.js';
 import { Router } from 'express';
 import { configureRoutes } from './routesManager.js';
 import { configureTelemetry } from './telemetry/telemetryConfigurator.js';
+import { isTelemetryConfigured, setTelemetryRouter, getTelemetryRouter } from './telemetry/telemetryRegistry.js';
 import { bootEnvVariables } from "./config/bootConfig.js";
-// WARN: If changed the API, also change in packages/lib/src/types/cjs-index.d.ts (used for CJS compilation)
 /**
  * Returns the OAS-Telemetry middleware.
  * All parameters are optional. However, either `spec` or `specFileName` must be provided to enable endpoint filtering.
  */
-export default function oasTelemetry(oasTlmInputConfig) {
+function oasTelemetry(oasTlmInputConfig) {
+    if (isTelemetryConfigured()) {
+        return getTelemetryRouter();
+    }
     const router = Router();
-    // This environment variable cannot be set via the config object,
-    // as it is required to disable OpenTelemetry SDK initialization,
-    // which occurs during the first import at the top of this file.
     if (bootEnvVariables.OASTLM_BOOT_MODULE_DISABLED) {
+        setTelemetryRouter(router);
         return router;
     }
     const oasTlmConfig = getConfig(oasTlmInputConfig);
-    logger.info("BaseUrl: ", oasTlmConfig.general.baseUrl);
-    if (!oasTlmConfig.general.spec && !oasTlmConfig.general.specFileName)
-        logger.warn("No spec provided, endpoint filtering will not be available. Please provide either `spec` or `specFileName` in the configuration.");
+    logger.info("BaseUrl: ", bootEnvVariables.OASTLM_BOOT_BASE_URL);
     configureTelemetry(oasTlmConfig);
     configureRoutes(router, oasTlmConfig);
+    setTelemetryRouter(router);
     return router;
 }
+export { oasTelemetry };

package/dist/esm/routesManager.js

@@ -13,21 +13,17 @@ import { getAIRoutes } from "./tlm-ai/aiRoutes.js";
 import { bootEnvVariables } from "./config/bootConfig.js";
 import { getPluginRoutes } from "./tlm-plugin/pluginRoutes.js";
 export const configureRoutes = (router, oasTlmConfig) => {
+    if (!oasTlmConfig.general.spec && !oasTlmConfig.general.specFileName) {
+        logger.warn("No spec provided, endpoint filtering will not be available. Please provide either `spec` or `specFileName` in the configuration.");
+    }
     if (bootEnvVariables.OASTLM_BOOT_ENV === 'development') {
         logger.info("Running in development mode, enabling CORS for all origins");
         router.use(cors({
-            origin: (origin, callback) => {
-                if (!origin || /^http:\/\/localhost:\d+$/.test(origin)) {
-                    callback(null, true);
-                }
-                else {
-                    callback(new Error('Not allowed by CORS'));
-                }
-            },
+            origin: true,
             credentials: true
         }));
     }
-    const telemetryBaseUrl = oasTlmConfig.general.baseUrl;
+    const telemetryBaseUrl = bootEnvVariables.OASTLM_BOOT_BASE_URL;
     // Sub-router for all telemetry endpoints
     const telemetryRouter = Router();
     // Body parser for JSON requests
@@ -35,7 +31,7 @@ export const configureRoutes = (router, oasTlmConfig) => {
         if (req.body !== undefined) {
             return next(); // Already parsed, no need to parse again.
         }
-        return json({ limit: '10mb' })(req, res, next);
+        return json({ limit: '500mb' })(req, res, next);
     });
     telemetryRouter.get('/health', (_req, res) => {
         res.status(200).send({ status: 'OK' });

package/dist/esm/telemetry/custom-implementations/exporters/InMemoryDbLogExporter.js

@@ -6,18 +6,36 @@ import { applyNesting, removeCircularRefs } from '../utils/circular.js';
 import { Enabler } from '../wrappers.js';
 import logger from '../../../utils/logger.js';
 import { pluginService } from '../../../tlm-plugin/pluginService.js';
+import { getStoragePath } from '../utils/storagePath.js';
 export class InMemoryDbLogExporter extends Enabler {
+    _db = null;
+    _miniSearch = null;
+    _retentionTimeInSeconds;
+    _storagePath = null;
+    _initialized = false;
     constructor(retentionTimeInSeconds = 3600) {
         super();
         this._retentionTimeInSeconds = retentionTimeInSeconds;
-        this._db = new Datastore({ timestampData: true });
+        this._storagePath = getStoragePath('logs');
+        this._startCleanupJob();
+    }
+    _ensureInitialized() {
+        if (this._initialized)
+            return;
+        this._initialized = true;
+        this._db = new Datastore(this._storagePath ? { filename: this._storagePath, timestampData: true, autoload: true } : { timestampData: true });
         this._db.ensureIndex({ fieldName: 'createdAt' });
         this._miniSearch = new MiniSearch({
             fields: ['body'],
             storeFields: ['_id'],
             idField: '_id',
         });
-        this._startCleanupJob();
+        if (this._storagePath) {
+            logger.info(`[LogExporter] Disk storage enabled at: ${this._storagePath}`);
+        }
+        else {
+            logger.info(`[LogExporter] Using in-memory storage`);
+        }
     }
     /*
     * SUPER WARNING:
@@ -31,6 +49,7 @@ export class InMemoryDbLogExporter extends Enabler {
      * @param resultCallback
      */
     export(logs, resultCallback) {
+        this._ensureInitialized();
         const logsToInsert = logs.map(logRecord => {
             // Remove circular references first, then apply nesting, then export info
             const formattedLog = this._formatLogRecord(logRecord);
@@ -48,7 +67,17 @@ export class InMemoryDbLogExporter extends Enabler {
         resultCallback({ code: ExportResultCode.SUCCESS });
     }
     reset() {
-        this._db = new Datastore();
+        this._ensureInitialized();
+        // Remove all logs from database but keep persistence enabled
+        this._db.remove({}, { multi: true }, (err) => {
+            if (err) {
+                logger.error(`[LogExporter] Error during reset: ${err.message}`);
+            }
+            else {
+                logger.info(`[LogExporter] Reset - all logs cleared`);
+            }
+        });
+        // Clear mini search index
         this._miniSearch = new MiniSearch({
             fields: ['body'],
             storeFields: ['_id'],
@@ -61,8 +90,10 @@ export class InMemoryDbLogExporter extends Enabler {
     async shutdown() {
         this._db = null;
         this._miniSearch = null;
+        this._initialized = false;
     }
     async find(findConfig) {
+        this._ensureInitialized();
         const { query, messageSearch, limit, sortOrder } = findConfig;
         const finalQuery = { ...query };
         const effectiveSortOrder = sortOrder || { timestamp: -1 };
@@ -73,10 +104,13 @@ export class InMemoryDbLogExporter extends Enabler {
             finalQuery._id = { $in: ids };
         }
         const docs = await new Promise((resolve, reject) => {
-            this._db.find(finalQuery)
-                .sort(effectiveSortOrder)
-                .limit(limit)
-                .exec((err, docs) => {
+            let query_exec = this._db.find(finalQuery)
+                .sort(effectiveSortOrder);
+            // Only apply limit if provided
+            if (limit !== undefined) {
+                query_exec = query_exec.limit(limit);
+            }
+            query_exec.exec((err, docs) => {
                 if (err)
                     reject(err);
                 else
@@ -103,6 +137,7 @@ export class InMemoryDbLogExporter extends Enabler {
         });
     }
     getFinishedLogs() {
+        this._ensureInitialized();
         return this._db.getAllData();
     }
     /**
@@ -136,6 +171,8 @@ export class InMemoryDbLogExporter extends Enabler {
         return this._retentionTimeInSeconds;
     }
     _insertLogs(logsToInsert, resultCallback) {
+        if (!this._db)
+            return resultCallback({ code: ExportResultCode.FAILED });
         this._db.insert(logsToInsert, (err, newDocs) => {
             if (err) {
                 console.dir(err);
@@ -143,7 +180,7 @@ export class InMemoryDbLogExporter extends Enabler {
                 return;
             }
             // console.dir(newDocs, { depth: 3 });
-            newDocs.forEach((doc) => this._miniSearch.add(doc));
+            newDocs.forEach((doc) => this._miniSearch?.add(doc));
             resultCallback({ code: ExportResultCode.SUCCESS });
         });
         return;
@@ -151,6 +188,8 @@ export class InMemoryDbLogExporter extends Enabler {
     _startCleanupJob() {
         const interval = 1000;
         setInterval(() => {
+            if (!this._db)
+                return; // Safety check
             const expirationDate = new Date(Date.now() - this._retentionTimeInSeconds * 1000);
             this._db.remove({ createdAt: { $lt: expirationDate } }, { multi: true }, (err, numRemoved) => {
                 if (err) {

package/dist/esm/telemetry/custom-implementations/exporters/InMemoryDbMetricExporter.js

@@ -1,32 +1,100 @@
 import { ExportResultCode } from '@opentelemetry/core';
-import dataStore from '@seald-io/nedb';
-import { applyNesting } from '../utils/circular.js';
 import { Enabler } from '../wrappers.js';
 import logger from '../../../utils/logger.js';
 import { pluginService } from '../../../tlm-plugin/pluginService.js';
+import { rawToOtel } from '../metrics/tsdb/utils.js';
+import { SeriesRegistry } from '../metrics/tsdb/SeriesRegistry.js';
+import { getStoragePath } from '../utils/storagePath.js';
+import fs from 'fs';
+/**
+ * In-Memory TSDB Metric Exporter
+ * Series identified by: scope + metricName + attributes
+ */
 export class InMemoryDbMetricExporter extends Enabler {
-    constructor(retentionTimeInSeconds = 3600) {
+    static DEFAULT_CONFIG = {
+        retentionTimeInSeconds: 3600,
+        chunkSize: 120,
+        maxChunks: 60
+    };
+    registry;
+    config;
+    cachedResource = null;
+    _storageFilePath = null;
+    _autoSaveInterval = null;
+    _storageLogged = false; // Track if we've logged storage info
+    get rawDataDB() {
+        // For debug/inspection only - metrics are stored in registry chunks
+        const stats = this.registry.getStats();
+        return stats.totalSamples > 0 ? [`[${stats.totalSamples} samples in registry]`] : [];
+    }
+    constructor(config) {
         super();
-        this._retentionTimeInSeconds = retentionTimeInSeconds;
-        this._metrics = new dataStore({ timestampData: true });
-        this._metrics.ensureIndex({ fieldName: 'createdAt' });
+        this.config = { ...InMemoryDbMetricExporter.DEFAULT_CONFIG, ...config };
+        this.registry = new SeriesRegistry(this.config.chunkSize, this.config.maxChunks);
+        // Load persisted metrics if disk storage is enabled
+        this._storageFilePath = getStoragePath('metrics');
+        if (this._storageFilePath) {
+            this._loadMetricsFromDisk();
+            this._startAutoSave();
+        }
         this._startCleanupJob();
     }
-    export(metrics, resultCallback) {
+    _loadMetricsFromDisk() {
+        if (!this._storageFilePath)
+            return;
+        try {
+            if (fs.existsSync(this._storageFilePath)) {
+                const ndjsonData = fs.readFileSync(this._storageFilePath, 'utf-8');
+                this.registry.deserializeFromNDJSON(ndjsonData);
+            }
+        }
+        catch {
+            // Silently fail during boot
+        }
+    }
+    _startAutoSave() {
+        if (!this._storageFilePath)
+            return;
+        // Save every 30 seconds
+        this._autoSaveInterval = setInterval(() => {
+            this._saveMetricsToDisk();
+        }, 30000);
+    }
+    _saveMetricsToDisk() {
+        if (!this._storageFilePath)
+            return;
+        try {
+            const ndjsonData = this.registry.serializeToNDJSON();
+            fs.writeFileSync(this._storageFilePath, ndjsonData);
+        }
+        catch {
+            // Silently fail to avoid logging issues
+        }
+    }
+    export(resourceMetrics, resultCallback) {
+        // Log storage info on first export (when logger is ready)
+        if (!this._storageLogged) {
+            this._storageLogged = true;
+            if (this._storageFilePath) {
+                logger.info(`[MetricExporter] Disk storage enabled at: ${this._storageFilePath} (auto-save every 30s)`);
+            }
+            else {
+                logger.info(`[MetricExporter] Using in-memory storage`);
+            }
+        }
         try {
-            const scopeMetrics = metrics?.scopeMetrics;
-            const cleanMetrics = applyNesting(scopeMetrics);
-            cleanMetrics.forEach((metric) => {
+            // Cache resource (unique per exporter instance)
+            if (!this.cachedResource) {
+                this.cachedResource = resourceMetrics.resource;
+            }
+            const scopeMetrics = resourceMetrics.scopeMetrics;
+            // Broadcast to plugins
+            scopeMetrics?.forEach((metric) => {
                 pluginService.broadcastMetric(metric);
             });
-            // Insert only if exporter is enabled
-            if (this.isEnabled()) {
-                this._metrics.insert(cleanMetrics, (err, _newDoc) => {
-                    if (err) {
-                        logger.error('Insertion Error:', err);
-                        return;
-                    }
-                });
+            // Store if enabled - use new storeScopeMetrics method
+            if (this.isEnabled() && scopeMetrics) {
+                this.registry.storeScopeMetrics(scopeMetrics);
             }
             setTimeout(() => resultCallback({ code: ExportResultCode.SUCCESS }), 0);
         }
@@ -40,48 +108,96 @@ export class InMemoryDbMetricExporter extends Enabler {
     }
     shutdown() {
         this._enabled = false;
-        this._metrics = new dataStore();
+        // Stop auto-save and save one final time
+        if (this._autoSaveInterval) {
+            clearInterval(this._autoSaveInterval);
+            this._autoSaveInterval = null;
+        }
+        if (this._storageFilePath) {
+            this._saveMetricsToDisk();
+            logger.info(`[MetricExporter] Metrics saved to disk at shutdown`);
+        }
+        this.registry.reset();
         return this.forceFlush();
     }
+    reset() {
+        // Clear metrics but save empty state to disk
+        this.registry.reset();
+        if (this._storageFilePath) {
+            this._saveMetricsToDisk();
+            logger.info(`[MetricExporter] Reset - all metrics cleared and saved to disk`);
+        }
+        else {
+            logger.info(`[MetricExporter] Reset - all metrics cleared`);
+        }
+    }
     forceFlush() {
         return Promise.resolve();
     }
-    find(search, callback) {
-        this._metrics.find(search, callback);
+    getCachedResource() {
+        return this.cachedResource;
     }
-    reset() {
-        this._metrics = new dataStore();
+    set retentionTimeInSeconds(value) {
+        this.config.retentionTimeInSeconds = value;
+        logger.info(`Retention time set to ${value} seconds`);
+    }
+    get retentionTimeInSeconds() {
+        return this.config.retentionTimeInSeconds;
     }
-    getFinishedMetrics() {
-        return this._metrics.getAllData();
+    getStats() {
+        return this.registry.getStats();
+    }
+    _startCleanupJob() {
+        setInterval(() => {
+            const retentionTimeNs = this.config.retentionTimeInSeconds * 1_000_000_000;
+            const result = this.registry.evictOldData(retentionTimeNs);
+            if (result.evictedChunks > 0 || result.evictedSeries > 0) {
+                logger.debug(`Cleanup: evicted ${result.evictedChunks} chunks, ${result.evictedSeries} series`);
+            }
+        }, 5000);
     }
     /**
-     * Inserts metrics into the in-memory database.
-     * @param metrics - The metrics to insert.
-     * @param callback - The callback to execute after insertion.
+     * Find metrics by scope+metric queries with filters
+     * Supports both raw and otel formats
      */
-    insert(metrics, callback) {
-        this._metrics.insert(metrics, callback);
+    find(request) {
+        const format = request.format || 'raw';
+        // Get raw results from registry using new unified query method
+        const rawResults = this.registry.query(request.scopeMetrics, request.from, request.to);
+        // Convert format if needed
+        if (format === 'otel') {
+            return { results: rawToOtel(rawResults) };
+        }
+        return { results: rawResults };
     }
-    set retentionTimeInSeconds(retentionTimeInSeconds) {
-        this._retentionTimeInSeconds = retentionTimeInSeconds;
-        logger.info(`InMemoryDbMetricExporter retention time set to ${this._retentionTimeInSeconds} seconds`);
+    /**
+     * Export all metrics to NDJSON format (one chunk per line)
+     */
+    exportToNDJSON() {
+        return this.registry.serializeToNDJSON();
     }
-    get retentionTimeInSeconds() {
-        return this._retentionTimeInSeconds;
+    /**
+     * Import metrics from NDJSON format
+     */
+    importFromNDJSON(ndjsonData) {
+        this.registry.deserializeFromNDJSON(ndjsonData);
     }
-    _startCleanupJob() {
-        const interval = 1000;
-        setInterval(() => {
-            const expirationDate = new Date(Date.now() - this._retentionTimeInSeconds * 1000);
-            this._metrics.remove({ createdAt: { $lt: expirationDate } }, { multi: true }, (err, numRemoved) => {
-                if (err) {
-                    logger.error('Error in TTL cleanup:', err);
-                }
-                else if (numRemoved > 0) {
-                    logger.debug(`TTL cleanup: removed ${numRemoved} expired metrics`);
-                }
-            });
-        }, interval);
+    /**
+ * Insert metrics in OpenTelemetry (OTEL) format directly into the registry
+ * @param scopeMetrics Array of ScopeMetrics (OTEL format)
+ */
+    insertOtel(scopeMetrics) {
+        // Store only in registry (chunks) - no duplication
+        if (this.isEnabled()) {
+            this.registry.storeScopeMetrics(scopeMetrics);
+        }
+    }
+    /**
+     * Insert metrics in raw format (MetricQueryResult[]), converts to OTEL and delegates to insertOtel
+     * @param rawScopeMetrics Array of MetricQueryResult (raw format)
+     */
+    insertRaw(rawScopeMetrics) {
+        const otelScopeMetrics = rawToOtel(rawScopeMetrics);
+        this.insertOtel(otelScopeMetrics);
     }
 }