@oas-tools/oas-telemetry 0.7.1 → 0.8.0-alpha.1

package/dist/esm/telemetry/custom-implementations/metrics/tsdb/types.js

@@ -0,0 +1 @@
+export {};

package/dist/esm/telemetry/custom-implementations/metrics/tsdb/utils.js

@@ -0,0 +1,74 @@
+/**
+ * Convert raw format to OpenTelemetry format
+ *
+ * Raw format structure (same for GET response and INSERT input):
+ * {
+ *   scope: { name, version },
+ *   descriptor: { unit, description, ... },
+ *   series: [{ attributes, startTimes[], endTimes[], values[] }]
+ * }
+ *
+ * OTEL format structure:
+ * {
+ *   scope: { name, version },
+ *   metrics: [{ descriptor, dataPoints: [{ attributes, startTime, endTime, value }] }]
+ * }
+ *
+ * Groups dataPoints by endTime to reconstruct original exports
+ */
+export function rawToOtel(rawScopeMetrics) {
+    const groupedByScope = new Map();
+    for (const result of rawScopeMetrics) {
+        // Validate that result matches expected raw format
+        if (!result.scope) {
+            throw new Error(`Invalid metric format: Missing 'scope' property. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}. ` +
+                `Got: ${JSON.stringify(result).substring(0, 200)}`);
+        }
+        if (!result.descriptor) {
+            throw new Error(`Invalid metric format: Missing 'descriptor' property in metric. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}`);
+        }
+        if (!Array.isArray(result.series)) {
+            throw new Error(`Invalid metric format: 'series' must be an array. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}`);
+        }
+        const scopeId = result.scope.version
+            ? `${result.scope.name}@${result.scope.version}`
+            : `${result.scope.name}@none`;
+        if (!groupedByScope.has(scopeId)) {
+            groupedByScope.set(scopeId, {
+                scope: {
+                    name: result.scope.name,
+                    version: result.scope.version || ''
+                },
+                metrics: []
+            });
+        }
+        const scopeData = groupedByScope.get(scopeId);
+        const dataPoints = [];
+        for (const series of result.series) {
+            for (let i = 0; i < series.endTimes.length; i++) {
+                const startTimeNs = series.startTimes?.[i];
+                const endTimeNs = series.endTimes[i];
+                if (!startTimeNs)
+                    continue;
+                const startSec = Math.floor(startTimeNs / 1_000_000_000);
+                const startNano = startTimeNs % 1_000_000_000;
+                const endSec = Math.floor(endTimeNs / 1_000_000_000);
+                const endNano = endTimeNs % 1_000_000_000;
+                dataPoints.push({
+                    attributes: series.attributes,
+                    startTime: [startSec, startNano],
+                    endTime: [endSec, endNano],
+                    value: series.values[i]
+                });
+            }
+        }
+        scopeData.metrics.push({
+            descriptor: result.descriptor,
+            dataPoints
+        });
+    }
+    return Array.from(groupedByScope.values());
+}

package/dist/esm/telemetry/custom-implementations/processors/dynamicMultiLogProcessor.js

@@ -1,8 +1,9 @@
 import { callWithTimeout } from '@opentelemetry/core';
 import logger from '../../../utils/logger.js'; // optional if you want logging
 export class DynamicMultiLogRecordProcessor {
+    _processors = [];
+    _forceFlushTimeoutMillis;
     constructor(initialProcessors = [], forceFlushTimeoutMillis = 30000) {
-        this._processors = [];
         this._processors = [...initialProcessors];
         this._forceFlushTimeoutMillis = forceFlushTimeoutMillis;
     }

package/dist/esm/telemetry/custom-implementations/processors/dynamicMultiSpanProcessor.js

@@ -1,7 +1,7 @@
 import logger from '../../../utils/logger.js';
 export class DynamicMultiSpanProcessor {
+    _spanProcessors = [];
     constructor(initialProcessors = []) {
-        this._spanProcessors = [];
         this._spanProcessors = [...initialProcessors];
     }
     /**

package/dist/esm/telemetry/custom-implementations/wrappers.js

@@ -1,8 +1,9 @@
+import { ExportResultCode } from "@opentelemetry/core";
 import { resourceFromAttributes } from "@opentelemetry/resources";
 import logger from "../../utils/logger.js";
 export class Enabler {
+    _enabled = true;
     constructor(enabled) {
-        this._enabled = true;
         if (typeof enabled === 'boolean') {
             this._enabled = enabled;
         }
@@ -40,6 +41,7 @@ export class Enabler {
  *
  */
 export class EnablerSpanExporter extends Enabler {
+    exporter;
     constructor(exporter) {
         super();
         this.exporter = exporter;
@@ -60,6 +62,7 @@ export class EnablerSpanExporter extends Enabler {
     }
 }
 export class EnablerLogExporter extends Enabler {
+    exporter;
     constructor(exporter) {
         super();
         this.exporter = exporter;
@@ -74,9 +77,9 @@ export class EnablerLogExporter extends Enabler {
     }
 }
 class EnablerMultiExporter extends Enabler {
+    _exporters = [];
     constructor(exporters) {
         super();
-        this._exporters = [];
         if (exporters && Array.isArray(exporters)) {
             this._exporters = exporters;
         }
@@ -92,13 +95,48 @@ class EnablerMultiExporter extends Enabler {
             this._exporters.push(exporter);
         }
     }
+    clearExporters() {
+        this._exporters = [];
+    }
 }
 export class EnablerMultiSpanExporter extends EnablerMultiExporter {
     export(spans, resultCallback) {
         logger.debug(`EnablerMultiSpanExporter.export called with ${spans.length} spans, exporters: ${this._exporters?.length ?? 0}, enabled: ${this.isEnabled()}`);
-        if (this.isEnabled() && this._exporters) {
-            this._exporters.forEach((exporter) => exporter.export(spans, resultCallback));
+        if (!this.isEnabled() || !this._exporters || this._exporters.length === 0) {
+            resultCallback({ code: ExportResultCode.SUCCESS });
+            return;
         }
+        let pending = this._exporters.length;
+        let failed = false;
+        let firstError;
+        this._exporters.forEach((exporter) => {
+            try {
+                exporter.export(spans, (result) => {
+                    if (result.code === ExportResultCode.FAILED) {
+                        failed = true;
+                        if (result.error && !firstError) {
+                            firstError = result.error;
+                        }
+                    }
+                    pending -= 1;
+                    if (pending === 0) {
+                        resultCallback(failed
+                            ? { code: ExportResultCode.FAILED, error: firstError }
+                            : { code: ExportResultCode.SUCCESS });
+                    }
+                });
+            }
+            catch (error) {
+                failed = true;
+                if (!firstError) {
+                    firstError = error instanceof Error ? error : new Error(String(error));
+                }
+                pending -= 1;
+                if (pending === 0) {
+                    resultCallback({ code: ExportResultCode.FAILED, error: firstError });
+                }
+            }
+        });
     }
     async shutdown() {
         if (this._exporters) {
@@ -109,9 +147,41 @@ export class EnablerMultiSpanExporter extends EnablerMultiExporter {
 }
 export class EnablerMultiLogExporter extends EnablerMultiExporter {
     export(logs, resultCallback) {
-        if (this.isEnabled() && this._exporters) {
-            this._exporters.forEach((exporter) => exporter.export(logs, resultCallback));
+        if (!this.isEnabled() || !this._exporters || this._exporters.length === 0) {
+            resultCallback({ code: ExportResultCode.SUCCESS });
+            return;
         }
+        let pending = this._exporters.length;
+        let failed = false;
+        let firstError;
+        this._exporters.forEach((exporter) => {
+            try {
+                exporter.export(logs, (result) => {
+                    if (result.code === ExportResultCode.FAILED) {
+                        failed = true;
+                        if (result.error && !firstError) {
+                            firstError = result.error;
+                        }
+                    }
+                    pending -= 1;
+                    if (pending === 0) {
+                        resultCallback(failed
+                            ? { code: ExportResultCode.FAILED, error: firstError }
+                            : { code: ExportResultCode.SUCCESS });
+                    }
+                });
+            }
+            catch (error) {
+                failed = true;
+                if (!firstError) {
+                    firstError = error instanceof Error ? error : new Error(String(error));
+                }
+                pending -= 1;
+                if (pending === 0) {
+                    resultCallback({ code: ExportResultCode.FAILED, error: firstError });
+                }
+            }
+        });
     }
     async shutdown() {
         if (this._exporters) {
@@ -121,6 +191,7 @@ export class EnablerMultiLogExporter extends EnablerMultiExporter {
     }
 }
 export class EnablerMetricReader extends Enabler {
+    reader;
     constructor(reader) {
         super();
         this.reader = reader;

package/dist/esm/telemetry/initializeTelemetry.js

@@ -1,79 +1,37 @@
-import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
 import logger from '../utils/logger.js';
-import { registerInstrumentations } from '@opentelemetry/instrumentation';
-import { dynamicMultiLogProcessor, dynamicMultiSpanProcessor, oasTelemetryResource, originalConsoleMethods } from './telemetryRegistry.js';
-import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
-import { SeverityNumber } from '@opentelemetry/api-logs';
-import { LoggerProvider } from '@opentelemetry/sdk-logs';
 import { bootEnvVariables } from '../config/bootConfig.js';
-import util from 'util';
+import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
+import { instrumentations, isBootInitialized, setBootInitialized } from './telemetryRegistry.js';
+import { LogsInstrumentation } from './custom-implementations/instrumentations/logsInstrumentation.js';
+import { RuntimeNodeInstrumentation } from '@opentelemetry/instrumentation-runtime-node';
 // THIS INSTRUMENTATIONS NEED TO BE LOADED BEFORE ANYTHING ELSE
 // They use monkey-patching to instrument the HTTP server and client.
 // THIS FILE MUST BE CALLED BEFORE ANYTHING ELSE
 if (bootEnvVariables.OASTLM_BOOT_MODULE_DISABLED) {
-    logger.info('🚫 OASTLM module is disabled, Providers not initialized.');
+    logger.info('🚫 OASTLM module is disabled, Auto Instrumentations not initialized.');
 }
 else {
-    logger.info('🚀 Initializing Open Telemetry');
-    initializeTraces();
-    initializeMetrics();
-    initializeLogs();
-    registerInstrumentations({
-        instrumentations: [
-            new HttpInstrumentation(),
-            // new ExpressInstrumentation(),
-        ],
-    });
-}
-function initializeTraces() {
-    logger.info('📊 Initializing TracerProvider');
-    const tracerProvider = new NodeTracerProvider({
-        resource: oasTelemetryResource,
-        spanProcessors: [dynamicMultiSpanProcessor]
-    });
-    tracerProvider.register();
-}
-function initializeLogs() {
-    logger.info('📜 Initializing LoggerProvider');
-    // Create and configure LoggerProvider
-    const loggerProvider = new LoggerProvider({
-        resource: oasTelemetryResource,
-        processors: [dynamicMultiLogProcessor]
-    });
-    // Get a logger instance
-    const loggerInstance = loggerProvider.getLogger('oas-telemetry'); // Use loggerProvider to get the logger
-    Object.keys(originalConsoleMethods).forEach((method) => {
-        // @ts-expect-error yes
-        console[method] = (...args) => {
-            const severity = getSeverityForMethod(method);
-            loggerInstance.emit({
-                severityNumber: severity.number,
-                severityText: severity.text,
-                body: util.format(...args),
-                attributes: { 'source': `console.${method}`, "library": "oas-telemetry" },
-            });
-            // @ts-expect-error yes
-            originalConsoleMethods[method](...args);
-        };
-    });
-}
-function initializeMetrics() {
-    logger.info('📈 Initializing MeterProvider');
-    // WARN: Default PeriodicExportingMetricReader is added post initialization (see telemetryConfigurator.ts)
-    // The in memory exporter is added by default to that reader. More readers are allowed to be added dynamically
-}
-function getSeverityForMethod(method) {
-    switch (method) {
-        case "log":
-        case "info":
-            return { number: SeverityNumber.INFO, text: "INFO" };
-        case "debug":
-            return { number: SeverityNumber.DEBUG, text: "DEBUG" };
-        case "warn":
-            return { number: SeverityNumber.WARN, text: "WARN" };
-        case "error":
-            return { number: SeverityNumber.ERROR, text: "ERROR" };
-        default:
-            return { number: SeverityNumber.INFO, text: "INFO" };
+    if (!isBootInitialized()) {
+        // Only HTTP instrumentation por ahora
+        const httpInstrumentation = new HttpInstrumentation({
+            // Ignore internal telemetry routes
+            ignoreIncomingRequestHook: (req) => {
+                const url = req.url || '';
+                const path = url.split('?')[0]; // Remove query params
+                // Ignore all baseUrl routes except those with 'generate'
+                if (path.includes(bootEnvVariables.OASTLM_BOOT_BASE_URL)) {
+                    return !path.includes('generate');
+                }
+                return false;
+            }
+        });
+        if (!bootEnvVariables.OASTLM_BOOT_AUTOINSTRUMENTATIONS_NODE_DISABLED) {
+            instrumentations.push(httpInstrumentation, new RuntimeNodeInstrumentation());
+        }
+        if (!bootEnvVariables.OASTLM_BOOT_AUTOINSTRUMENTATIONS_LOGS_DISABLED) {
+            instrumentations.push(new LogsInstrumentation());
+        }
+        setBootInitialized(true);
+        logger.info('✅ Auto Instrumentations created successfully');
     }
 }

package/dist/esm/telemetry/persistence/DiskImporter.js

@@ -0,0 +1,78 @@
+import fs from 'fs/promises';
+import logger from '../../utils/logger.js';
+import { decodeRecordBatchFrame, listSegments, DEFAULT_SEGMENT_PREFIX } from './DiskUtils.js';
+export class DiskImporter {
+    directoryPath;
+    segmentPrefix;
+    importBatchSize;
+    constructor(options) {
+        this.directoryPath = options.directoryPath;
+        this.segmentPrefix = options.segmentPrefix || DEFAULT_SEGMENT_PREFIX;
+        this.importBatchSize = options.importBatchSize || 500;
+    }
+    async import(onBatch) {
+        const stats = {
+            segmentFilesRead: 0,
+            importedRecords: 0,
+            failedFrames: 0,
+        };
+        const segments = await listSegments(this.directoryPath, this.segmentPrefix);
+        if (!segments.length) {
+            return stats;
+        }
+        const importBuffer = [];
+        const flushImportBuffer = async () => {
+            if (!importBuffer.length)
+                return;
+            const batch = importBuffer.splice(0, importBuffer.length);
+            await onBatch(batch);
+            stats.importedRecords += batch.length;
+        };
+        for (const segment of segments) {
+            stats.segmentFilesRead += 1;
+            let fileBuffer;
+            try {
+                fileBuffer = await fs.readFile(segment.filePath);
+            }
+            catch (error) {
+                logger.warn(`[DiskImporter] Could not read segment ${segment.filePath}: ${error?.message || error}`);
+                continue;
+            }
+            let offset = 0;
+            while (offset + 4 <= fileBuffer.length) {
+                const frameLength = fileBuffer.readUInt32LE(offset);
+                offset += 4;
+                if (frameLength <= 0) {
+                    stats.failedFrames += 1;
+                    logger.warn(`[DiskImporter] Invalid frame length in ${segment.filePath}; stopping segment parse.`);
+                    break;
+                }
+                if (offset + frameLength > fileBuffer.length) {
+                    stats.failedFrames += 1;
+                    logger.warn(`[DiskImporter] Truncated frame in ${segment.filePath}; stopping segment parse.`);
+                    break;
+                }
+                const framePayload = fileBuffer.subarray(offset, offset + frameLength);
+                offset += frameLength;
+                try {
+                    const decodedSpans = decodeRecordBatchFrame(framePayload);
+                    importBuffer.push(...decodedSpans);
+                    while (importBuffer.length >= this.importBatchSize) {
+                        const batch = importBuffer.splice(0, this.importBatchSize);
+                        await onBatch(batch);
+                        stats.importedRecords += batch.length;
+                    }
+                }
+                catch (error) {
+                    stats.failedFrames += 1;
+                    logger.warn(`[DiskImporter] Failed to decode frame from ${segment.filePath}: ${error?.message || error}`);
+                }
+            }
+            if (offset !== fileBuffer.length) {
+                logger.warn(`[DiskImporter] Segment ${segment.filePath} has trailing or incomplete bytes.`);
+            }
+        }
+        await flushImportBuffer();
+        return stats;
+    }
+}

package/dist/esm/telemetry/persistence/DiskUtils.js

@@ -0,0 +1,51 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { gzipSync, gunzipSync } from 'zlib';
+export const DEFAULT_SEGMENT_PREFIX = 'traces';
+export const SEGMENT_EXT = '.oastlm';
+export const DEFAULT_MAX_SEGMENT_BYTES = 8 * 1024 * 1024;
+export function getSegmentName(index, prefix = DEFAULT_SEGMENT_PREFIX) {
+    return `${prefix}-${index.toString().padStart(6, '0')}${SEGMENT_EXT}`;
+}
+export async function listSegments(directoryPath, prefix = DEFAULT_SEGMENT_PREFIX) {
+    try {
+        const files = await fs.readdir(directoryPath);
+        const regex = new RegExp(`^${prefix}-(\\d{6})${SEGMENT_EXT.replace('.', '\\.')}$`);
+        return files
+            .map((name) => {
+            const match = name.match(regex);
+            if (!match)
+                return null;
+            const index = Number.parseInt(match[1], 10);
+            return { index, filePath: path.join(directoryPath, name) };
+        })
+            .filter((value) => value !== null)
+            .sort((a, b) => a.index - b.index);
+    }
+    catch (error) {
+        if (error?.code === 'ENOENT') {
+            return [];
+        }
+        throw error;
+    }
+}
+export function encodeRecordBatchFrame(records) {
+    const lineDelimitedJson = records.map((record) => JSON.stringify(record)).join('\n') + '\n';
+    const compressed = gzipSync(Buffer.from(lineDelimitedJson, 'utf-8'));
+    const frame = Buffer.allocUnsafe(4 + compressed.length);
+    frame.writeUInt32LE(compressed.length, 0);
+    compressed.copy(frame, 4);
+    return frame;
+}
+export function decodeRecordBatchFrame(framePayload) {
+    const decompressed = gunzipSync(framePayload).toString('utf-8');
+    const lines = decompressed.split('\n').filter((line) => line.trim().length > 0);
+    return lines.map((line, index) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch {
+            throw new Error(`Invalid JSON line at index ${index}`);
+        }
+    });
+}

package/dist/esm/telemetry/persistence/DiskWriter.js

@@ -0,0 +1,59 @@
+import fs from 'fs/promises';
+import path from 'path';
+import { encodeRecordBatchFrame, getSegmentName, listSegments, DEFAULT_MAX_SEGMENT_BYTES, DEFAULT_SEGMENT_PREFIX, } from './DiskUtils.js';
+export class DiskWriter {
+    directoryPath;
+    segmentPrefix;
+    maxSegmentBytes;
+    initialized = false;
+    currentSegmentIndex = 0;
+    currentSegmentPath = '';
+    currentSegmentSize = 0;
+    writeQueue = Promise.resolve();
+    constructor(options) {
+        this.directoryPath = path.resolve(options.directoryPath);
+        this.segmentPrefix = options.segmentPrefix || DEFAULT_SEGMENT_PREFIX;
+        this.maxSegmentBytes = options.maxSegmentBytes || DEFAULT_MAX_SEGMENT_BYTES;
+    }
+    async appendRecords(records) {
+        if (!records.length)
+            return;
+        const frame = encodeRecordBatchFrame(records);
+        this.writeQueue = this.writeQueue.then(() => this.appendFrame(frame));
+        return this.writeQueue;
+    }
+    async flush() {
+        await this.writeQueue;
+    }
+    async ensureInitialized() {
+        if (this.initialized)
+            return;
+        await fs.mkdir(this.directoryPath, { recursive: true });
+        const segments = await listSegments(this.directoryPath, this.segmentPrefix);
+        if (segments.length === 0) {
+            this.currentSegmentIndex = 0;
+            this.currentSegmentPath = path.join(this.directoryPath, getSegmentName(this.currentSegmentIndex, this.segmentPrefix));
+            this.currentSegmentSize = 0;
+        }
+        else {
+            const lastSegment = segments[segments.length - 1];
+            this.currentSegmentIndex = lastSegment.index;
+            this.currentSegmentPath = lastSegment.filePath;
+            this.currentSegmentSize = (await fs.stat(lastSegment.filePath)).size;
+        }
+        this.initialized = true;
+    }
+    async rotateSegment() {
+        this.currentSegmentIndex += 1;
+        this.currentSegmentPath = path.join(this.directoryPath, getSegmentName(this.currentSegmentIndex, this.segmentPrefix));
+        this.currentSegmentSize = 0;
+    }
+    async appendFrame(frame) {
+        await this.ensureInitialized();
+        if (this.currentSegmentSize > 0 && this.currentSegmentSize + frame.length > this.maxSegmentBytes) {
+            await this.rotateSegment();
+        }
+        await fs.appendFile(this.currentSegmentPath, frame);
+        this.currentSegmentSize += frame.length;
+    }
+}