@oas-tools/oas-telemetry 0.7.1 → 0.8.0-alpha.0

package/dist/cjs/telemetry/custom-implementations/metrics/tsdb/SeriesRegistry.cjs

@@ -0,0 +1,389 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SeriesRegistry = void 0;
+const Series_js_1 = require("./Series.cjs");
+const Chunk_js_1 = require("./Chunk.cjs");
+/*
+OTEL export = Resource (unchanged) + SCOPE METRICS
+scopeMetrics: {
+    scope: {name: library, version: 1.0.0},; //repeated for each metric
+    metrics: [
+        {
+            //Metadata (Repeated for each dataPoint)
+            descriptor: {name: metricName, unit, description, ...},
+            aggregationTemporality,
+            dataPointType,
+            // Actual dataPoints are stored in Series
+            dataPoints: []
+        }
+    ]
+}
+*/
+// Import fs at top level for disk operations
+const fs_1 = __importDefault(require("fs"));
+class SeriesRegistry {
+    // Main storage
+    series = new Map(); // seriesId (scopeId:metricName:attributesId) -> Series
+    // Cached repeated data
+    scopes = new Map(); // scopeId -> scope
+    metricMetadataMap = new Map(); // "scopeId:metricName" -> MetricMetadata
+    // Faster lookup index
+    metricIdIndex = new Map(); // "scopeId:metricName" -> Set<seriesId>
+    chunkSize;
+    maxChunks;
+    constructor(chunkSize = 120, maxChunks = 60) {
+        this.chunkSize = chunkSize;
+        this.maxChunks = maxChunks;
+    }
+    /**
+     * Store entire ScopeMetrics array efficiently
+     * Replaces addSample - processes all metrics in one batch
+     */
+    storeScopeMetrics(scopeMetrics) {
+        for (const scopeMetric of scopeMetrics) {
+            const scope = scopeMetric.scope;
+            const scopeId = scopeToId(scope);
+            this.scopes.set(scopeId, scope);
+            for (const metricData of scopeMetric.metrics) {
+                const metricName = metricData.descriptor.name;
+                const metricId = makeMetricId(scopeId, metricName);
+                if (!this.metricMetadataMap.has(metricId)) {
+                    const { dataPoints: _dataPoints, ...metadata } = metricData;
+                    this.metricMetadataMap.set(metricId, metadata);
+                }
+                for (const dp of metricData.dataPoints) {
+                    const startTime = dp.startTime[0] * 1_000_000_000 + dp.startTime[1];
+                    const endTime = dp.endTime[0] * 1_000_000_000 + dp.endTime[1];
+                    const attributes = dp.attributes;
+                    const attributesId = attributesToId(attributes);
+                    const seriesId = makeSeriesId(metricId, attributesId);
+                    if (!this.series.has(seriesId)) {
+                        this.series.set(seriesId, new Series_js_1.Series({ hash: 0, labels: attributes, originalAttributes: attributes }, this.metricMetadataMap.get(metricId), this.chunkSize, this.maxChunks));
+                        (this.metricIdIndex.get(metricId) ?? this.metricIdIndex.set(metricId, new Set()).get(metricId)).add(seriesId);
+                    }
+                    this.series.get(seriesId).append(startTime, endTime, dp.value);
+                }
+            }
+        }
+    }
+    query(scopeMetrics, startTime, endTime) {
+        // If no queries provided, get all scopeMetrics
+        if (!scopeMetrics || scopeMetrics.length === 0) {
+            const queries = Array.from(this.metricIdIndex.keys()).map(indexKey => {
+                const [scopeId, metricName] = indexKey.split(':').slice(0, 2);
+                const scope = this.scopes.get(scopeId);
+                return {
+                    scope,
+                    descriptor: { name: metricName },
+                    filters: undefined
+                };
+            });
+            return queries
+                .map(query => this._querySingle(query, startTime, endTime))
+                .filter((result) => result !== null);
+        }
+        return scopeMetrics
+            .map(query => this._querySingle(query, startTime, endTime))
+            .filter((result) => result !== null);
+    }
+    /**
+     * Query single metric with attribute filters
+     */
+    _querySingle(query, startTime, endTime) {
+        const scopeId = scopeToId(query.scope);
+        const indexKey = makeMetricId(scopeId, query.descriptor.name);
+        const seriesKeysInMetric = this.metricIdIndex.get(indexKey);
+        if (!seriesKeysInMetric || seriesKeysInMetric.size === 0) {
+            return null;
+        }
+        const scope = this.scopes.get(scopeId);
+        const metricData = this.metricMetadataMap.get(indexKey);
+        if (!scope || !metricData) {
+            return null;
+        }
+        // Filter by attributes if provided
+        const filteredSeriesKeys = query.filters
+            ? Array.from(seriesKeysInMetric).filter(key => {
+                const series = this.series.get(key);
+                return this._matchesAttributeFilters(series, query.filters);
+            })
+            : Array.from(seriesKeysInMetric);
+        if (filteredSeriesKeys.length === 0) {
+            return null;
+        }
+        const matchingSeries = filteredSeriesKeys
+            .map(key => {
+            const series = this.series.get(key);
+            const { startTimes, endTimes, values } = series.querySlices({ startTime, endTime, includeStartTimes: true });
+            return {
+                id: key,
+                attributes: series.getOriginalAttributes(),
+                startTimes: startTimes ? Array.from(startTimes) : undefined,
+                endTimes: Array.from(endTimes),
+                values: Array.isArray(values) ? values : Array.from(values)
+            };
+        })
+            .filter(s => s.endTimes.length > 0);
+        return {
+            scope,
+            descriptor: metricData.descriptor,
+            series: matchingSeries
+        };
+    }
+    /**
+     * Match series attributes against filters
+     * Supports exact match, negation (!), and regex (~)
+     */
+    _matchesAttributeFilters(series, filters) {
+        if (!filters || Object.keys(filters).length === 0) {
+            return true;
+        }
+        const attrs = series.getOriginalAttributes();
+        for (const [key, filterValue] of Object.entries(filters)) {
+            const attrValue = String(attrs[key] ?? '');
+            // Regex match: status=~4.*
+            if (filterValue.startsWith('~')) {
+                const pattern = filterValue.slice(1);
+                try {
+                    const regex = new RegExp(pattern);
+                    if (!regex.test(attrValue)) {
+                        return false;
+                    }
+                }
+                catch {
+                    if (attrValue !== filterValue) {
+                        return false;
+                    }
+                }
+            }
+            // Negation: status=!200
+            else if (filterValue.startsWith('!')) {
+                const negatedValue = filterValue.slice(1);
+                if (attrValue === negatedValue) {
+                    return false;
+                }
+            }
+            // Exact match: method=GET
+            else {
+                if (attrValue !== filterValue) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+    evictOldData(retentionTimeNs) {
+        const thresholdTime = Date.now() * 1_000_000 - retentionTimeNs;
+        let evictedChunks = 0;
+        let evictedSeries = 0;
+        for (const [key, series] of this.series.entries()) {
+            evictedChunks += series.evictOldChunks(thresholdTime);
+            if (series.isEmpty()) {
+                this.series.delete(key);
+                evictedSeries++;
+            }
+        }
+        return { evictedChunks, evictedSeries };
+    }
+    getStats() {
+        const seriesStats = Array.from(this.series.values()).map(s => s.getStats());
+        const totalSamples = seriesStats.reduce((sum, s) => sum + s.samples, 0);
+        const totalMemory = seriesStats.reduce((sum, s) => sum + s.memoryBytes, 0);
+        return {
+            totalMetrics: this.metricMetadataMap.size,
+            totalScopes: this.scopes.size,
+            totalSeries: this.series.size,
+            totalSamples,
+            memoryUsageBytes: totalMemory
+        };
+    }
+    reset() {
+        this.series.clear();
+        this.scopes.clear();
+        this.metricMetadataMap.clear();
+        this.metricIdIndex.clear();
+    }
+    size() {
+        return this.series.size;
+    }
+    /**
+     * Serialize registry to NDJSON format (newline-delimited JSON)
+     * Each line is a complete record: metadata, scope, metric, or data point
+     * Format:
+     *   {"type":"header","version":1,"timestamp":"...","stats":{...}}
+     *   {"type":"scope","id":"...","data":{...}}
+     *   {"type":"metric","id":"...","data":{...}}
+     *   {"type":"series","id":"...","labelSet":{...},"chunks":[...]}
+     */
+    serializeToNDJSON() {
+        const lines = [];
+        // Header
+        lines.push(JSON.stringify({
+            type: 'header',
+            version: 1,
+            timestamp: new Date().toISOString(),
+            stats: this.getStats()
+        }));
+        // Scopes
+        for (const [scopeId, scope] of this.scopes.entries()) {
+            lines.push(JSON.stringify({
+                type: 'scope',
+                id: scopeId,
+                data: scope
+            }));
+        }
+        // Metric metadata
+        for (const [metricId, metadata] of this.metricMetadataMap.entries()) {
+            lines.push(JSON.stringify({
+                type: 'metric',
+                id: metricId,
+                data: metadata
+            }));
+        }
+        // Series headers + chunks (one chunk per line for streaming efficiency)
+        const serializedSeriesHeaders = new Set();
+        for (const [seriesId, series] of this.series.entries()) {
+            const seriesPrivate = series;
+            // Emit series header once per series
+            if (!serializedSeriesHeaders.has(seriesId)) {
+                lines.push(JSON.stringify({
+                    type: 'series',
+                    seriesId: seriesId,
+                    labelSet: seriesPrivate.labelSet,
+                    metadata: seriesPrivate.metadata
+                }));
+                serializedSeriesHeaders.add(seriesId);
+            }
+            (seriesPrivate.chunks || []).forEach((chunk, chunkIndex) => {
+                const slicedStartTimes = chunk.startTimes.slice(0, chunk.cursor);
+                const slicedEndTimes = chunk.endTimes.slice(0, chunk.cursor);
+                const slicedValues = chunk.values.slice(0, chunk.cursor);
+                const slicedHistograms = chunk.histograms.slice(0, chunk.cursor);
+                lines.push(JSON.stringify({
+                    type: 'chunk',
+                    seriesId: seriesId,
+                    chunkIndex: chunkIndex,
+                    startTimes: Array.from(slicedStartTimes),
+                    endTimes: Array.from(slicedEndTimes),
+                    values: Array.from(slicedValues),
+                    histograms: Array.from(slicedHistograms),
+                    cursor: chunk.cursor,
+                    minEndTime: chunk.minEndTime,
+                    maxEndTime: chunk.maxEndTime,
+                    isHistogram: chunk.isHistogram
+                }));
+            });
+        }
+        // Metric ID index for fast lookup
+        for (const [metricId, seriesIdSet] of this.metricIdIndex.entries()) {
+            lines.push(JSON.stringify({
+                type: 'index',
+                id: metricId,
+                seriesIds: Array.from(seriesIdSet)
+            }));
+        }
+        return lines.join('\n');
+    }
+    /**
+     * Deserialize from NDJSON format - restore from chunk lines
+     */
+    deserializeFromNDJSON(ndjsonData) {
+        try {
+            const lines = ndjsonData.trim().split('\n');
+            for (const line of lines) {
+                if (!line.trim())
+                    continue;
+                const record = JSON.parse(line);
+                switch (record.type) {
+                    case 'header':
+                        // Just metadata
+                        break;
+                    case 'scope':
+                        this.scopes.set(record.id, record.data);
+                        break;
+                    case 'metric':
+                        this.metricMetadataMap.set(record.id, record.data);
+                        break;
+                    case 'index':
+                        this.metricIdIndex.set(record.id, new Set(record.seriesIds));
+                        break;
+                    case 'series': {
+                        // Create series stub - will be populated by subsequent chunk lines
+                        const metadata = this.metricMetadataMap.get(record.seriesId.substring(0, record.seriesId.lastIndexOf('$'))) || record.metadata;
+                        if (metadata) {
+                            const series = new Series_js_1.Series(record.labelSet, metadata, this.chunkSize, this.maxChunks);
+                            this.series.set(record.seriesId, series);
+                        }
+                        break;
+                    }
+                    case 'chunk': {
+                        // Restore chunk to series
+                        const series = this.series.get(record.seriesId);
+                        if (series) {
+                            const chunk = new Chunk_js_1.Chunk(record.cursor || record.startTimes.length, record.isHistogram);
+                            const chunkPrivate = chunk;
+                            chunkPrivate.startTimes = new Float64Array(record.startTimes);
+                            chunkPrivate.endTimes = new Float64Array(record.endTimes);
+                            chunkPrivate.values = new Float64Array(record.values);
+                            chunkPrivate.histograms = record.histograms;
+                            chunkPrivate.cursor = record.cursor;
+                            chunkPrivate.minEndTime = record.minEndTime;
+                            chunkPrivate.maxEndTime = record.maxEndTime;
+                            chunkPrivate.isHistogram = record.isHistogram;
+                            const seriesPrivate = series;
+                            seriesPrivate.chunks.push(chunk);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+        catch {
+            // Silently fail if any parsing fails
+        }
+    }
+    /**
+     * Save registry to disk as NDJSON (one chunk per line)
+     */
+    saveToDisk(filePath) {
+        try {
+            const ndjsonData = this.serializeToNDJSON();
+            fs_1.default.writeFileSync(filePath, ndjsonData);
+        }
+        catch {
+            // Silently fail - don't interrupt operations
+        }
+    }
+    /**
+     * Load registry from disk (NDJSON format)
+     */
+    loadFromDisk(filePath) {
+        try {
+            if (!fs_1.default.existsSync(filePath)) {
+                return;
+            }
+            const ndjsonData = fs_1.default.readFileSync(filePath, 'utf-8');
+            this.deserializeFromNDJSON(ndjsonData);
+        }
+        catch {
+            // Silently fail during boot
+        }
+    }
+}
+exports.SeriesRegistry = SeriesRegistry;
+// Utility: Create deterministic ID from attributes
+function attributesToId(attributes) {
+    return Object.keys(attributes).sort().map(key => `${key}=${attributes[key]}`).join(',') || 'no_attrs';
+}
+function scopeToId(scope) {
+    return `${scope.name}@${scope.version ?? 'no_version'}`;
+}
+function makeMetricId(scopeId, metricName) {
+    return `${scopeId}:${metricName}`;
+}
+function makeSeriesId(metricId, attributesId) {
+    return `${metricId}$${attributesId}`;
+}

package/dist/cjs/telemetry/custom-implementations/metrics/tsdb/types.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/cjs/telemetry/custom-implementations/metrics/tsdb/utils.cjs

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rawToOtel = rawToOtel;
+/**
+ * Convert raw format to OpenTelemetry format
+ *
+ * Raw format structure (same for GET response and INSERT input):
+ * {
+ *   scope: { name, version },
+ *   descriptor: { unit, description, ... },
+ *   series: [{ attributes, startTimes[], endTimes[], values[] }]
+ * }
+ *
+ * OTEL format structure:
+ * {
+ *   scope: { name, version },
+ *   metrics: [{ descriptor, dataPoints: [{ attributes, startTime, endTime, value }] }]
+ * }
+ *
+ * Groups dataPoints by endTime to reconstruct original exports
+ */
+function rawToOtel(rawScopeMetrics) {
+    const groupedByScope = new Map();
+    for (const result of rawScopeMetrics) {
+        // Validate that result matches expected raw format
+        if (!result.scope) {
+            throw new Error(`Invalid metric format: Missing 'scope' property. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}. ` +
+                `Got: ${JSON.stringify(result).substring(0, 200)}`);
+        }
+        if (!result.descriptor) {
+            throw new Error(`Invalid metric format: Missing 'descriptor' property in metric. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}`);
+        }
+        if (!Array.isArray(result.series)) {
+            throw new Error(`Invalid metric format: 'series' must be an array. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}`);
+        }
+        const scopeId = result.scope.version
+            ? `${result.scope.name}@${result.scope.version}`
+            : `${result.scope.name}@none`;
+        if (!groupedByScope.has(scopeId)) {
+            groupedByScope.set(scopeId, {
+                scope: {
+                    name: result.scope.name,
+                    version: result.scope.version || ''
+                },
+                metrics: []
+            });
+        }
+        const scopeData = groupedByScope.get(scopeId);
+        const dataPoints = [];
+        for (const series of result.series) {
+            for (let i = 0; i < series.endTimes.length; i++) {
+                const startTimeNs = series.startTimes?.[i];
+                const endTimeNs = series.endTimes[i];
+                if (!startTimeNs)
+                    continue;
+                const startSec = Math.floor(startTimeNs / 1_000_000_000);
+                const startNano = startTimeNs % 1_000_000_000;
+                const endSec = Math.floor(endTimeNs / 1_000_000_000);
+                const endNano = endTimeNs % 1_000_000_000;
+                dataPoints.push({
+                    attributes: series.attributes,
+                    startTime: [startSec, startNano],
+                    endTime: [endSec, endNano],
+                    value: series.values[i]
+                });
+            }
+        }
+        scopeData.metrics.push({
+            descriptor: result.descriptor,
+            dataPoints
+        });
+    }
+    return Array.from(groupedByScope.values());
+}

package/dist/cjs/telemetry/custom-implementations/processors/dynamicMultiLogProcessor.cjs

@@ -1,70 +1,72 @@
 "use strict";
-
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
 exports.DynamicMultiLogRecordProcessor = void 0;
-var _core = require("@opentelemetry/core");
-var _logger = _interopRequireDefault(require("../../../utils/logger.cjs"));
-function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
-// optional if you want logging
+const core_1 = require("@opentelemetry/core");
+const logger_js_1 = __importDefault(require("../../../utils/logger.cjs")); // optional if you want logging
 class DynamicMultiLogRecordProcessor {
-  constructor(initialProcessors = [], forceFlushTimeoutMillis = 30000) {
-    this._processors = [];
-    this._processors = [...initialProcessors];
-    this._forceFlushTimeoutMillis = forceFlushTimeoutMillis;
-  }
-  /**
-   * Add a new LogRecordProcessor or an array of LogRecordProcessors at runtime.
-   */
-  addProcessors(processor) {
-    if (Array.isArray(processor)) {
-      this._processors.push(...processor);
-    } else {
-      this._processors.push(processor);
+    _processors = [];
+    _forceFlushTimeoutMillis;
+    constructor(initialProcessors = [], forceFlushTimeoutMillis = 30000) {
+        this._processors = [...initialProcessors];
+        this._forceFlushTimeoutMillis = forceFlushTimeoutMillis;
     }
-  }
-  /**
-   * Remove a specific LogRecordProcessor.
-   */
-  removeProcessor(processor) {
-    this._processors = this._processors.filter(p => p !== processor);
-  }
-  /**
-   * Clear all LogRecordProcessors.
-   */
-  clearProcessors() {
-    this._processors = [];
-  }
-  /**
-   * Called when a log record is emitted.
-   */
-  onEmit(logRecord, context) {
-    for (const processor of this._processors) {
-      try {
-        processor.onEmit(logRecord, context);
-      } catch (error) {
-        _logger.default?.error?.('Error in onEmit of LogRecordProcessor:', error);
-      }
+    /**
+     * Add a new LogRecordProcessor or an array of LogRecordProcessors at runtime.
+     */
+    addProcessors(processor) {
+        if (Array.isArray(processor)) {
+            this._processors.push(...processor);
+        }
+        else {
+            this._processors.push(processor);
+        }
     }
-  }
-  /**
-   * Force flush all processors with timeout.
-   */
-  async forceFlush() {
-    const timeout = this._forceFlushTimeoutMillis;
-    const promises = this._processors.map(p => (0, _core.callWithTimeout)(p.forceFlush(), timeout));
-    try {
-      await Promise.all(promises);
-    } catch (error) {
-      _logger.default?.error?.('Error during forceFlush in DynamicMultiLogRecordProcessor:', error);
+    /**
+     * Remove a specific LogRecordProcessor.
+     */
+    removeProcessor(processor) {
+        this._processors = this._processors.filter(p => p !== processor);
+    }
+    /**
+     * Clear all LogRecordProcessors.
+     */
+    clearProcessors() {
+        this._processors = [];
+    }
+    /**
+     * Called when a log record is emitted.
+     */
+    onEmit(logRecord, context) {
+        for (const processor of this._processors) {
+            try {
+                processor.onEmit(logRecord, context);
+            }
+            catch (error) {
+                logger_js_1.default?.error?.('Error in onEmit of LogRecordProcessor:', error);
+            }
+        }
+    }
+    /**
+     * Force flush all processors with timeout.
+     */
+    async forceFlush() {
+        const timeout = this._forceFlushTimeoutMillis;
+        const promises = this._processors.map(p => (0, core_1.callWithTimeout)(p.forceFlush(), timeout));
+        try {
+            await Promise.all(promises);
+        }
+        catch (error) {
+            logger_js_1.default?.error?.('Error during forceFlush in DynamicMultiLogRecordProcessor:', error);
+        }
+    }
+    /**
+     * Shutdown all processors.
+     */
+    async shutdown() {
+        await Promise.all(this._processors.map(p => p.shutdown()));
     }
-  }
-  /**
-   * Shutdown all processors.
-   */
-  async shutdown() {
-    await Promise.all(this._processors.map(p => p.shutdown()));
-  }
 }
-exports.DynamicMultiLogRecordProcessor = DynamicMultiLogRecordProcessor;
+exports.DynamicMultiLogRecordProcessor = DynamicMultiLogRecordProcessor;