@oas-tools/oas-telemetry 0.7.1 → 0.8.0-alpha.1

package/dist/esm/telemetry/custom-implementations/metrics/tsdb/Chunk.js

@@ -0,0 +1,155 @@
+export class Chunk {
+    maxSamples;
+    startTimes;
+    endTimes;
+    values;
+    histograms;
+    cursor = 0;
+    minEndTime = 0;
+    maxEndTime = 0;
+    isHistogram;
+    constructor(maxSamples = 120, isHistogram = false) {
+        this.maxSamples = maxSamples;
+        this.startTimes = new Float64Array(maxSamples);
+        this.endTimes = new Float64Array(maxSamples);
+        this.values = new Float64Array(maxSamples);
+        this.histograms = new Array(maxSamples).fill(null);
+        this.isHistogram = isHistogram;
+    }
+    /**
+     * Append a sample to the chunk.
+     * Returns true if successful or false if the chunk is full.
+     */
+    append(startTime, endTime, value) {
+        if (this.cursor >= this.maxSamples) {
+            return false;
+        }
+        if (this.cursor === 0) {
+            this.minEndTime = endTime;
+        }
+        this.startTimes[this.cursor] = startTime;
+        this.endTimes[this.cursor] = endTime;
+        if (this.isHistogram && typeof value === 'object') {
+            this.values[this.cursor] = value.count;
+            this.histograms[this.cursor] = value;
+        }
+        else if (typeof value === 'number') {
+            this.values[this.cursor] = value;
+            this.histograms[this.cursor] = null;
+        }
+        else {
+            const hv = value;
+            this.values[this.cursor] = hv.count;
+            this.histograms[this.cursor] = hv;
+        }
+        this.maxEndTime = endTime;
+        this.cursor++;
+        return true;
+    }
+    /**
+     * Binary search for first index with endTime >= targetTime
+     */
+    _binarySearchStart(targetTime) {
+        let left = 0;
+        let right = this.cursor - 1;
+        let result = this.cursor;
+        while (left <= right) {
+            const mid = (left + right) >> 1;
+            if (this.endTimes[mid] >= targetTime) {
+                result = mid;
+                right = mid - 1;
+            }
+            else {
+                left = mid + 1;
+            }
+        }
+        return result;
+    }
+    /**
+     * Binary search for last index with endTime <= targetTime
+     */
+    _binarySearchEnd(targetTime, startIdx = 0) {
+        let left = startIdx;
+        let right = this.cursor - 1;
+        let result = startIdx - 1;
+        while (left <= right) {
+            const mid = (left + right) >> 1;
+            if (this.endTimes[mid] <= targetTime) {
+                result = mid;
+                left = mid + 1;
+            }
+            else {
+                right = mid - 1;
+            }
+        }
+        return result;
+    }
+    /**
+     * Efficient slicing method for bulk operations.
+     * Always returns slices. Values can be either a Float64Array (numeric)
+     * or a JS array of HistogramValue objects (histogram series).
+     */
+    getSlices(startTime, endTime) {
+        const emptyResult = {
+            startTimes: new Float64Array(0),
+            endTimes: new Float64Array(0),
+            values: this.isHistogram ? [] : new Float64Array(0),
+        };
+        const start = startTime ?? 0;
+        const end = endTime ?? Number.MAX_SAFE_INTEGER;
+        if (this.cursor === 0 || this.maxEndTime < start || this.minEndTime > end) {
+            return emptyResult;
+        }
+        const startIdx = this._binarySearchStart(start);
+        const endIdx = this._binarySearchEnd(end, startIdx);
+        if (startIdx > endIdx) {
+            return emptyResult;
+        }
+        const startTimesNew = this.startTimes.subarray(startIdx, endIdx + 1);
+        const endTimesNew = this.endTimes.subarray(startIdx, endIdx + 1);
+        if (this.isHistogram) {
+            // Return histogram objects as a simple JS array slice
+            return {
+                startTimes: startTimesNew,
+                endTimes: endTimesNew,
+                values: this.histograms.slice(startIdx, endIdx + 1),
+            };
+        }
+        return {
+            startTimes: startTimesNew,
+            endTimes: endTimesNew,
+            values: this.values.subarray(startIdx, endIdx + 1),
+        };
+    }
+    isFull() {
+        return this.cursor >= this.maxSamples;
+    }
+    overlaps(startTime, endTime) {
+        if (this.cursor === 0)
+            return false;
+        return !(this.maxEndTime < startTime || this.minEndTime > endTime);
+    }
+    getStats() {
+        return {
+            samples: this.cursor,
+            maxSamples: this.maxSamples,
+            minEndTime: this.minEndTime,
+            maxEndTime: this.maxEndTime,
+            memoryBytes: this.getMemoryUsage(),
+        };
+    }
+    getMemoryUsage() {
+        const arrayMemory = this.maxSamples * 8 * 3;
+        const histogramMemory = this.histograms.filter(h => h !== null).length * 200;
+        return arrayMemory + histogramMemory;
+    }
+    getMinTime() {
+        return this.minEndTime;
+    }
+    getMaxTime() {
+        return this.maxEndTime;
+    }
+    size() {
+        return this.cursor;
+    }
+}

package/dist/esm/telemetry/custom-implementations/metrics/tsdb/Series.js

@@ -0,0 +1,164 @@
+/**
+ * Series: Time-series storage with metadata and chunks
+ * One series per unique metric+labels combination
+ */
+import { Chunk } from './Chunk.js';
+export class Series {
+    labelSet;
+    metadata;
+    chunks = [];
+    maxChunks;
+    chunkSize;
+    isHistogram;
+    constructor(labelSet, metadata, chunkSize = 120, maxChunks = 60 // ~1 hour at 1min intervals with 120 samples/chunk
+    ) {
+        this.labelSet = labelSet;
+        this.metadata = metadata;
+        this.chunkSize = chunkSize;
+        this.maxChunks = maxChunks;
+        this.isHistogram = metadata.dataPointType === 0; // HISTOGRAM = 0
+    }
+    /**
+     * Append a sample to the series
+     */
+    append(startTime, endTime, value) {
+        // Get or create current chunk
+        let currentChunk = this.chunks[this.chunks.length - 1];
+        if (!currentChunk || currentChunk.isFull()) {
+            // Create new chunk
+            currentChunk = new Chunk(this.chunkSize, this.isHistogram);
+            this.chunks.push(currentChunk);
+            // Enforce max chunks limit (circular buffer behavior)
+            if (this.chunks.length > this.maxChunks) {
+                this.chunks.shift(); // Remove oldest chunk
+            }
+        }
+        currentChunk.append(startTime, endTime, value);
+    }
+    /**
+     * Query slices with options object. Example:
+     *   querySlices({ startTime, endTime, includeStartTimes: true })
+     *   includeStartTimes defaults to false.
+     *   If both startTime and endTime are undefined, returns all data (no time filtering).
+     */
+    querySlices(options) {
+        const start = options?.startTime ?? 0;
+        const end = options?.endTime ?? Number.MAX_VALUE;
+        const includeStartTimes = options?.includeStartTimes ?? false;
+        let totalLength = 0;
+        for (const chunk of this.chunks) {
+            if (!chunk.overlaps(start, end))
+                continue;
+            const slice = chunk.getSlices(start, end);
+            totalLength += slice.startTimes.length;
+        }
+        if (totalLength === 0) {
+            const empty = {
+                endTimes: new Float64Array(0),
+                values: this.isHistogram ? [] : new Float64Array(0),
+            };
+            if (includeStartTimes) {
+                empty.startTimes = new Float64Array(0);
+            }
+            return empty;
+        }
+        let resultStart;
+        if (includeStartTimes) {
+            resultStart = new Float64Array(totalLength);
+        }
+        const resultEnd = new Float64Array(totalLength);
+        const resultValues = this.isHistogram
+            ? new Array(totalLength)
+            : new Float64Array(totalLength);
+        let offset = 0;
+        for (const chunk of this.chunks) {
+            if (!chunk.overlaps(start, end))
+                continue;
+            const slice = chunk.getSlices(start, end);
+            const len = slice.startTimes.length;
+            if (len === 0)
+                continue;
+            // Copy start and end times
+            if (includeStartTimes && resultStart) {
+                resultStart.set(slice.startTimes, offset);
+            }
+            resultEnd.set(slice.endTimes, offset);
+            // Copy values (different type depending on metric)
+            if (this.isHistogram) {
+                resultValues.splice(offset, len, ...slice.values);
+            }
+            else {
+                resultValues.set(slice.values, offset);
+            }
+            offset += len;
+        }
+        const result = {
+            endTimes: resultEnd,
+            values: resultValues
+        };
+        if (includeStartTimes && resultStart) {
+            result.startTimes = resultStart;
+        }
+        return result;
+    }
+    /**
+     * Remove chunks older than threshold
+     */
+    evictOldChunks(thresholdTime) {
+        let evicted = 0;
+        while (this.chunks.length > 0 && this.chunks[0].getMaxTime() < thresholdTime) {
+            this.chunks.shift();
+            evicted++;
+        }
+        return evicted;
+    }
+    /**
+     * Get series metadata
+     */
+    getMetadata() {
+        return this.metadata;
+    }
+    getLabels() {
+        return this.labelSet.labels;
+    }
+    getOriginalAttributes() {
+        return this.labelSet.originalAttributes;
+    }
+    /**
+     * Get label hash
+     */
+    getLabelHash() {
+        return this.labelSet.hash;
+    }
+    getStats() {
+        const totalSamples = this.chunks.reduce((sum, chunk) => sum + chunk.size(), 0);
+        const memoryBytes = this.chunks.reduce((sum, chunk) => sum + chunk.getMemoryUsage(), 0);
+        return {
+            metricName: this.metadata?.descriptor?.name ?? 'unknown',
+            labels: this.labelSet?.labels ?? {},
+            chunks: this.chunks.length,
+            samples: totalSamples,
+            memoryBytes,
+            oldestTime: this.chunks[0]?.getMinTime() ?? 0,
+            newestTime: this.chunks[this.chunks.length - 1]?.getMaxTime() ?? 0
+        };
+    }
+    /**
+     * Check if series has any samples
+     */
+    isEmpty() {
+        return this.chunks.length === 0 || this.chunks.every(c => c.size() === 0);
+    }
+    /**
+     * Get time range covered by this series
+     */
+    getTimeRange() {
+        if (this.chunks.length === 0) {
+            return { min: 0, max: 0 };
+        }
+        return {
+            min: this.chunks[0].getMinTime(),
+            max: this.chunks[this.chunks.length - 1].getMaxTime()
+        };
+    }
+}

package/dist/esm/telemetry/custom-implementations/metrics/tsdb/SeriesRegistry.js

@@ -0,0 +1,385 @@
+import { Series } from './Series.js';
+import { Chunk } from './Chunk.js';
+/*
+OTEL export = Resource (unchanged) + SCOPE METRICS
+scopeMetrics: {
+    scope: {name: library, version: 1.0.0},; //repeated for each metric
+    metrics: [
+        {
+            //Metadata (Repeated for each dataPoint)
+            descriptor: {name: metricName, unit, description, ...},
+            aggregationTemporality,
+            dataPointType,
+            // Actual dataPoints are stored in Series
+            dataPoints: []
+        }
+    ]
+}
+*/
+// Import fs at top level for disk operations
+import fs from 'fs';
+import logger from '../../../../utils/logger.js';
+export class SeriesRegistry {
+    // Main storage
+    series = new Map(); // seriesId (scopeId:metricName:attributesId) -> Series
+    // Cached repeated data
+    scopes = new Map(); // scopeId -> scope
+    metricMetadataMap = new Map(); // "scopeId:metricName" -> MetricMetadata
+    // Faster lookup index
+    metricIdIndex = new Map(); // "scopeId:metricName" -> Set<seriesId>
+    chunkSize;
+    maxChunks;
+    constructor(chunkSize = 120, maxChunks = 60) {
+        this.chunkSize = chunkSize;
+        this.maxChunks = maxChunks;
+    }
+    /**
+     * Store entire ScopeMetrics array efficiently
+     * Replaces addSample - processes all metrics in one batch
+     */
+    storeScopeMetrics(scopeMetrics) {
+        for (const scopeMetric of scopeMetrics) {
+            const scope = scopeMetric.scope;
+            const scopeId = scopeToId(scope);
+            this.scopes.set(scopeId, scope);
+            for (const metricData of scopeMetric.metrics) {
+                const metricName = metricData.descriptor.name;
+                const metricId = makeMetricId(scopeId, metricName);
+                if (!this.metricMetadataMap.has(metricId)) {
+                    const { dataPoints: _dataPoints, ...metadata } = metricData;
+                    this.metricMetadataMap.set(metricId, metadata);
+                }
+                for (const dp of metricData.dataPoints) {
+                    const startTime = dp.startTime[0] * 1_000_000_000 + dp.startTime[1];
+                    const endTime = dp.endTime[0] * 1_000_000_000 + dp.endTime[1];
+                    const attributes = dp.attributes;
+                    const attributesId = attributesToId(attributes);
+                    const seriesId = makeSeriesId(metricId, attributesId);
+                    if (!this.series.has(seriesId)) {
+                        this.series.set(seriesId, new Series({ hash: 0, labels: attributes, originalAttributes: attributes }, this.metricMetadataMap.get(metricId), this.chunkSize, this.maxChunks));
+                        (this.metricIdIndex.get(metricId) ?? this.metricIdIndex.set(metricId, new Set()).get(metricId)).add(seriesId);
+                    }
+                    this.series.get(seriesId).append(startTime, endTime, dp.value);
+                }
+            }
+        }
+    }
+    query(scopeMetrics, startTime, endTime) {
+        // If no queries provided, get all scopeMetrics
+        if (!scopeMetrics || scopeMetrics.length === 0) {
+            const queries = Array.from(this.metricIdIndex.keys()).map(indexKey => {
+                const [scopeId, metricName] = indexKey.split(':').slice(0, 2);
+                const scope = this.scopes.get(scopeId);
+                return {
+                    scope,
+                    descriptor: { name: metricName },
+                    filters: undefined
+                };
+            });
+            return queries
+                .map(query => this._querySingle(query, startTime, endTime))
+                .filter((result) => result !== null);
+        }
+        return scopeMetrics
+            .map(query => this._querySingle(query, startTime, endTime))
+            .filter((result) => result !== null);
+    }
+    /**
+     * Query single metric with attribute filters
+     */
+    _querySingle(query, startTime, endTime) {
+        const scopeId = scopeToId(query.scope);
+        const indexKey = makeMetricId(scopeId, query.descriptor.name);
+        const seriesKeysInMetric = this.metricIdIndex.get(indexKey);
+        if (!seriesKeysInMetric || seriesKeysInMetric.size === 0) {
+            return null;
+        }
+        const scope = this.scopes.get(scopeId);
+        const metricData = this.metricMetadataMap.get(indexKey);
+        if (!scope || !metricData) {
+            return null;
+        }
+        // Filter by attributes if provided
+        const filteredSeriesKeys = query.filters
+            ? Array.from(seriesKeysInMetric).filter(key => {
+                const series = this.series.get(key);
+                return this._matchesAttributeFilters(series, query.filters);
+            })
+            : Array.from(seriesKeysInMetric);
+        if (filteredSeriesKeys.length === 0) {
+            return null;
+        }
+        const matchingSeries = filteredSeriesKeys
+            .map(key => {
+            const series = this.series.get(key);
+            const { startTimes, endTimes, values } = series.querySlices({ startTime, endTime, includeStartTimes: true });
+            return {
+                id: key,
+                attributes: series.getOriginalAttributes(),
+                startTimes: startTimes ? Array.from(startTimes) : undefined,
+                endTimes: Array.from(endTimes),
+                values: Array.isArray(values) ? values : Array.from(values)
+            };
+        })
+            .filter(s => s.endTimes.length > 0);
+        return {
+            scope,
+            descriptor: metricData.descriptor,
+            series: matchingSeries
+        };
+    }
+    /**
+     * Match series attributes against filters
+     * Supports exact match, negation (!), and regex (~)
+     */
+    _matchesAttributeFilters(series, filters) {
+        if (!filters || Object.keys(filters).length === 0) {
+            return true;
+        }
+        const attrs = series.getOriginalAttributes();
+        for (const [key, filterValue] of Object.entries(filters)) {
+            const attrValue = String(attrs[key] ?? '');
+            // Regex match: status=~4.*
+            if (filterValue.startsWith('~')) {
+                const pattern = filterValue.slice(1);
+                try {
+                    const regex = new RegExp(pattern);
+                    if (!regex.test(attrValue)) {
+                        return false;
+                    }
+                }
+                catch {
+                    if (attrValue !== filterValue) {
+                        return false;
+                    }
+                }
+            }
+            // Negation: status=!200
+            else if (filterValue.startsWith('!')) {
+                const negatedValue = filterValue.slice(1);
+                if (attrValue === negatedValue) {
+                    return false;
+                }
+            }
+            // Exact match: method=GET
+            else {
+                if (attrValue !== filterValue) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+    evictOldData(retentionTimeNs) {
+        const thresholdTime = Date.now() * 1_000_000 - retentionTimeNs;
+        let evictedChunks = 0;
+        let evictedSeries = 0;
+        for (const [key, series] of this.series.entries()) {
+            evictedChunks += series.evictOldChunks(thresholdTime);
+            if (series.isEmpty()) {
+                this.series.delete(key);
+                evictedSeries++;
+            }
+        }
+        return { evictedChunks, evictedSeries };
+    }
+    getStats() {
+        const seriesStats = Array.from(this.series.values()).map(s => s.getStats());
+        const totalSamples = seriesStats.reduce((sum, s) => sum + s.samples, 0);
+        const totalMemory = seriesStats.reduce((sum, s) => sum + s.memoryBytes, 0);
+        return {
+            totalMetrics: this.metricMetadataMap.size,
+            totalScopes: this.scopes.size,
+            totalSeries: this.series.size,
+            totalSamples,
+            memoryUsageBytes: totalMemory
+        };
+    }
+    reset() {
+        this.series.clear();
+        this.scopes.clear();
+        this.metricMetadataMap.clear();
+        this.metricIdIndex.clear();
+    }
+    size() {
+        return this.series.size;
+    }
+    /**
+     * Serialize registry to line-delimited JSON format (newline-delimited JSON)
+     * Each line is a complete record: metadata, scope, metric, or data point
+     * Format:
+     *   {"type":"header","version":1,"timestamp":"...","stats":{...}}
+     *   {"type":"scope","id":"...","data":{...}}
+     *   {"type":"metric","id":"...","data":{...}}
+     *   {"type":"series","id":"...","labelSet":{...},"chunks":[...]}
+     */
+    serializeToLineDelimitedJson() {
+        const lines = [];
+        // Header
+        lines.push(JSON.stringify({
+            type: 'header',
+            version: 1,
+            timestamp: new Date().toISOString(),
+            stats: this.getStats()
+        }));
+        // Scopes
+        for (const [scopeId, scope] of this.scopes.entries()) {
+            lines.push(JSON.stringify({
+                type: 'scope',
+                id: scopeId,
+                data: scope
+            }));
+        }
+        // Metric metadata
+        for (const [metricId, metadata] of this.metricMetadataMap.entries()) {
+            lines.push(JSON.stringify({
+                type: 'metric',
+                id: metricId,
+                data: metadata
+            }));
+        }
+        // Series headers + chunks (one chunk per line for streaming efficiency)
+        const serializedSeriesHeaders = new Set();
+        for (const [seriesId, series] of this.series.entries()) {
+            const seriesPrivate = series;
+            // Emit series header once per series
+            if (!serializedSeriesHeaders.has(seriesId)) {
+                lines.push(JSON.stringify({
+                    type: 'series',
+                    seriesId: seriesId,
+                    labelSet: seriesPrivate.labelSet,
+                    metadata: seriesPrivate.metadata
+                }));
+                serializedSeriesHeaders.add(seriesId);
+            }
+            (seriesPrivate.chunks || []).forEach((chunk, chunkIndex) => {
+                const slicedStartTimes = chunk.startTimes.slice(0, chunk.cursor);
+                const slicedEndTimes = chunk.endTimes.slice(0, chunk.cursor);
+                const slicedValues = chunk.values.slice(0, chunk.cursor);
+                const slicedHistograms = chunk.histograms.slice(0, chunk.cursor);
+                lines.push(JSON.stringify({
+                    type: 'chunk',
+                    seriesId: seriesId,
+                    chunkIndex: chunkIndex,
+                    startTimes: Array.from(slicedStartTimes),
+                    endTimes: Array.from(slicedEndTimes),
+                    values: Array.from(slicedValues),
+                    histograms: Array.from(slicedHistograms),
+                    cursor: chunk.cursor,
+                    minEndTime: chunk.minEndTime,
+                    maxEndTime: chunk.maxEndTime,
+                    isHistogram: chunk.isHistogram
+                }));
+            });
+        }
+        // Metric ID index for fast lookup
+        for (const [metricId, seriesIdSet] of this.metricIdIndex.entries()) {
+            lines.push(JSON.stringify({
+                type: 'index',
+                id: metricId,
+                seriesIds: Array.from(seriesIdSet)
+            }));
+        }
+        return lines.join('\n');
+    }
+    /**
+     * Deserialize from line-delimited JSON format - restore from chunk lines
+     */
+    deserializeFromLineDelimitedJson(lineDelimitedJsonData) {
+        try {
+            const lines = lineDelimitedJsonData.trim().split('\n');
+            for (const line of lines) {
+                if (!line.trim())
+                    continue;
+                const record = JSON.parse(line);
+                switch (record.type) {
+                    case 'header':
+                        // Just metadata
+                        break;
+                    case 'scope':
+                        this.scopes.set(record.id, record.data);
+                        break;
+                    case 'metric':
+                        this.metricMetadataMap.set(record.id, record.data);
+                        break;
+                    case 'index':
+                        this.metricIdIndex.set(record.id, new Set(record.seriesIds));
+                        break;
+                    case 'series': {
+                        // Create series stub - will be populated by subsequent chunk lines
+                        const metadata = this.metricMetadataMap.get(record.seriesId.substring(0, record.seriesId.lastIndexOf('$'))) || record.metadata;
+                        if (metadata) {
+                            const series = new Series(record.labelSet, metadata, this.chunkSize, this.maxChunks);
+                            this.series.set(record.seriesId, series);
+                        }
+                        break;
+                    }
+                    case 'chunk': {
+                        // Restore chunk to series
+                        const series = this.series.get(record.seriesId);
+                        if (series) {
+                            const chunk = new Chunk(record.cursor || record.startTimes.length, record.isHistogram);
+                            const chunkPrivate = chunk;
+                            chunkPrivate.startTimes = new Float64Array(record.startTimes);
+                            chunkPrivate.endTimes = new Float64Array(record.endTimes);
+                            chunkPrivate.values = new Float64Array(record.values);
+                            chunkPrivate.histograms = record.histograms;
+                            chunkPrivate.cursor = record.cursor;
+                            chunkPrivate.minEndTime = record.minEndTime;
+                            chunkPrivate.maxEndTime = record.maxEndTime;
+                            chunkPrivate.isHistogram = record.isHistogram;
+                            const seriesPrivate = series;
+                            seriesPrivate.chunks.push(chunk);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+        catch {
+            logger.error(`Failed to deserialize metrics from line-delimited JSON`);
+        }
+    }
+    /**
+     * Save registry to disk as line-delimited JSON (one chunk per line)
+     */
+    saveToDisk(filePath) {
+        try {
+            const lineDelimitedJsonData = this.serializeToLineDelimitedJson();
+            fs.writeFileSync(filePath, lineDelimitedJsonData);
+        }
+        catch {
+            logger.error(`Failed to save metrics to disk at ${filePath}`);
+        }
+    }
+    /**
+     * Load registry from disk (line-delimited JSON format)
+     */
+    loadFromDisk(filePath) {
+        try {
+            if (!fs.existsSync(filePath)) {
+                return;
+            }
+            const lineDelimitedJsonData = fs.readFileSync(filePath, 'utf-8');
+            this.deserializeFromLineDelimitedJson(lineDelimitedJsonData);
+        }
+        catch {
+            logger.error(`Failed to load metrics from disk at ${filePath}`);
+            // Fallback to memory.
+            this.reset();
+        }
+    }
+}
+// Utility: Create deterministic ID from attributes
+function attributesToId(attributes) {
+    return Object.keys(attributes).sort().map(key => `${key}=${attributes[key]}`).join(',') || 'no_attrs';
+}
+function scopeToId(scope) {
+    return `${scope.name}@${scope.version ?? 'no_version'}`;
+}
+function makeMetricId(scopeId, metricName) {
+    return `${scopeId}:${metricName}`;
+}
+function makeSeriesId(metricId, attributesId) {
+    return `${metricId}$${attributesId}`;
+}