@oas-tools/oas-telemetry 0.7.1 → 0.8.0-alpha.1

package/dist/cjs/telemetry/custom-implementations/metrics/tsdb/Series.cjs

@@ -0,0 +1,168 @@
+"use strict";
+/**
+ * Series: Time-series storage with metadata and chunks
+ * One series per unique metric+labels combination
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Series = void 0;
+const Chunk_js_1 = require("./Chunk.cjs");
+class Series {
+    labelSet;
+    metadata;
+    chunks = [];
+    maxChunks;
+    chunkSize;
+    isHistogram;
+    constructor(labelSet, metadata, chunkSize = 120, maxChunks = 60 // ~1 hour at 1min intervals with 120 samples/chunk
+    ) {
+        this.labelSet = labelSet;
+        this.metadata = metadata;
+        this.chunkSize = chunkSize;
+        this.maxChunks = maxChunks;
+        this.isHistogram = metadata.dataPointType === 0; // HISTOGRAM = 0
+    }
+    /**
+     * Append a sample to the series
+     */
+    append(startTime, endTime, value) {
+        // Get or create current chunk
+        let currentChunk = this.chunks[this.chunks.length - 1];
+        if (!currentChunk || currentChunk.isFull()) {
+            // Create new chunk
+            currentChunk = new Chunk_js_1.Chunk(this.chunkSize, this.isHistogram);
+            this.chunks.push(currentChunk);
+            // Enforce max chunks limit (circular buffer behavior)
+            if (this.chunks.length > this.maxChunks) {
+                this.chunks.shift(); // Remove oldest chunk
+            }
+        }
+        currentChunk.append(startTime, endTime, value);
+    }
+    /**
+     * Query slices with options object. Example:
+     *   querySlices({ startTime, endTime, includeStartTimes: true })
+     *   includeStartTimes defaults to false.
+     *   If both startTime and endTime are undefined, returns all data (no time filtering).
+     */
+    querySlices(options) {
+        const start = options?.startTime ?? 0;
+        const end = options?.endTime ?? Number.MAX_VALUE;
+        const includeStartTimes = options?.includeStartTimes ?? false;
+        let totalLength = 0;
+        for (const chunk of this.chunks) {
+            if (!chunk.overlaps(start, end))
+                continue;
+            const slice = chunk.getSlices(start, end);
+            totalLength += slice.startTimes.length;
+        }
+        if (totalLength === 0) {
+            const empty = {
+                endTimes: new Float64Array(0),
+                values: this.isHistogram ? [] : new Float64Array(0),
+            };
+            if (includeStartTimes) {
+                empty.startTimes = new Float64Array(0);
+            }
+            return empty;
+        }
+        let resultStart;
+        if (includeStartTimes) {
+            resultStart = new Float64Array(totalLength);
+        }
+        const resultEnd = new Float64Array(totalLength);
+        const resultValues = this.isHistogram
+            ? new Array(totalLength)
+            : new Float64Array(totalLength);
+        let offset = 0;
+        for (const chunk of this.chunks) {
+            if (!chunk.overlaps(start, end))
+                continue;
+            const slice = chunk.getSlices(start, end);
+            const len = slice.startTimes.length;
+            if (len === 0)
+                continue;
+            // Copy start and end times
+            if (includeStartTimes && resultStart) {
+                resultStart.set(slice.startTimes, offset);
+            }
+            resultEnd.set(slice.endTimes, offset);
+            // Copy values (different type depending on metric)
+            if (this.isHistogram) {
+                resultValues.splice(offset, len, ...slice.values);
+            }
+            else {
+                resultValues.set(slice.values, offset);
+            }
+            offset += len;
+        }
+        const result = {
+            endTimes: resultEnd,
+            values: resultValues
+        };
+        if (includeStartTimes && resultStart) {
+            result.startTimes = resultStart;
+        }
+        return result;
+    }
+    /**
+     * Remove chunks older than threshold
+     */
+    evictOldChunks(thresholdTime) {
+        let evicted = 0;
+        while (this.chunks.length > 0 && this.chunks[0].getMaxTime() < thresholdTime) {
+            this.chunks.shift();
+            evicted++;
+        }
+        return evicted;
+    }
+    /**
+     * Get series metadata
+     */
+    getMetadata() {
+        return this.metadata;
+    }
+    getLabels() {
+        return this.labelSet.labels;
+    }
+    getOriginalAttributes() {
+        return this.labelSet.originalAttributes;
+    }
+    /**
+     * Get label hash
+     */
+    getLabelHash() {
+        return this.labelSet.hash;
+    }
+    getStats() {
+        const totalSamples = this.chunks.reduce((sum, chunk) => sum + chunk.size(), 0);
+        const memoryBytes = this.chunks.reduce((sum, chunk) => sum + chunk.getMemoryUsage(), 0);
+        return {
+            metricName: this.metadata?.descriptor?.name ?? 'unknown',
+            labels: this.labelSet?.labels ?? {},
+            chunks: this.chunks.length,
+            samples: totalSamples,
+            memoryBytes,
+            oldestTime: this.chunks[0]?.getMinTime() ?? 0,
+            newestTime: this.chunks[this.chunks.length - 1]?.getMaxTime() ?? 0
+        };
+    }
+    /**
+     * Check if series has any samples
+     */
+    isEmpty() {
+        return this.chunks.length === 0 || this.chunks.every(c => c.size() === 0);
+    }
+    /**
+     * Get time range covered by this series
+     */
+    getTimeRange() {
+        if (this.chunks.length === 0) {
+            return { min: 0, max: 0 };
+        }
+        return {
+            min: this.chunks[0].getMinTime(),
+            max: this.chunks[this.chunks.length - 1].getMaxTime()
+        };
+    }
+}
+exports.Series = Series;

package/dist/cjs/telemetry/custom-implementations/metrics/tsdb/SeriesRegistry.cjs

@@ -0,0 +1,392 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SeriesRegistry = void 0;
+const Series_js_1 = require("./Series.cjs");
+const Chunk_js_1 = require("./Chunk.cjs");
+/*
+OTEL export = Resource (unchanged) + SCOPE METRICS
+scopeMetrics: {
+    scope: {name: library, version: 1.0.0},; //repeated for each metric
+    metrics: [
+        {
+            //Metadata (Repeated for each dataPoint)
+            descriptor: {name: metricName, unit, description, ...},
+            aggregationTemporality,
+            dataPointType,
+            // Actual dataPoints are stored in Series
+            dataPoints: []
+        }
+    ]
+}
+*/
+// Import fs at top level for disk operations
+const fs_1 = __importDefault(require("fs"));
+const logger_js_1 = __importDefault(require("../../../../utils/logger.cjs"));
+class SeriesRegistry {
+    // Main storage
+    series = new Map(); // seriesId (scopeId:metricName:attributesId) -> Series
+    // Cached repeated data
+    scopes = new Map(); // scopeId -> scope
+    metricMetadataMap = new Map(); // "scopeId:metricName" -> MetricMetadata
+    // Faster lookup index
+    metricIdIndex = new Map(); // "scopeId:metricName" -> Set<seriesId>
+    chunkSize;
+    maxChunks;
+    constructor(chunkSize = 120, maxChunks = 60) {
+        this.chunkSize = chunkSize;
+        this.maxChunks = maxChunks;
+    }
+    /**
+     * Store entire ScopeMetrics array efficiently
+     * Replaces addSample - processes all metrics in one batch
+     */
+    storeScopeMetrics(scopeMetrics) {
+        for (const scopeMetric of scopeMetrics) {
+            const scope = scopeMetric.scope;
+            const scopeId = scopeToId(scope);
+            this.scopes.set(scopeId, scope);
+            for (const metricData of scopeMetric.metrics) {
+                const metricName = metricData.descriptor.name;
+                const metricId = makeMetricId(scopeId, metricName);
+                if (!this.metricMetadataMap.has(metricId)) {
+                    const { dataPoints: _dataPoints, ...metadata } = metricData;
+                    this.metricMetadataMap.set(metricId, metadata);
+                }
+                for (const dp of metricData.dataPoints) {
+                    const startTime = dp.startTime[0] * 1_000_000_000 + dp.startTime[1];
+                    const endTime = dp.endTime[0] * 1_000_000_000 + dp.endTime[1];
+                    const attributes = dp.attributes;
+                    const attributesId = attributesToId(attributes);
+                    const seriesId = makeSeriesId(metricId, attributesId);
+                    if (!this.series.has(seriesId)) {
+                        this.series.set(seriesId, new Series_js_1.Series({ hash: 0, labels: attributes, originalAttributes: attributes }, this.metricMetadataMap.get(metricId), this.chunkSize, this.maxChunks));
+                        (this.metricIdIndex.get(metricId) ?? this.metricIdIndex.set(metricId, new Set()).get(metricId)).add(seriesId);
+                    }
+                    this.series.get(seriesId).append(startTime, endTime, dp.value);
+                }
+            }
+        }
+    }
+    query(scopeMetrics, startTime, endTime) {
+        // If no queries provided, get all scopeMetrics
+        if (!scopeMetrics || scopeMetrics.length === 0) {
+            const queries = Array.from(this.metricIdIndex.keys()).map(indexKey => {
+                const [scopeId, metricName] = indexKey.split(':').slice(0, 2);
+                const scope = this.scopes.get(scopeId);
+                return {
+                    scope,
+                    descriptor: { name: metricName },
+                    filters: undefined
+                };
+            });
+            return queries
+                .map(query => this._querySingle(query, startTime, endTime))
+                .filter((result) => result !== null);
+        }
+        return scopeMetrics
+            .map(query => this._querySingle(query, startTime, endTime))
+            .filter((result) => result !== null);
+    }
+    /**
+     * Query single metric with attribute filters
+     */
+    _querySingle(query, startTime, endTime) {
+        const scopeId = scopeToId(query.scope);
+        const indexKey = makeMetricId(scopeId, query.descriptor.name);
+        const seriesKeysInMetric = this.metricIdIndex.get(indexKey);
+        if (!seriesKeysInMetric || seriesKeysInMetric.size === 0) {
+            return null;
+        }
+        const scope = this.scopes.get(scopeId);
+        const metricData = this.metricMetadataMap.get(indexKey);
+        if (!scope || !metricData) {
+            return null;
+        }
+        // Filter by attributes if provided
+        const filteredSeriesKeys = query.filters
+            ? Array.from(seriesKeysInMetric).filter(key => {
+                const series = this.series.get(key);
+                return this._matchesAttributeFilters(series, query.filters);
+            })
+            : Array.from(seriesKeysInMetric);
+        if (filteredSeriesKeys.length === 0) {
+            return null;
+        }
+        const matchingSeries = filteredSeriesKeys
+            .map(key => {
+            const series = this.series.get(key);
+            const { startTimes, endTimes, values } = series.querySlices({ startTime, endTime, includeStartTimes: true });
+            return {
+                id: key,
+                attributes: series.getOriginalAttributes(),
+                startTimes: startTimes ? Array.from(startTimes) : undefined,
+                endTimes: Array.from(endTimes),
+                values: Array.isArray(values) ? values : Array.from(values)
+            };
+        })
+            .filter(s => s.endTimes.length > 0);
+        return {
+            scope,
+            descriptor: metricData.descriptor,
+            series: matchingSeries
+        };
+    }
+    /**
+     * Match series attributes against filters
+     * Supports exact match, negation (!), and regex (~)
+     */
+    _matchesAttributeFilters(series, filters) {
+        if (!filters || Object.keys(filters).length === 0) {
+            return true;
+        }
+        const attrs = series.getOriginalAttributes();
+        for (const [key, filterValue] of Object.entries(filters)) {
+            const attrValue = String(attrs[key] ?? '');
+            // Regex match: status=~4.*
+            if (filterValue.startsWith('~')) {
+                const pattern = filterValue.slice(1);
+                try {
+                    const regex = new RegExp(pattern);
+                    if (!regex.test(attrValue)) {
+                        return false;
+                    }
+                }
+                catch {
+                    if (attrValue !== filterValue) {
+                        return false;
+                    }
+                }
+            }
+            // Negation: status=!200
+            else if (filterValue.startsWith('!')) {
+                const negatedValue = filterValue.slice(1);
+                if (attrValue === negatedValue) {
+                    return false;
+                }
+            }
+            // Exact match: method=GET
+            else {
+                if (attrValue !== filterValue) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+    evictOldData(retentionTimeNs) {
+        const thresholdTime = Date.now() * 1_000_000 - retentionTimeNs;
+        let evictedChunks = 0;
+        let evictedSeries = 0;
+        for (const [key, series] of this.series.entries()) {
+            evictedChunks += series.evictOldChunks(thresholdTime);
+            if (series.isEmpty()) {
+                this.series.delete(key);
+                evictedSeries++;
+            }
+        }
+        return { evictedChunks, evictedSeries };
+    }
+    getStats() {
+        const seriesStats = Array.from(this.series.values()).map(s => s.getStats());
+        const totalSamples = seriesStats.reduce((sum, s) => sum + s.samples, 0);
+        const totalMemory = seriesStats.reduce((sum, s) => sum + s.memoryBytes, 0);
+        return {
+            totalMetrics: this.metricMetadataMap.size,
+            totalScopes: this.scopes.size,
+            totalSeries: this.series.size,
+            totalSamples,
+            memoryUsageBytes: totalMemory
+        };
+    }
+    reset() {
+        this.series.clear();
+        this.scopes.clear();
+        this.metricMetadataMap.clear();
+        this.metricIdIndex.clear();
+    }
+    size() {
+        return this.series.size;
+    }
+    /**
+     * Serialize registry to line-delimited JSON format (newline-delimited JSON)
+     * Each line is a complete record: metadata, scope, metric, or data point
+     * Format:
+     *   {"type":"header","version":1,"timestamp":"...","stats":{...}}
+     *   {"type":"scope","id":"...","data":{...}}
+     *   {"type":"metric","id":"...","data":{...}}
+     *   {"type":"series","id":"...","labelSet":{...},"chunks":[...]}
+     */
+    serializeToLineDelimitedJson() {
+        const lines = [];
+        // Header
+        lines.push(JSON.stringify({
+            type: 'header',
+            version: 1,
+            timestamp: new Date().toISOString(),
+            stats: this.getStats()
+        }));
+        // Scopes
+        for (const [scopeId, scope] of this.scopes.entries()) {
+            lines.push(JSON.stringify({
+                type: 'scope',
+                id: scopeId,
+                data: scope
+            }));
+        }
+        // Metric metadata
+        for (const [metricId, metadata] of this.metricMetadataMap.entries()) {
+            lines.push(JSON.stringify({
+                type: 'metric',
+                id: metricId,
+                data: metadata
+            }));
+        }
+        // Series headers + chunks (one chunk per line for streaming efficiency)
+        const serializedSeriesHeaders = new Set();
+        for (const [seriesId, series] of this.series.entries()) {
+            const seriesPrivate = series;
+            // Emit series header once per series
+            if (!serializedSeriesHeaders.has(seriesId)) {
+                lines.push(JSON.stringify({
+                    type: 'series',
+                    seriesId: seriesId,
+                    labelSet: seriesPrivate.labelSet,
+                    metadata: seriesPrivate.metadata
+                }));
+                serializedSeriesHeaders.add(seriesId);
+            }
+            (seriesPrivate.chunks || []).forEach((chunk, chunkIndex) => {
+                const slicedStartTimes = chunk.startTimes.slice(0, chunk.cursor);
+                const slicedEndTimes = chunk.endTimes.slice(0, chunk.cursor);
+                const slicedValues = chunk.values.slice(0, chunk.cursor);
+                const slicedHistograms = chunk.histograms.slice(0, chunk.cursor);
+                lines.push(JSON.stringify({
+                    type: 'chunk',
+                    seriesId: seriesId,
+                    chunkIndex: chunkIndex,
+                    startTimes: Array.from(slicedStartTimes),
+                    endTimes: Array.from(slicedEndTimes),
+                    values: Array.from(slicedValues),
+                    histograms: Array.from(slicedHistograms),
+                    cursor: chunk.cursor,
+                    minEndTime: chunk.minEndTime,
+                    maxEndTime: chunk.maxEndTime,
+                    isHistogram: chunk.isHistogram
+                }));
+            });
+        }
+        // Metric ID index for fast lookup
+        for (const [metricId, seriesIdSet] of this.metricIdIndex.entries()) {
+            lines.push(JSON.stringify({
+                type: 'index',
+                id: metricId,
+                seriesIds: Array.from(seriesIdSet)
+            }));
+        }
+        return lines.join('\n');
+    }
+    /**
+     * Deserialize from line-delimited JSON format - restore from chunk lines
+     */
+    deserializeFromLineDelimitedJson(lineDelimitedJsonData) {
+        try {
+            const lines = lineDelimitedJsonData.trim().split('\n');
+            for (const line of lines) {
+                if (!line.trim())
+                    continue;
+                const record = JSON.parse(line);
+                switch (record.type) {
+                    case 'header':
+                        // Just metadata
+                        break;
+                    case 'scope':
+                        this.scopes.set(record.id, record.data);
+                        break;
+                    case 'metric':
+                        this.metricMetadataMap.set(record.id, record.data);
+                        break;
+                    case 'index':
+                        this.metricIdIndex.set(record.id, new Set(record.seriesIds));
+                        break;
+                    case 'series': {
+                        // Create series stub - will be populated by subsequent chunk lines
+                        const metadata = this.metricMetadataMap.get(record.seriesId.substring(0, record.seriesId.lastIndexOf('$'))) || record.metadata;
+                        if (metadata) {
+                            const series = new Series_js_1.Series(record.labelSet, metadata, this.chunkSize, this.maxChunks);
+                            this.series.set(record.seriesId, series);
+                        }
+                        break;
+                    }
+                    case 'chunk': {
+                        // Restore chunk to series
+                        const series = this.series.get(record.seriesId);
+                        if (series) {
+                            const chunk = new Chunk_js_1.Chunk(record.cursor || record.startTimes.length, record.isHistogram);
+                            const chunkPrivate = chunk;
+                            chunkPrivate.startTimes = new Float64Array(record.startTimes);
+                            chunkPrivate.endTimes = new Float64Array(record.endTimes);
+                            chunkPrivate.values = new Float64Array(record.values);
+                            chunkPrivate.histograms = record.histograms;
+                            chunkPrivate.cursor = record.cursor;
+                            chunkPrivate.minEndTime = record.minEndTime;
+                            chunkPrivate.maxEndTime = record.maxEndTime;
+                            chunkPrivate.isHistogram = record.isHistogram;
+                            const seriesPrivate = series;
+                            seriesPrivate.chunks.push(chunk);
+                        }
+                        break;
+                    }
+                }
+            }
+        }
+        catch {
+            logger_js_1.default.error(`Failed to deserialize metrics from line-delimited JSON`);
+        }
+    }
+    /**
+     * Save registry to disk as line-delimited JSON (one chunk per line)
+     */
+    saveToDisk(filePath) {
+        try {
+            const lineDelimitedJsonData = this.serializeToLineDelimitedJson();
+            fs_1.default.writeFileSync(filePath, lineDelimitedJsonData);
+        }
+        catch {
+            logger_js_1.default.error(`Failed to save metrics to disk at ${filePath}`);
+        }
+    }
+    /**
+     * Load registry from disk (line-delimited JSON format)
+     */
+    loadFromDisk(filePath) {
+        try {
+            if (!fs_1.default.existsSync(filePath)) {
+                return;
+            }
+            const lineDelimitedJsonData = fs_1.default.readFileSync(filePath, 'utf-8');
+            this.deserializeFromLineDelimitedJson(lineDelimitedJsonData);
+        }
+        catch {
+            logger_js_1.default.error(`Failed to load metrics from disk at ${filePath}`);
+            // Fallback to memory.
+            this.reset();
+        }
+    }
+}
+exports.SeriesRegistry = SeriesRegistry;
+// Utility: Create deterministic ID from attributes
+function attributesToId(attributes) {
+    return Object.keys(attributes).sort().map(key => `${key}=${attributes[key]}`).join(',') || 'no_attrs';
+}
+function scopeToId(scope) {
+    return `${scope.name}@${scope.version ?? 'no_version'}`;
+}
+function makeMetricId(scopeId, metricName) {
+    return `${scopeId}:${metricName}`;
+}
+function makeSeriesId(metricId, attributesId) {
+    return `${metricId}$${attributesId}`;
+}

package/dist/cjs/telemetry/custom-implementations/metrics/tsdb/types.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/cjs/telemetry/custom-implementations/metrics/tsdb/utils.cjs

@@ -0,0 +1,77 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rawToOtel = rawToOtel;
+/**
+ * Convert raw format to OpenTelemetry format
+ *
+ * Raw format structure (same for GET response and INSERT input):
+ * {
+ *   scope: { name, version },
+ *   descriptor: { unit, description, ... },
+ *   series: [{ attributes, startTimes[], endTimes[], values[] }]
+ * }
+ *
+ * OTEL format structure:
+ * {
+ *   scope: { name, version },
+ *   metrics: [{ descriptor, dataPoints: [{ attributes, startTime, endTime, value }] }]
+ * }
+ *
+ * Groups dataPoints by endTime to reconstruct original exports
+ */
+function rawToOtel(rawScopeMetrics) {
+    const groupedByScope = new Map();
+    for (const result of rawScopeMetrics) {
+        // Validate that result matches expected raw format
+        if (!result.scope) {
+            throw new Error(`Invalid metric format: Missing 'scope' property. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}. ` +
+                `Got: ${JSON.stringify(result).substring(0, 200)}`);
+        }
+        if (!result.descriptor) {
+            throw new Error(`Invalid metric format: Missing 'descriptor' property in metric. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}`);
+        }
+        if (!Array.isArray(result.series)) {
+            throw new Error(`Invalid metric format: 'series' must be an array. ` +
+                `Expected format: {scope: {name, version}, descriptor: {name, ...}, series: [{attributes, startTimes, endTimes, values}]}`);
+        }
+        const scopeId = result.scope.version
+            ? `${result.scope.name}@${result.scope.version}`
+            : `${result.scope.name}@none`;
+        if (!groupedByScope.has(scopeId)) {
+            groupedByScope.set(scopeId, {
+                scope: {
+                    name: result.scope.name,
+                    version: result.scope.version || ''
+                },
+                metrics: []
+            });
+        }
+        const scopeData = groupedByScope.get(scopeId);
+        const dataPoints = [];
+        for (const series of result.series) {
+            for (let i = 0; i < series.endTimes.length; i++) {
+                const startTimeNs = series.startTimes?.[i];
+                const endTimeNs = series.endTimes[i];
+                if (!startTimeNs)
+                    continue;
+                const startSec = Math.floor(startTimeNs / 1_000_000_000);
+                const startNano = startTimeNs % 1_000_000_000;
+                const endSec = Math.floor(endTimeNs / 1_000_000_000);
+                const endNano = endTimeNs % 1_000_000_000;
+                dataPoints.push({
+                    attributes: series.attributes,
+                    startTime: [startSec, startNano],
+                    endTime: [endSec, endNano],
+                    value: series.values[i]
+                });
+            }
+        }
+        scopeData.metrics.push({
+            descriptor: result.descriptor,
+            dataPoints
+        });
+    }
+    return Array.from(groupedByScope.values());
+}