@kopai/sqlite-datasource 0.3.0 → 0.5.0

package/dist/index.mjs

@@ -1,6 +1,6 @@
 import { DatabaseSync } from "node:sqlite";
-import { DummyDriver, Kysely, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler, sql } from "kysely";
 import { otlp } from "@kopai/core";
+import { DummyDriver, Kysely, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler, sql } from "kysely";
 
 //#region src/sqlite-datasource-error.ts
 var SqliteDatasourceError = class extends Error {
@@ -17,14 +17,14 @@ var SqliteDatasourceQueryError = class extends SqliteDatasourceError {
 };
 
 //#endregion
-//#region src/datasource.ts
+//#region src/db-datasource.ts
 const queryBuilder = new Kysely({ dialect: {
 	createAdapter: () => new SqliteAdapter(),
 	createDriver: () => new DummyDriver(),
 	createIntrospector: (db) => new SqliteIntrospector(db),
 	createQueryCompiler: () => new SqliteQueryCompiler()
 } });
-var NodeSqliteTelemetryDatasource = class {
+var DbDatasource = class {
 	constructor(sqliteConnection) {
 		this.sqliteConnection = sqliteConnection;
 	}
@@ -46,6 +46,9 @@ var NodeSqliteTelemetryDatasource = class {
 					if (metric.summary) for (const dataPoint of metric.summary.dataPoints ?? []) summaryRows.push(toSummaryRow(resource, resourceSchemaUrl, scope, scopeSchemaUrl, metric, dataPoint));
 				}
 			}
+		}
+		this.sqliteConnection.exec("BEGIN");
+		try {
 			for (const { table, rows } of [
 				{
 					table: "otel_metrics_gauge",
@@ -71,6 +74,10 @@ var NodeSqliteTelemetryDatasource = class {
 				const { sql, parameters } = queryBuilder.insertInto(table).values(row).compile();
 				this.sqliteConnection.prepare(sql).run(...parameters);
 			}
+			this.sqliteConnection.exec("COMMIT");
+		} catch (error) {
+			this.sqliteConnection.exec("ROLLBACK");
+			throw error;
 		}
 		return { rejectedDataPoints: "" };
 	}
@@ -99,20 +106,27 @@ var NodeSqliteTelemetryDatasource = class {
 				}
 			}
 		}
-		for (const row of spanRows) {
-			const { sql, parameters } = queryBuilder.insertInto("otel_traces").values(row).compile();
-			this.sqliteConnection.prepare(sql).run(...parameters);
-		}
-		for (const [traceId, { min, max }] of traceTimestamps) {
-			const { sql, parameters } = queryBuilder.insertInto("otel_traces_trace_id_ts").values({
-				TraceId: traceId,
-				Start: min,
-				End: max
-			}).onConflict((oc) => oc.column("TraceId").doUpdateSet({
-				Start: (eb) => eb.fn("min", [eb.ref("otel_traces_trace_id_ts.Start"), eb.val(min)]),
-				End: (eb) => eb.fn("max", [eb.ref("otel_traces_trace_id_ts.End"), eb.val(max)])
-			})).compile();
-			this.sqliteConnection.prepare(sql).run(...parameters);
+		this.sqliteConnection.exec("BEGIN");
+		try {
+			for (const row of spanRows) {
+				const { sql, parameters } = queryBuilder.insertInto("otel_traces").values(row).compile();
+				this.sqliteConnection.prepare(sql).run(...parameters);
+			}
+			for (const [traceId, { min, max }] of traceTimestamps) {
+				const { sql, parameters } = queryBuilder.insertInto("otel_traces_trace_id_ts").values({
+					TraceId: traceId,
+					Start: min,
+					End: max
+				}).onConflict((oc) => oc.column("TraceId").doUpdateSet({
+					Start: (eb) => eb.fn("min", [eb.ref("otel_traces_trace_id_ts.Start"), eb.val(min)]),
+					End: (eb) => eb.fn("max", [eb.ref("otel_traces_trace_id_ts.End"), eb.val(max)])
+				})).compile();
+				this.sqliteConnection.prepare(sql).run(...parameters);
+			}
+			this.sqliteConnection.exec("COMMIT");
+		} catch (error) {
+			this.sqliteConnection.exec("ROLLBACK");
+			throw error;
 		}
 		return { rejectedSpans: "" };
 	}
@@ -125,9 +139,16 @@ var NodeSqliteTelemetryDatasource = class {
 				for (const logRecord of scopeLog.logRecords ?? []) logRows.push(toLogRow(resource, resourceSchemaUrl, scope, scopeSchemaUrl, logRecord));
 			}
 		}
-		for (const row of logRows) {
-			const { sql, parameters } = queryBuilder.insertInto("otel_logs").values(row).compile();
-			this.sqliteConnection.prepare(sql).run(...parameters);
+		this.sqliteConnection.exec("BEGIN");
+		try {
+			for (const row of logRows) {
+				const { sql, parameters } = queryBuilder.insertInto("otel_logs").values(row).compile();
+				this.sqliteConnection.prepare(sql).run(...parameters);
+			}
+			this.sqliteConnection.exec("COMMIT");
+		} catch (error) {
+			this.sqliteConnection.exec("ROLLBACK");
+			throw error;
 		}
 		return { rejectedLogRecords: "" };
 	}
@@ -362,58 +383,72 @@ var NodeSqliteTelemetryDatasource = class {
 	}
 	async discoverMetrics() {
 		try {
-			const distinctMetricsSql = METRIC_TABLES.map(({ table, type }) => `SELECT DISTINCT MetricName, MetricUnit, MetricDescription, '${type}' as MetricType FROM ${table}`).join(" UNION ");
-			const distinctMetrics = this.sqliteConnection.prepare(distinctMetricsSql).all();
+			const discoveryState = /* @__PURE__ */ new Map();
+			const attrTuplesSql = METRIC_TABLES.map(({ table, type }) => `SELECT MetricName, MetricUnit, MetricDescription, '${type}' as MetricType, json_each.key as attr_key, json_each.value as attr_value
+           FROM ${table}, json_each(Attributes)`).join(" UNION ALL ");
+			const attrTuples = this.sqliteConnection.prepare(attrTuplesSql).all();
+			const resAttrTuplesSql = METRIC_TABLES.map(({ table, type }) => `SELECT MetricName, MetricUnit, MetricDescription, '${type}' as MetricType, json_each.key as attr_key, json_each.value as attr_value
+           FROM ${table}, json_each(ResourceAttributes)`).join(" UNION ALL ");
+			const resAttrTuples = this.sqliteConnection.prepare(resAttrTuplesSql).all();
+			for (const tuple of attrTuples) {
+				const metricKey = `${tuple.MetricName}:${tuple.MetricType}`;
+				let state = discoveryState.get(metricKey);
+				if (!state) {
+					state = {
+						name: tuple.MetricName,
+						type: tuple.MetricType,
+						unit: tuple.MetricUnit || void 0,
+						description: tuple.MetricDescription || void 0,
+						attributes: /* @__PURE__ */ new Map(),
+						resourceAttributes: /* @__PURE__ */ new Map()
+					};
+					discoveryState.set(metricKey, state);
+				}
+				if (!state.attributes.has(tuple.attr_key)) state.attributes.set(tuple.attr_key, /* @__PURE__ */ new Set());
+				state.attributes.get(tuple.attr_key).add(String(tuple.attr_value));
+			}
+			for (const tuple of resAttrTuples) {
+				const metricKey = `${tuple.MetricName}:${tuple.MetricType}`;
+				let state = discoveryState.get(metricKey);
+				if (!state) {
+					state = {
+						name: tuple.MetricName,
+						type: tuple.MetricType,
+						unit: tuple.MetricUnit || void 0,
+						description: tuple.MetricDescription || void 0,
+						attributes: /* @__PURE__ */ new Map(),
+						resourceAttributes: /* @__PURE__ */ new Map()
+					};
+					discoveryState.set(metricKey, state);
+				}
+				if (!state.resourceAttributes.has(tuple.attr_key)) state.resourceAttributes.set(tuple.attr_key, /* @__PURE__ */ new Set());
+				state.resourceAttributes.get(tuple.attr_key).add(String(tuple.attr_value));
+			}
 			const metrics = [];
-			for (const metric of distinctMetrics) {
-				const table = METRIC_TABLES.find((t) => t.type === metric.MetricType).table;
-				const attrKeysSql = `
-          SELECT DISTINCT json_each.key as key
-          FROM ${table}, json_each(Attributes)
-          WHERE MetricName = ?
-        `;
-				const attrKeys = this.sqliteConnection.prepare(attrKeysSql).all(metric.MetricName);
+			for (const state of discoveryState.values()) {
 				let attrsTruncated = false;
 				const attributes = {};
-				for (const { key } of attrKeys) {
-					const valuesSql = `
-            SELECT DISTINCT json_each.value as value
-            FROM ${table}, json_each(Attributes)
-            WHERE MetricName = ? AND json_each.key = ?
-            LIMIT ${MAX_ATTR_VALUES + 1}
-          `;
-					const values = this.sqliteConnection.prepare(valuesSql).all(metric.MetricName, key);
-					if (values.length > MAX_ATTR_VALUES) {
+				for (const [key, valueSet] of state.attributes) {
+					const values = Array.from(valueSet);
+					if (values.length > MAX_ATTR_VALUES$1) {
 						attrsTruncated = true;
-						attributes[key] = values.slice(0, MAX_ATTR_VALUES).map((v) => String(v.value));
-					} else attributes[key] = values.map((v) => String(v.value));
+						attributes[key] = values.slice(0, MAX_ATTR_VALUES$1);
+					} else attributes[key] = values;
 				}
-				const resAttrKeysSql = `
-          SELECT DISTINCT json_each.key as key
-          FROM ${table}, json_each(ResourceAttributes)
-          WHERE MetricName = ?
-        `;
-				const resAttrKeys = this.sqliteConnection.prepare(resAttrKeysSql).all(metric.MetricName);
 				let resAttrsTruncated = false;
 				const resourceAttributes = {};
-				for (const { key } of resAttrKeys) {
-					const valuesSql = `
-            SELECT DISTINCT json_each.value as value
-            FROM ${table}, json_each(ResourceAttributes)
-            WHERE MetricName = ? AND json_each.key = ?
-            LIMIT ${MAX_ATTR_VALUES + 1}
-          `;
-					const values = this.sqliteConnection.prepare(valuesSql).all(metric.MetricName, key);
-					if (values.length > MAX_ATTR_VALUES) {
+				for (const [key, valueSet] of state.resourceAttributes) {
+					const values = Array.from(valueSet);
+					if (values.length > MAX_ATTR_VALUES$1) {
 						resAttrsTruncated = true;
-						resourceAttributes[key] = values.slice(0, MAX_ATTR_VALUES).map((v) => String(v.value));
-					} else resourceAttributes[key] = values.map((v) => String(v.value));
+						resourceAttributes[key] = values.slice(0, MAX_ATTR_VALUES$1);
+					} else resourceAttributes[key] = values;
 				}
 				metrics.push({
-					name: metric.MetricName,
-					type: metric.MetricType,
-					unit: metric.MetricUnit || void 0,
-					description: metric.MetricDescription || void 0,
+					name: state.name,
+					type: state.type,
+					unit: state.unit,
+					description: state.description,
 					attributes: {
 						values: attributes,
 						...attrsTruncated && { _truncated: true }
@@ -511,7 +546,7 @@ function toGaugeRow(resource, resourceSchemaUrl, scope, scopeSchemaUrl, metric,
 		"Exemplars.TimeUnix": exemplarsArrayToJson(exemplars, (e) => String(nanosToSqlite(e.timeUnixNano))),
 		"Exemplars.Value": exemplarsArrayToJson(exemplars, (e) => e.asDouble ?? Number(e.asInt ?? 0)),
 		"Exemplars.SpanId": exemplarsArrayToJson(exemplars, (e) => e.spanId ?? ""),
-		"Exemplars.TraceId": exemplarsArrayToJson(exemplars, (e) => e.traceId ? bufferToHex(e.traceId) : "")
+		"Exemplars.TraceId": exemplarsArrayToJson(exemplars, (e) => e.traceId ?? "")
 	};
 }
 function toSumRow(resource, resourceSchemaUrl, scope, scopeSchemaUrl, metric, dataPoint, aggregationTemporality, isMonotonic) {
@@ -537,7 +572,7 @@ function toSumRow(resource, resourceSchemaUrl, scope, scopeSchemaUrl, metric, da
 		"Exemplars.TimeUnix": exemplarsArrayToJson(exemplars, (e) => String(nanosToSqlite(e.timeUnixNano))),
 		"Exemplars.Value": exemplarsArrayToJson(exemplars, (e) => e.asDouble ?? Number(e.asInt ?? 0)),
 		"Exemplars.SpanId": exemplarsArrayToJson(exemplars, (e) => e.spanId ?? ""),
-		"Exemplars.TraceId": exemplarsArrayToJson(exemplars, (e) => e.traceId ? bufferToHex(e.traceId) : ""),
+		"Exemplars.TraceId": exemplarsArrayToJson(exemplars, (e) => e.traceId ?? ""),
 		AggTemporality: aggTemporalityToString(aggregationTemporality),
 		IsMonotonic: isMonotonic ? 1 : 0
 	};
@@ -569,7 +604,7 @@ function toHistogramRow(resource, resourceSchemaUrl, scope, scopeSchemaUrl, metr
 		"Exemplars.TimeUnix": exemplarsArrayToJson(exemplars, (e) => String(nanosToSqlite(e.timeUnixNano))),
 		"Exemplars.Value": exemplarsArrayToJson(exemplars, (e) => e.asDouble ?? Number(e.asInt ?? 0)),
 		"Exemplars.SpanId": exemplarsArrayToJson(exemplars, (e) => e.spanId ?? ""),
-		"Exemplars.TraceId": exemplarsArrayToJson(exemplars, (e) => e.traceId ? bufferToHex(e.traceId) : ""),
+		"Exemplars.TraceId": exemplarsArrayToJson(exemplars, (e) => e.traceId ?? ""),
 		AggTemporality: aggTemporalityToString(aggregationTemporality)
 	};
 }
@@ -605,7 +640,7 @@ function toExpHistogramRow(resource, resourceSchemaUrl, scope, scopeSchemaUrl, m
 		"Exemplars.TimeUnix": exemplarsArrayToJson(exemplars, (e) => String(nanosToSqlite(e.timeUnixNano))),
 		"Exemplars.Value": exemplarsArrayToJson(exemplars, (e) => e.asDouble ?? Number(e.asInt ?? 0)),
 		"Exemplars.SpanId": exemplarsArrayToJson(exemplars, (e) => e.spanId ?? ""),
-		"Exemplars.TraceId": exemplarsArrayToJson(exemplars, (e) => e.traceId ? bufferToHex(e.traceId) : ""),
+		"Exemplars.TraceId": exemplarsArrayToJson(exemplars, (e) => e.traceId ?? ""),
 		AggTemporality: aggTemporalityToString(aggregationTemporality)
 	};
 }
@@ -678,9 +713,6 @@ function exemplarsArrayToJson(exemplars, extractor) {
 	if (exemplars.length === 0) return "[]";
 	return JSON.stringify(exemplars.map(extractor));
 }
-function bufferToHex(buf) {
-	return Array.from(buf).map((b) => b.toString(16).padStart(2, "0")).join("");
-}
 function mapRowToOtelTraces(row) {
 	return {
 		TraceId: row.TraceId,
@@ -785,7 +817,7 @@ const METRIC_TABLES = [
 		type: "Summary"
 	}
 ];
-const MAX_ATTR_VALUES = 100;
+const MAX_ATTR_VALUES$1 = 100;
 function mapRowToOtelMetrics(row, metricType) {
 	const base = {
 		TimeUnix: String(row.TimeUnix),
@@ -858,6 +890,154 @@ function mapRowToOtelMetrics(row, metricType) {
 	};
 }
 
+//#endregion
+//#region src/optimized-datasource.ts
+const MAX_ATTR_VALUES = 100;
+var OptimizedDatasource = class {
+	discoveryState = /* @__PURE__ */ new Map();
+	constructor(dbDatasource) {
+		this.dbDatasource = dbDatasource;
+	}
+	async writeMetrics(metricsData) {
+		const result = await this.dbDatasource.writeMetrics(metricsData);
+		for (const resourceMetric of metricsData.resourceMetrics ?? []) {
+			const { resource } = resourceMetric;
+			const resourceAttrs = this.extractResourceAttributes(resource);
+			for (const scopeMetric of resourceMetric.scopeMetrics ?? []) for (const metric of scopeMetric.metrics ?? []) {
+				const metricType = this.getMetricType(metric);
+				if (!metricType) continue;
+				const metricKey = `${metric.name}:${metricType}`;
+				let state = this.discoveryState.get(metricKey);
+				if (!state) {
+					state = {
+						name: metric.name ?? "",
+						type: metricType,
+						unit: metric.unit || void 0,
+						description: metric.description || void 0,
+						attributes: /* @__PURE__ */ new Map(),
+						resourceAttributes: /* @__PURE__ */ new Map()
+					};
+					this.discoveryState.set(metricKey, state);
+				}
+				for (const [key, value] of Object.entries(resourceAttrs)) {
+					if (!state.resourceAttributes.has(key)) state.resourceAttributes.set(key, /* @__PURE__ */ new Set());
+					state.resourceAttributes.get(key).add(String(value));
+				}
+				const dataPoints = this.getDataPoints(metric);
+				for (const dp of dataPoints) {
+					const attrs = this.extractAttributes(dp.attributes);
+					for (const [key, value] of Object.entries(attrs)) {
+						if (!state.attributes.has(key)) state.attributes.set(key, /* @__PURE__ */ new Set());
+						state.attributes.get(key).add(String(value));
+					}
+				}
+			}
+		}
+		return result;
+	}
+	async writeTraces(tracesData) {
+		return this.dbDatasource.writeTraces(tracesData);
+	}
+	async writeLogs(logsData) {
+		return this.dbDatasource.writeLogs(logsData);
+	}
+	async getTraces(filter) {
+		return this.dbDatasource.getTraces(filter);
+	}
+	async getMetrics(filter) {
+		return this.dbDatasource.getMetrics(filter);
+	}
+	async getLogs(filter) {
+		return this.dbDatasource.getLogs(filter);
+	}
+	async discoverMetrics() {
+		const metrics = [];
+		for (const state of this.discoveryState.values()) {
+			let attrsTruncated = false;
+			const attributes = {};
+			for (const [key, valueSet] of state.attributes) {
+				const values = Array.from(valueSet);
+				if (values.length > MAX_ATTR_VALUES) {
+					attrsTruncated = true;
+					attributes[key] = values.slice(0, MAX_ATTR_VALUES);
+				} else attributes[key] = values;
+			}
+			let resAttrsTruncated = false;
+			const resourceAttributes = {};
+			for (const [key, valueSet] of state.resourceAttributes) {
+				const values = Array.from(valueSet);
+				if (values.length > MAX_ATTR_VALUES) {
+					resAttrsTruncated = true;
+					resourceAttributes[key] = values.slice(0, MAX_ATTR_VALUES);
+				} else resourceAttributes[key] = values;
+			}
+			metrics.push({
+				name: state.name,
+				type: state.type,
+				unit: state.unit,
+				description: state.description,
+				attributes: {
+					values: attributes,
+					...attrsTruncated && { _truncated: true }
+				},
+				resourceAttributes: {
+					values: resourceAttributes,
+					...resAttrsTruncated && { _truncated: true }
+				}
+			});
+		}
+		return { metrics };
+	}
+	getMetricType(metric) {
+		if (metric.gauge) return "Gauge";
+		if (metric.sum) return "Sum";
+		if (metric.histogram) return "Histogram";
+		if (metric.exponentialHistogram) return "ExponentialHistogram";
+		if (metric.summary) return "Summary";
+		return null;
+	}
+	getDataPoints(metric) {
+		if (metric.gauge) return metric.gauge.dataPoints ?? [];
+		if (metric.sum) return metric.sum.dataPoints ?? [];
+		if (metric.histogram) return metric.histogram.dataPoints ?? [];
+		if (metric.exponentialHistogram) return metric.exponentialHistogram.dataPoints ?? [];
+		if (metric.summary) return metric.summary.dataPoints ?? [];
+		return [];
+	}
+	extractResourceAttributes(resource) {
+		if (!resource?.attributes) return {};
+		const result = {};
+		for (const attr of resource.attributes) if (attr.key) result[attr.key] = this.extractAnyValue(attr.value);
+		return result;
+	}
+	extractAttributes(attributes) {
+		if (!attributes) return {};
+		const result = {};
+		for (const attr of attributes) if (attr.key) result[attr.key] = this.extractAnyValue(attr.value);
+		return result;
+	}
+	extractAnyValue(value) {
+		if (!value || typeof value !== "object") return value;
+		const v = value;
+		if (v.stringValue !== void 0) return v.stringValue;
+		if (v.boolValue !== void 0) return v.boolValue;
+		if (v.intValue !== void 0) return v.intValue;
+		if (v.doubleValue !== void 0) return v.doubleValue;
+		if (v.bytesValue !== void 0) return v.bytesValue;
+		if (v.arrayValue && typeof v.arrayValue === "object") return (v.arrayValue.values ?? []).map((item) => this.extractAnyValue(item));
+		if (v.kvlistValue && typeof v.kvlistValue === "object") {
+			const kvlist = v.kvlistValue;
+			const result = {};
+			for (const kv of kvlist.values ?? []) if (kv.key) result[kv.key] = this.extractAnyValue(kv.value);
+			return result;
+		}
+		return value;
+	}
+};
+function createOptimizedDatasource(sqliteConnection) {
+	return new OptimizedDatasource(new DbDatasource(sqliteConnection));
+}
+
 //#endregion
 //#region src/sqlite-opentelemetry-ddl.ts
 const ddl = `
@@ -1122,5 +1302,5 @@ function initializeDatabase(path, opts) {
 }
 
 //#endregion
-export { NodeSqliteTelemetryDatasource, initializeDatabase };
+export { DbDatasource, OptimizedDatasource, createOptimizedDatasource, initializeDatabase };
 //# sourceMappingURL=index.mjs.map