@wizeworks/lens-cloud 0.3.0-alpha.1 → 0.3.0-alpha.3

package/dist/telemetry/ch/metrics.js

@@ -0,0 +1,61 @@
+// Metric rollups on ClickHouse: the same series state and tier rows as Postgres, merged by the tables instead of upserts.
+import { createHash } from "node:crypto";
+import { groupMetricSeries, mergeStoredHists, metricBatchWrites, metricHistOfColumns, METRIC_SERIES_COLUMNS, METRIC_TIER_COLUMNS, metricSeriesValues, metricStateOfColumns, metricTierValues, seriesKeyOf, TIERS, } from "@wizeworks/lens-core";
+import { nextVer, rowOf } from "./util.js";
+const sha1 = (text) => createHash("sha1").update(text).digest("hex");
+const cols = (list) => list.map((c) => (c === "project" ? "project_id" : c));
+const TIER_INPUT = [
+    "project_id String, env String, series_key String, name String, bucket_ms Float64, cnt Float64, vsum Float64, vmin Nullable(Float64)",
+    "vmax Nullable(Float64), first_ms Float64, first_v Nullable(Float64), last_ms Float64, last_v Nullable(Float64), rate_delta Float64, rate_ms Float64",
+    "h_count Float64, h_sum Float64, h_min Nullable(Float64), h_max Nullable(Float64), h_bounds Nullable(String), h_counts Nullable(String), ver UInt64, expire_ms Float64",
+].join(", ");
+/** Sums add, extremes keep extremes, first and last follow time, and histogram counts follow the newest write that had some. */
+const tierInsert = (table) => `${table}_t SELECT project_id, env, series_key, name, bucket_ms, sum(cnt), sum(vsum), min(vmin), max(vmax),
+  min(first_ms), argMinState(first_v, first_ms), max(last_ms), argMaxState(last_v, last_ms), sum(rate_delta), sum(rate_ms), sum(h_count), sum(h_sum),
+  min(h_min), max(h_max), argMaxState(h_bounds, ver), argMaxState(h_counts, ver), max(expire_ms) FROM input('${TIER_INPUT}')
+  GROUP BY project_id, env, series_key, name, bucket_ms`;
+/** Stored histograms of the buckets this batch adds counts to. */
+async function storedHists(ch, project, env, tiers) {
+    const stored = new Map();
+    for (const tier of TIERS) {
+        const mine = tiers.filter((t) => t.tier === tier && t.row.hist);
+        if (!mine.length)
+            continue;
+        const sql = `SELECT series_key, bucket_ms, h_count, h_sum, h_min, h_max, h_bounds, h_counts FROM metric_${tier.name}
+      WHERE project_id = {p1:String} AND env = {p2:String} AND has({p3:Array(String)}, series_key) AND has({p4:Array(Float64)}, bucket_ms) AND h_count > 0`;
+        const rows = await ch.query(sql, [project, env, [...new Set(mine.map((t) => t.key))], [...new Set(mine.map((t) => t.bucket))]]);
+        for (const r of rows)
+            stored.set(`${tier.name} ${r.series_key} ${Number(r.bucket_ms)}`, metricHistOfColumns(r));
+    }
+    return (t) => stored.get(`${t.tier.name} ${t.key} ${t.bucket}`) ?? null;
+}
+async function writeOnce(ch, project, env, points, now, retentionMs) {
+    const groups = groupMetricSeries(points, (p) => sha1(seriesKeyOf(p)));
+    const states = await ch.query("SELECT series_key, last_ms, last_value, last_hist FROM metric_series WHERE project_id = {p1:String} AND env = {p2:String} AND has({p3:Array(String)}, series_key)", [project, env, [...groups.keys()]]);
+    const { series, tiers } = metricBatchWrites(groups, new Map(states.map((r) => [String(r.series_key), metricStateOfColumns(r)])), now);
+    mergeStoredHists(tiers, await storedHists(ch, project, env, tiers));
+    const ver = nextVer(now);
+    const tierCols = cols(METRIC_TIER_COLUMNS);
+    for (const tier of TIERS) {
+        const rows = tiers.filter((t) => t.tier === tier).map((t) => ({ ...rowOf(tierCols, metricTierValues("ch", project, env, t)), ver, expire_ms: t.bucket + retentionMs }));
+        await ch.insert(tierInsert(`metric_${tier.name}`), rows);
+    }
+    const seriesRows = series.map((s) => ({ ...rowOf(cols(METRIC_SERIES_COLUMNS), metricSeriesValues("ch", project, env, s)), ver, expire_ms: s.state.lastMs + retentionMs }));
+    await ch.insert("metric_series_t", seriesRows);
+}
+/** Metric writes one at a time: each reads the series state it then advances. */
+export class MetricWriter {
+    ch;
+    chain = Promise.resolve();
+    constructor(ch) {
+        this.ch = ch;
+    }
+    write(project, env, points, now, retentionMs) {
+        if (!points.length)
+            return Promise.resolve();
+        const run = this.chain.then(() => writeOnce(this.ch, project, env, points, now, retentionMs));
+        this.chain = run.catch(() => undefined);
+        return run;
+    }
+}
+//# sourceMappingURL=metrics.js.map

package/dist/telemetry/ch/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/telemetry/ch/metrics.ts"],"names":[],"mappings":"AAAA,0HAA0H;AAC1H,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,iBAAiB,EAAwB,gBAAgB,EAAE,iBAAiB,EAAE,mBAAmB,EAAoB,qBAAqB,EAC1I,mBAAmB,EAAE,kBAAkB,EAAE,oBAAoB,EAAwB,gBAAgB,EAAE,WAAW,EAAE,KAAK,GAC1H,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAE3C,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC7E,MAAM,IAAI,GAAG,CAAC,IAAuB,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAEhG,MAAM,UAAU,GAAG;IACjB,qIAAqI;IACrI,qJAAqJ;IACrJ,uKAAuK;CACxK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,gIAAgI;AAChI,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,GAAG,KAAK;;+GAEiE,UAAU;wDACjE,CAAC;AAEzD,kEAAkE;AAClE,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,OAAe,EAAE,GAAW,EAAE,KAAiC;IACtG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkC,CAAC;IACzD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAS;QAC3B,MAAM,GAAG,GAAG,8FAA8F,IAAI,CAAC,IAAI;2JACoC,CAAC;QACxJ,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChI,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,UAAU,IAAI,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;IAClH,CAAC;IACD,OAAO,CAAC,CAAkB,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC;AAC3F,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,EAAY,EAAE,OAAe,EAAE,GAAW,EAAE,MAA8B,EAAE,GAAW,EAAE,WAAmB;IACnI,MAAM,MAAM,GAAG,iBAAiB,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,KAAK,CAC3B,mKAAmK,EACnK,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CACnC,CAAC;IACF,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,iBAAiB,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtI,gBAAgB,CAAC,KAAK,EAAE,MAAM,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;IACpE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,gBAAgB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC;QACxK,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,UAAU,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC;IAC3K,MAAM,EAAE,CAAC,MAAM,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;AACjD,CAAC;AAED,iFAAiF;AACjF,MAAM,OAAO,YAAY;IAGM,EAAE;IAFvB,KAAK,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEjD,YAA6B,EAAY;kBAAZ,EAAE;IAAa,CAAC;IAE7C,KAAK,CAAC,OAAe,EAAE,GAAW,EAAE,MAA8B,EAAE,GAAW,EAAE,WAAmB;QAClG,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC7C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC9F,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACxC,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}

package/dist/telemetry/ch/queries.d.ts

@@ -0,0 +1,19 @@
+import type { TimeWindow } from "@wizeworks/lens-core";
+import type { ChClient } from "./http.js";
+/** Traces and spans of a project. */
+export declare function counts(ch: ChClient, projectId: string): Promise<{
+    traces: number;
+    spans: number;
+}>;
+export declare function traceEnvs(ch: ChClient, projectId: string): Promise<string[]>;
+/** Distinct hours with a trace start in (from, to]. */
+export declare function activeHours(ch: ChClient, projectId: string, from: number, to: number): Promise<number>;
+/** The newest trace start per project, over the last month (older projects count as quiet). */
+export declare function lastTraceMs(ch: ChClient, now: number): Promise<Map<string, number>>;
+/** The slowest endpoints by p95 (interpolated, like Postgres percentile_cont) with at least `minCount` requests. */
+export declare function slowestEndpoints(ch: ChClient, projectId: string, w: TimeWindow, minCount: number, limit: number): Promise<{
+    endpoint: string;
+    p95Ms: number;
+    count: number;
+}[]>;
+//# sourceMappingURL=queries.d.ts.map

package/dist/telemetry/ch/queries.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queries.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/queries.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AACvD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,qCAAqC;AACrC,wBAAsB,MAAM,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAIxG;AAED,wBAAsB,SAAS,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAElF;AAED,uDAAuD;AACvD,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAO5G;AAED,+FAA+F;AAC/F,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAOzF;AAED,oHAAoH;AACpH,wBAAsB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM;;;;KAQrH"}

package/dist/telemetry/ch/queries.js

@@ -0,0 +1,30 @@
+const MONTH_MS = 30 * 86_400_000;
+/** Traces and spans of a project. */
+export async function counts(ch, projectId) {
+    const [t] = await ch.query("SELECT uniqExact(trace_id) AS n FROM trace_heads WHERE project_id = {p1:String}", [projectId]);
+    const [s] = await ch.query("SELECT count() AS n FROM spans_raw FINAL WHERE project_id = {p1:String}", [projectId]);
+    return { traces: Number(t?.n ?? 0), spans: Number(s?.n ?? 0) };
+}
+export async function traceEnvs(ch, projectId) {
+    return (await ch.query("SELECT DISTINCT env FROM trace_heads WHERE project_id = {p1:String}", [projectId])).map((r) => r.env);
+}
+/** Distinct hours with a trace start in (from, to]. */
+export async function activeHours(ch, projectId, from, to) {
+    const [row] = await ch.query(`SELECT uniqExact(intDiv(toInt64(s), 3600000)) AS n FROM (SELECT min(start_ms) AS s FROM trace_heads WHERE project_id = {p1:String}
+     AND start_ms <= {p3:Float64} GROUP BY trace_id) WHERE s > {p2:Float64} AND s <= {p3:Float64}`, [projectId, from, to]);
+    return Number(row?.n ?? 0);
+}
+/** The newest trace start per project, over the last month (older projects count as quiet). */
+export async function lastTraceMs(ch, now) {
+    const rows = await ch.query(`SELECT project_id, max(s) AS s FROM (SELECT project_id, trace_id, min(start_ms) AS s FROM trace_heads WHERE start_ms > {p1:Float64}
+     GROUP BY project_id, trace_id) GROUP BY project_id`, [now - MONTH_MS]);
+    return new Map(rows.map((r) => [r.project_id, Number(r.s)]));
+}
+/** The slowest endpoints by p95 (interpolated, like Postgres percentile_cont) with at least `minCount` requests. */
+export async function slowestEndpoints(ch, projectId, w, minCount, limit) {
+    const rows = await ch.query(`SELECT endpoint, quantileExactInclusive(0.95)(duration_ms) AS p95, count() AS n FROM fact_requests
+     WHERE project_id = {p1:String} AND start_ms >= {p2:Float64} AND start_ms < {p3:Float64} GROUP BY endpoint HAVING n >= {p4:Float64}
+     ORDER BY p95 DESC LIMIT ${Math.max(0, Math.floor(limit))}`, [projectId, w.from, w.to, minCount]);
+    return rows.map((r) => ({ endpoint: r.endpoint, p95Ms: Number(r.p95), count: Number(r.n) }));
+}
+//# sourceMappingURL=queries.js.map

package/dist/telemetry/ch/queries.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queries.js","sourceRoot":"","sources":["../../../src/telemetry/ch/queries.ts"],"names":[],"mappings":"AAIA,MAAM,QAAQ,GAAG,EAAE,GAAG,UAAU,CAAC;AAEjC,qCAAqC;AACrC,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,EAAY,EAAE,SAAiB;IAC1D,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAgB,iFAAiF,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IAC1I,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAgB,yEAAyE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;IAClI,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;AACjE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAAY,EAAE,SAAiB;IAC7D,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,CAAkB,qEAAqE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AACjJ,CAAC;AAED,uDAAuD;AACvD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,SAAiB,EAAE,IAAY,EAAE,EAAU;IACzF,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAC1B;kGAC8F,EAC9F,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC,CACtB,CAAC;IACF,OAAO,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;AAC7B,CAAC;AAED,+FAA+F;AAC/F,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,GAAW;IACzD,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CACzB;wDACoD,EACpD,CAAC,GAAG,GAAG,QAAQ,CAAC,CACjB,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,oHAAoH;AACpH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAY,EAAE,SAAiB,EAAE,CAAa,EAAE,QAAgB,EAAE,KAAa;IACpH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CACzB;;+BAE2B,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAC3D,CAAC,SAAS,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,CACpC,CAAC;IACF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/F,CAAC"}

package/dist/telemetry/ch/queue.d.ts

@@ -0,0 +1,34 @@
+import type { ChClient } from "./http.js";
+export interface Dirty {
+    project: string;
+    traceId: string;
+    env: string;
+    /** The newest write. */
+    updatedMs: number;
+    /** When it last settled; 0 when never. */
+    settledMs: number;
+}
+export declare class SettleQueue {
+    private readonly items;
+    /** Keys a settle batch holds, so an overlapping batch skips them. */
+    private readonly busy;
+    /** A trace got new parts. */
+    touch(project: string, traceId: string, env: string, now: number): void;
+    /** Traces quiet for a while, or busy ones not settled for long: newest first. */
+    due(now: number, limit: number): Dirty[];
+    /** Due traces for one batch, held until settled or released. Copies, so a write that lands meanwhile keeps its trace dirty. */
+    take(now: number, limit: number): Dirty[];
+    /** A failed batch: its traces wait for the next run. */
+    release(items: readonly Dirty[]): void;
+    /** Settled as of `updatedMs`: done unless something newer came in since. */
+    settled(d: Dirty, updatedMs: number): void;
+    /** How many traces of a project (optionally one environment) still wait. */
+    count(project: string, env?: string): number;
+    clear(project?: string): void;
+    get size(): number;
+    /** On start: traces written lately that never settled (or changed after), from the update log. */
+    recover(ch: ChClient, now: number): Promise<number>;
+    /** Every trace (after the facts version changed): its stats are rebuilt from scratch. */
+    requeueAll(ch: ChClient): Promise<number>;
+}
+//# sourceMappingURL=queue.d.ts.map

package/dist/telemetry/ch/queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/queue.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,MAAM,WAAW,KAAK;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,wBAAwB;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;CACnB;AAMD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAA4B;IAClD,qEAAqE;IACrE,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAqB;IAE1C,6BAA6B;IAC7B,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,IAAI,CAKtE;IAED,iFAAiF;IACjF,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK,EAAE,CAGvC;IAED,+HAA+H;IAC/H,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK,EAAE,CAIxC;IAED,wDAAwD;IACxD,OAAO,CAAC,KAAK,EAAE,SAAS,KAAK,EAAE,GAAG,IAAI,CAErC;IAED,4EAA4E;IAC5E,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAOzC;IAED,4EAA4E;IAC5E,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,CAI3C;IAED,KAAK,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAE5B;IAED,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,kGAAkG;IAC5F,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CASxD;IAED,yFAAyF;IACnF,UAAU,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAM9C;CACF"}

package/dist/telemetry/ch/queue.js

@@ -0,0 +1,79 @@
+const QUIET_MS = 5_000;
+const BUSY_MS = 30_000;
+const RECOVER_MS = 6 * 3_600_000;
+export class SettleQueue {
+    items = new Map();
+    /** Keys a settle batch holds, so an overlapping batch skips them. */
+    busy = new Set();
+    /** A trace got new parts. */
+    touch(project, traceId, env, now) {
+        const k = `${project} ${traceId}`;
+        const it = this.items.get(k);
+        if (it)
+            (it.updatedMs = now), (it.env ||= env);
+        else
+            this.items.set(k, { project, traceId, env, updatedMs: now, settledMs: 0 });
+    }
+    /** Traces quiet for a while, or busy ones not settled for long: newest first. */
+    due(now, limit) {
+        const ready = [...this.items.entries()].filter(([k, d]) => !this.busy.has(k) && d.settledMs < d.updatedMs && (d.updatedMs <= now - QUIET_MS || d.settledMs <= now - BUSY_MS));
+        return ready.map(([, d]) => d).sort((a, b) => b.updatedMs - a.updatedMs).slice(0, limit);
+    }
+    /** Due traces for one batch, held until settled or released. Copies, so a write that lands meanwhile keeps its trace dirty. */
+    take(now, limit) {
+        const out = this.due(now, limit).map((d) => ({ ...d }));
+        for (const d of out)
+            this.busy.add(`${d.project} ${d.traceId}`);
+        return out;
+    }
+    /** A failed batch: its traces wait for the next run. */
+    release(items) {
+        for (const d of items)
+            this.busy.delete(`${d.project} ${d.traceId}`);
+    }
+    /** Settled as of `updatedMs`: done unless something newer came in since. */
+    settled(d, updatedMs) {
+        const k = `${d.project} ${d.traceId}`;
+        this.busy.delete(k);
+        const it = this.items.get(k);
+        if (!it)
+            return;
+        if (it.updatedMs <= updatedMs)
+            this.items.delete(k);
+        else
+            it.settledMs = updatedMs;
+    }
+    /** How many traces of a project (optionally one environment) still wait. */
+    count(project, env) {
+        let n = 0;
+        for (const d of this.items.values())
+            if (d.project === project && (!env || d.env === env))
+                n++;
+        return n;
+    }
+    clear(project) {
+        for (const [k, d] of this.items)
+            if (!project || d.project === project)
+                (this.items.delete(k), this.busy.delete(k));
+    }
+    get size() {
+        return this.items.size;
+    }
+    /** On start: traces written lately that never settled (or changed after), from the update log. */
+    async recover(ch, now) {
+        const rows = await ch.query(`SELECT u.project_id, u.trace_id, any(u.env) AS env, max(u.updated_ms) AS u, any(s.settled_ms) AS s FROM trace_updates u
+       LEFT JOIN (SELECT project_id, trace_id, settled_ms FROM settle_state FINAL) s ON s.project_id = u.project_id AND s.trace_id = u.trace_id
+       WHERE u.updated_ms > {p1:Float64} GROUP BY u.project_id, u.trace_id HAVING u > s`, [now - RECOVER_MS]);
+        for (const r of rows)
+            this.items.set(`${r.project_id} ${r.trace_id}`, { project: r.project_id, traceId: r.trace_id, env: r.env, updatedMs: r.u, settledMs: r.s });
+        return rows.length;
+    }
+    /** Every trace (after the facts version changed): its stats are rebuilt from scratch. */
+    async requeueAll(ch) {
+        const rows = await ch.query("SELECT project_id, trace_id, max(env) AS env, max(updated_ms) AS u FROM trace_heads GROUP BY project_id, trace_id");
+        for (const r of rows)
+            this.items.set(`${r.project_id} ${r.trace_id}`, { project: r.project_id, traceId: r.trace_id, env: r.env, updatedMs: r.u, settledMs: 0 });
+        return rows.length;
+    }
+}
+//# sourceMappingURL=queue.js.map

package/dist/telemetry/ch/queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../../src/telemetry/ch/queue.ts"],"names":[],"mappings":"AAaA,MAAM,QAAQ,GAAG,KAAK,CAAC;AACvB,MAAM,OAAO,GAAG,MAAM,CAAC;AACvB,MAAM,UAAU,GAAG,CAAC,GAAG,SAAS,CAAC;AAEjC,MAAM,OAAO,WAAW;IACL,KAAK,GAAG,IAAI,GAAG,EAAiB,CAAC;IAClD,qEAAqE;IACpD,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,6BAA6B;IAC7B,KAAK,CAAC,OAAe,EAAE,OAAe,EAAE,GAAW,EAAE,GAAW;QAC9D,MAAM,CAAC,GAAG,GAAG,OAAO,IAAI,OAAO,EAAE,CAAC;QAClC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,EAAE;YAAE,CAAC,EAAE,CAAC,SAAS,GAAG,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;;YAC1C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC;IAED,iFAAiF;IACjF,GAAG,CAAC,GAAW,EAAE,KAAa;QAC5B,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,GAAG,GAAG,QAAQ,IAAI,CAAC,CAAC,SAAS,IAAI,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC;QAC9K,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAC3F,CAAC;IAED,+HAA+H;IAC/H,IAAI,CAAC,GAAW,EAAE,KAAa;QAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACxD,KAAK,MAAM,CAAC,IAAI,GAAG;YAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAChE,OAAO,GAAG,CAAC;IACb,CAAC;IAED,wDAAwD;IACxD,OAAO,CAAC,KAAuB;QAC7B,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,4EAA4E;IAC5E,OAAO,CAAC,CAAQ,EAAE,SAAiB;QACjC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACpB,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,EAAE;YAAE,OAAO;QAChB,IAAI,EAAE,CAAC,SAAS,IAAI,SAAS;YAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;;YAC/C,EAAE,CAAC,SAAS,GAAG,SAAS,CAAC;IAChC,CAAC;IAED,4EAA4E;IAC5E,KAAK,CAAC,OAAe,EAAE,GAAY;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,CAAC,OAAO,KAAK,OAAO,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC;gBAAE,CAAC,EAAE,CAAC;QAC/F,OAAO,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,OAAgB;QACpB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK;YAAE,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC,OAAO,KAAK,OAAO;gBAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACtH,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,kGAAkG;IAClG,KAAK,CAAC,OAAO,CAAC,EAAY,EAAE,GAAW;QACrC,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CACzB;;wFAEkF,EAClF,CAAC,GAAG,GAAG,UAAU,CAAC,CACnB,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAClK,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,yFAAyF;IACzF,KAAK,CAAC,UAAU,CAAC,EAAY;QAC3B,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CACzB,mHAAmH,CACpH,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;QAChK,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;CACF"}

package/dist/telemetry/ch/read.d.ts

@@ -0,0 +1,12 @@
+import { type Trace } from "@wizeworks/lens-core";
+import type { TraceQuery } from "../store.js";
+import type { ChClient } from "./http.js";
+/** Newest traces that match, widening the lookback until enough are found. */
+export declare function findTraceIds(ch: ChClient, t: TraceQuery, now: number): Promise<string[]>;
+/** Whole traces of some projects, in the order of `ids`. Spans that two projects both sent count once. */
+export declare function loadTraces(ch: ChClient, projects: readonly string[], ids: readonly string[]): Promise<Trace[]>;
+/** One trace, or undefined when nothing of it is stored. */
+export declare function getTrace(ch: ChClient, projects: readonly string[], traceId: string): Promise<Trace | undefined>;
+/** The primary project's trace joined with its parts from `scope`; undefined when the primary project has none. */
+export declare function joinedTrace(ch: ChClient, primaryId: string, scope: readonly string[], traceId: string): Promise<Trace | undefined>;
+//# sourceMappingURL=read.d.ts.map

package/dist/telemetry/ch/read.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/read.ts"],"names":[],"mappings":"AACA,OAAO,EAAyC,KAAK,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACzF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC9C,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAwB1C,8EAA8E;AAC9E,wBAAsB,YAAY,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAM9F;AAWD,0GAA0G;AAC1G,wBAAsB,UAAU,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CASpH;AAED,4DAA4D;AAC5D,wBAAsB,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,CAGrH;AAED,mHAAmH;AACnH,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,GAAG,SAAS,CAAC,CAIxI"}

package/dist/telemetry/ch/read.js

@@ -0,0 +1,72 @@
+// Trace reads on ClickHouse: lists from the bounds table, whole traces from the raw spans and logs.
+import { buildTrace } from "@wizeworks/lens-core";
+const CHUNK = 500;
+/** Lists look back this far first, and further only when they find too few traces. */
+const LOOKBACKS_MS = [3_600_000, 86_400_000, 7 * 86_400_000, Number.POSITIVE_INFINITY];
+/** Trace ids, newest start first, for one lookback. Row filters narrow before grouping; HAVING checks the merged bounds. */
+async function idsWithin(ch, t, now, backMs) {
+    const params = [t.projectId];
+    const p = (v, type) => (params.push(v), `{p${params.length}:${type}}`);
+    const rows = ["project_id = {p1:String}"];
+    if (Number.isFinite(backMs))
+        rows.push(`start_ms >= ${p(now - backMs, "Float64")}`);
+    if (t.since !== undefined)
+        rows.push(`updated_ms >= ${p(t.since, "Float64")}`);
+    if (t.before !== undefined)
+        rows.push(`start_ms < ${p(t.before, "Float64")}`);
+    if (t.session)
+        rows.push(`trace_id IN (SELECT trace_id FROM spans_raw WHERE project_id = {p1:String} AND session = ${p(t.session, "String")})`);
+    const having = ["1"];
+    if (t.from !== undefined)
+        having.push(`s >= ${p(t.from, "Float64")}`);
+    if (t.before !== undefined)
+        having.push(`s < ${p(t.before, "Float64")}`);
+    if (t.env)
+        having.push(`e = ${p(t.env, "String")}`);
+    const sql = `SELECT trace_id, min(start_ms) AS s, max(env) AS e FROM trace_heads WHERE ${rows.join(" AND ")} GROUP BY trace_id
+    HAVING ${having.join(" AND ")} ORDER BY s DESC, trace_id LIMIT ${Math.max(1, Math.floor(t.limit))}`;
+    return (await ch.query(sql, params)).map((r) => r.trace_id);
+}
+/** Newest traces that match, widening the lookback until enough are found. */
+export async function findTraceIds(ch, t, now) {
+    for (const back of LOOKBACKS_MS) {
+        const ids = await idsWithin(ch, t, now, back);
+        if (ids.length >= t.limit || !Number.isFinite(back))
+            return ids;
+    }
+    return [];
+}
+async function jsonOf(ch, table, projects, ids) {
+    const out = new Map();
+    for (let i = 0; i < ids.length; i += CHUNK) {
+        const sql = `SELECT trace_id, json FROM ${table} FINAL WHERE has({p1:Array(String)}, project_id) AND has({p2:Array(String)}, trace_id)`;
+        for (const r of await ch.query(sql, [projects, ids.slice(i, i + CHUNK)]))
+            out.set(r.trace_id, [...(out.get(r.trace_id) ?? []), r.json]);
+    }
+    return out;
+}
+/** Whole traces of some projects, in the order of `ids`. Spans that two projects both sent count once. */
+export async function loadTraces(ch, projects, ids) {
+    if (ids.length === 0)
+        return [];
+    const [spans, logs] = await Promise.all([jsonOf(ch, "spans_raw", projects, ids), jsonOf(ch, "logs_raw", projects, ids)]);
+    return ids.map((id) => {
+        const seen = new Set();
+        const s = (spans.get(id) ?? []).map((j) => JSON.parse(j)).filter((x) => !seen.has(x.spanId) && seen.add(x.spanId));
+        const l = (logs.get(id) ?? []).map((j) => JSON.parse(j)).sort((a, b) => a.timeMs - b.timeMs);
+        return buildTrace(id, s, l, []);
+    });
+}
+/** One trace, or undefined when nothing of it is stored. */
+export async function getTrace(ch, projects, traceId) {
+    const [t] = await loadTraces(ch, projects, [traceId]);
+    return t && t.spanCount + t.logs.length > 0 ? t : undefined;
+}
+/** The primary project's trace joined with its parts from `scope`; undefined when the primary project has none. */
+export async function joinedTrace(ch, primaryId, scope, traceId) {
+    const own = await getTrace(ch, [primaryId], traceId);
+    if (!own || scope.every((id) => id === primaryId))
+        return own;
+    return getTrace(ch, [...new Set([primaryId, ...scope])], traceId);
+}
+//# sourceMappingURL=read.js.map

package/dist/telemetry/ch/read.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"read.js","sourceRoot":"","sources":["../../../src/telemetry/ch/read.ts"],"names":[],"mappings":"AAAA,oGAAoG;AACpG,OAAO,EAAE,UAAU,EAAyC,MAAM,sBAAsB,CAAC;AAIzF,MAAM,KAAK,GAAG,GAAG,CAAC;AAClB,sFAAsF;AACtF,MAAM,YAAY,GAAG,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC,GAAG,UAAU,EAAE,MAAM,CAAC,iBAAiB,CAAC,CAAC;AAEvF,4HAA4H;AAC5H,KAAK,UAAU,SAAS,CAAC,EAAY,EAAE,CAAa,EAAE,GAAW,EAAE,MAAc;IAC/E,MAAM,MAAM,GAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,CAAC,GAAG,CAAC,CAAU,EAAE,IAAY,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,MAAM,CAAC,MAAM,IAAI,IAAI,GAAG,CAAC,CAAC;IACxF,MAAM,IAAI,GAAG,CAAC,0BAA0B,CAAC,CAAC;IAC1C,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,GAAG,GAAG,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IACpF,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAC/E,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IAC9E,IAAI,CAAC,CAAC,OAAO;QAAE,IAAI,CAAC,IAAI,CAAC,4FAA4F,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC;IAChJ,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS;QAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IACtE,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;IACzE,IAAI,CAAC,CAAC,GAAG;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;IACpD,MAAM,GAAG,GAAG,6EAA6E,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;aAChG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,oCAAoC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;IACtG,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,CAAuB,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;AACpF,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EAAY,EAAE,CAAa,EAAE,GAAW;IACzE,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QAChC,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;QAC9C,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;IAClE,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,EAAY,EAAE,KAA+B,EAAE,QAA2B,EAAE,GAAsB;IACtH,MAAM,GAAG,GAAG,IAAI,GAAG,EAAoB,CAAC;IACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;QAC3C,MAAM,GAAG,GAAG,8BAA8B,KAAK,wFAAwF,CAAC;QACxI,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC,KAAK,CAAqC,GAAG,EAAE,CAAC,QAAQ,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC9K,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,0GAA0G;AAC1G,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAY,EAAE,QAA2B,EAAE,GAAsB;IAChG,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAChC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,EAAE,WAAW,EAAE,QAAQ,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACzH,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACpB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3H,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAC1G,OAAO,UAAU,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,4DAA4D;AAC5D,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAY,EAAE,QAA2B,EAAE,OAAe;IACvF,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IACtD,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC9D,CAAC;AAED,mHAAmH;AACnH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,SAAiB,EAAE,KAAwB,EAAE,OAAe;IAC1G,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;IACrD,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,KAAK,SAAS,CAAC;QAAE,OAAO,GAAG,CAAC;IAC9D,OAAO,QAAQ,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;AACpE,CAAC"}

package/dist/telemetry/ch/schema-facts.d.ts

@@ -0,0 +1,6 @@
+export declare const FACT_DDL: readonly string[];
+/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. */
+export declare const FACT_ALTERS: readonly string[];
+/** Fact tables (storage names) and the columns that identify a row within a trace, for tombstones. */
+export declare const FACT_KEYS: Record<string, readonly string[]>;
+//# sourceMappingURL=schema-facts.d.ts.map

package/dist/telemetry/ch/schema-facts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-facts.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AA+BA,eAAO,MAAM,QAAQ,EAAE,SAAS,MAAM,EASrC,CAAC;AAEF,iKAAiK;AACjK,eAAO,MAAM,WAAW,EAAE,SAAS,MAAM,EAGxC,CAAC;AAEF,sGAAsG;AACtG,eAAO,MAAM,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAOvD,CAAC"}

package/dist/telemetry/ch/schema-facts.js

@@ -0,0 +1,49 @@
+// ClickHouse fact tables (one row per request, trace, log line, slow span, search document, attribute), and views with the Postgres names.
+import { EXPIRE, monthOf } from "./schema-raw.js";
+const P = "project_id LowCardinality(String), trace_id String";
+const VERSIONED = "ver UInt64, is_deleted UInt8, expire_ms Float64";
+/** A fact table: replaced by version, deleted by tombstone; the view reads only live rows, with the Postgres columns. */
+function fact(name, cols, order, time, indexes = "") {
+    const names = cols.split(/,\s*/).map((c) => c.trim().split(/\s+/)[0]);
+    return [
+        `CREATE TABLE IF NOT EXISTS ${name}_t (${P}, ${cols}, ${VERSIONED}${indexes ? `, ${indexes}` : ""})
+     ENGINE = ReplacingMergeTree(ver, is_deleted) PARTITION BY ${monthOf(time)} ORDER BY (${order}) ${EXPIRE}`,
+        `CREATE OR REPLACE VIEW ${name} AS SELECT project_id, trace_id, ${names.join(", ")} FROM ${name}_t FINAL`,
+    ];
+}
+const REQUEST_COLS = [
+    "env LowCardinality(String), start_ms Float64, method LowCardinality(String), path String, endpoint String, status Nullable(Int32)",
+    "status_class LowCardinality(String), failed Bool, dur_class LowCardinality(String), aborted Bool, duration_ms Float64, server_ms Nullable(Float64)",
+    "db_ms Float64, db_count Int32, fetch_count Int32, fetch_ms Float64, render_ms Nullable(Float64), network_ms Nullable(Float64)",
+    "service LowCardinality(String), services String, errors Int32, insights Int32, health LowCardinality(String), user_agent Nullable(String)",
+    "client_address Nullable(String), end_user Nullable(String), request_id Nullable(String), tags String, search String",
+].join(", ");
+const SEARCH_COLS = [
+    "kind LowCardinality(String), owner String, env LowCardinality(String), time_ms Float64, span_id Nullable(String), service String",
+    "name Nullable(String), method Nullable(String), path Nullable(String), route Nullable(String), status Nullable(Int32), outcome Nullable(String)",
+    "duration_ms Nullable(Float64), level_rank Nullable(Int16), category Nullable(String), span_kind Nullable(String), end_user Nullable(String)",
+    "error Bool, message Nullable(String), text String",
+].join(", ");
+export const FACT_DDL = [
+    ...fact("fact_requests", REQUEST_COLS, "project_id, start_ms, trace_id", "start_ms"),
+    ...fact("fact_traces", "env LowCardinality(String), start_ms Float64, path String, duration_ms Float64, health Nullable(String), status String, services String, span_count Int32, slow Bool, search String", "project_id, start_ms, trace_id", "start_ms"),
+    ...fact("fact_logs", "idx Int32, env LowCardinality(String), time_ms Float64, level LowCardinality(String), level_rank Int16, service LowCardinality(String), category String, message String, trace_name String, span_id Nullable(String), attributes String, search String", "project_id, time_ms, trace_id, idx", "time_ms"),
+    ...fact("fact_critical", "span_id String, env LowCardinality(String), start_ms Float64, duration_ms Float64, name String, service LowCardinality(String), layer LowCardinality(String)", "project_id, start_ms, trace_id, span_id", "start_ms"),
+    ...fact("fact_search", SEARCH_COLS, "project_id, kind, time_ms, trace_id, owner", "time_ms", "INDEX ix_text text TYPE ngrambf_v1(3, 65536, 2, 0) GRANULARITY 1, INDEX ix_trace trace_id TYPE bloom_filter GRANULARITY 1"),
+    ...fact("fact_attrs", "src LowCardinality(String), owner String, key LowCardinality(String), value String, num Nullable(Float64), time_ms Float64", "project_id, key, value, trace_id, src, owner", "time_ms", "INDEX ix_trace trace_id TYPE bloom_filter GRANULARITY 1"),
+];
+/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. */
+export const FACT_ALTERS = [
+    "ALTER TABLE fact_search_t ADD INDEX IF NOT EXISTS ix_text_ngrams text TYPE text(tokenizer = ngrams(3)) GRANULARITY 1",
+    "ALTER TABLE fact_search_t DROP INDEX IF EXISTS ix_text",
+];
+/** Fact tables (storage names) and the columns that identify a row within a trace, for tombstones. */
+export const FACT_KEYS = {
+    fact_requests_t: ["start_ms"],
+    fact_traces_t: ["start_ms"],
+    fact_logs_t: ["time_ms", "idx"],
+    fact_critical_t: ["start_ms", "span_id"],
+    fact_search_t: ["kind", "time_ms", "owner"],
+    fact_attrs_t: ["key", "value", "src", "owner"],
+};
+//# sourceMappingURL=schema-facts.js.map

package/dist/telemetry/ch/schema-facts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-facts.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AAAA,2IAA2I;AAC3I,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,CAAC,GAAG,oDAAoD,CAAC;AAC/D,MAAM,SAAS,GAAG,iDAAiD,CAAC;AAEpE,yHAAyH;AACzH,SAAS,IAAI,CAAC,IAAY,EAAE,IAAY,EAAE,KAAa,EAAE,IAAY,EAAE,OAAO,GAAG,EAAE;IACjF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IACvE,OAAO;QACL,8BAA8B,IAAI,OAAO,CAAC,KAAK,IAAI,KAAK,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;iEACpC,OAAO,CAAC,IAAI,CAAC,cAAc,KAAK,KAAK,MAAM,EAAE;QAC1G,0BAA0B,IAAI,oCAAoC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,UAAU;KAC1G,CAAC;AACJ,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,mIAAmI;IACnI,oJAAoJ;IACpJ,+HAA+H;IAC/H,2IAA2I;IAC3I,qHAAqH;CACtH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,WAAW,GAAG;IAClB,kIAAkI;IAClI,iJAAiJ;IACjJ,6IAA6I;IAC7I,mDAAmD;CACpD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,CAAC,MAAM,QAAQ,GAAsB;IACzC,GAAG,IAAI,CAAC,eAAe,EAAE,YAAY,EAAE,gCAAgC,EAAE,UAAU,CAAC;IACpF,GAAG,IAAI,CAAC,aAAa,EAAE,qLAAqL,EAAE,gCAAgC,EAAE,UAAU,CAAC;IAC3P,GAAG,IAAI,CAAC,WAAW,EAAE,wPAAwP,EAAE,oCAAoC,EAAE,SAAS,CAAC;IAC/T,GAAG,IAAI,CAAC,eAAe,EAAE,8JAA8J,EAAE,yCAAyC,EAAE,UAAU,CAAC;IAC/O,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,4CAA4C,EAAE,SAAS,EACzF,2HAA2H,CAAC;IAC9H,GAAG,IAAI,CAAC,YAAY,EAAE,4HAA4H,EAAE,8CAA8C,EAAE,SAAS,EAC3M,yDAAyD,CAAC;CAC7D,CAAC;AAEF,iKAAiK;AACjK,MAAM,CAAC,MAAM,WAAW,GAAsB;IAC5C,sHAAsH;IACtH,wDAAwD;CACzD,CAAC;AAEF,sGAAsG;AACtG,MAAM,CAAC,MAAM,SAAS,GAAsC;IAC1D,eAAe,EAAE,CAAC,UAAU,CAAC;IAC7B,aAAa,EAAE,CAAC,UAAU,CAAC;IAC3B,WAAW,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC;IAC/B,eAAe,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;IACxC,aAAa,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC;IAC3C,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;CAC/C,CAAC"}

package/dist/telemetry/ch/schema-raw.d.ts

@@ -0,0 +1,8 @@
+/** Epoch ms in a column as a DateTime, for partitions and TTLs. */
+export declare const secondsOf: (col: string) => string;
+export declare const monthOf: (col: string) => string;
+export declare const dayOf: (col: string) => string;
+/** Rows go when their organization's retention runs out, set when they are written. */
+export declare const EXPIRE: string;
+export declare const RAW_TABLES: readonly string[];
+//# sourceMappingURL=schema-raw.d.ts.map

package/dist/telemetry/ch/schema-raw.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-raw.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAEA,mEAAmE;AACnE,eAAO,MAAM,SAAS,QAAS,MAAM,KAAG,MAA2F,CAAC;AACpI,eAAO,MAAM,OAAO,QAAS,MAAM,KAAG,MAAuC,CAAC;AAC9E,eAAO,MAAM,KAAK,QAAS,MAAM,KAAG,MAAyC,CAAC;AAC9E,uFAAuF;AACvF,eAAO,MAAM,MAAM,QAAyC,CAAC;AAE7D,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EA0BvC,CAAC"}

package/dist/telemetry/ch/schema-raw.js

@@ -0,0 +1,35 @@
+// ClickHouse tables for what add-ons send: spans, logs, trace bounds, metric points, and the settle state.
+/** Epoch ms in a column as a DateTime, for partitions and TTLs. */
+export const secondsOf = (col) => `toDateTime(intDiv(toUInt64(least(greatest(${col}, 0.), 4102444800000.)), 1000))`;
+export const monthOf = (col) => `toYYYYMM(${secondsOf(col)})`;
+export const dayOf = (col) => `toYYYYMMDD(${secondsOf(col)})`;
+/** Rows go when their organization's retention runs out, set when they are written. */
+export const EXPIRE = `TTL ${secondsOf("expire_ms")} DELETE`;
+export const RAW_TABLES = [
+    `CREATE TABLE IF NOT EXISTS spans_raw (
+    project_id LowCardinality(String), trace_id String, span_id String, service LowCardinality(String), start_ms Float64, duration_ms Float64,
+    session Nullable(String), json String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64,
+    INDEX ix_session session TYPE bloom_filter GRANULARITY 4
+  ) ENGINE = ReplacingMergeTree(ver) PARTITION BY ${monthOf("start_ms")} ORDER BY (project_id, trace_id, span_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS logs_raw (
+    project_id LowCardinality(String), trace_id String, key String, time_ms Float64, json String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) PARTITION BY ${monthOf("time_ms")} ORDER BY (project_id, trace_id, key) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS trace_heads (
+    project_id LowCardinality(String), trace_id String, start_ms SimpleAggregateFunction(min, Float64), end_ms SimpleAggregateFunction(max, Float64),
+    updated_ms SimpleAggregateFunction(max, Float64), env SimpleAggregateFunction(max, String), expire_ms SimpleAggregateFunction(max, Float64)
+  ) ENGINE = AggregatingMergeTree ORDER BY (project_id, trace_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS trace_updates (
+    project_id LowCardinality(String), trace_id String, updated_ms Float64, env String
+  ) ENGINE = MergeTree ORDER BY updated_ms TTL ${secondsOf("updated_ms")} + INTERVAL 2 DAY DELETE`,
+    `CREATE TABLE IF NOT EXISTS settle_state (
+    project_id LowCardinality(String), trace_id String, settled_ms Float64, contrib String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY (project_id, trace_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS metric_points_raw (
+    project_id LowCardinality(String), time_ms Float64, env LowCardinality(String), service LowCardinality(String), name LowCardinality(String),
+    series_key String, json String CODEC(ZSTD(3)), expire_ms Float64
+  ) ENGINE = MergeTree PARTITION BY ${monthOf("time_ms")} ORDER BY (project_id, name, time_ms) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS project_retention (
+    project_id String, days UInt32, ver UInt64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY project_id`,
+];
+//# sourceMappingURL=schema-raw.js.map

package/dist/telemetry/ch/schema-raw.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-raw.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAAA,2GAA2G;AAE3G,mEAAmE;AACnE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,6CAA6C,GAAG,iCAAiC,CAAC;AACpI,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,YAAY,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,cAAc,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,uFAAuF;AACvF,MAAM,CAAC,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC;AAE7D,MAAM,CAAC,MAAM,UAAU,GAAsB;IAC3C;;;;oDAIkD,OAAO,CAAC,UAAU,CAAC,6CAA6C,MAAM,EAAE;IAC1H;;oDAEkD,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACrH;;;oEAGkE,MAAM,EAAE;IAC1E;;iDAE+C,SAAS,CAAC,YAAY,CAAC,0BAA0B;IAChG;;uEAEqE,MAAM,EAAE;IAC7E;;;sCAGoC,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACvG;;yDAEuD;CACxD,CAAC"}

package/dist/telemetry/ch/schema-rollups.d.ts

@@ -0,0 +1,2 @@
+export declare const ROLLUP_DDL: readonly string[];
+//# sourceMappingURL=schema-rollups.d.ts.map

package/dist/telemetry/ch/schema-rollups.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-rollups.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-rollups.ts"],"names":[],"mappings":"AAiEA,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAA8E,CAAC"}

package/dist/telemetry/ch/schema-rollups.js

@@ -0,0 +1,61 @@
+// ClickHouse rollup, histogram, and metric tiers: rows add up when they merge, and views read them merged with the Postgres names.
+import { IDS_MAX, TIERS } from "@wizeworks/lens-core";
+import { dayOf, EXPIRE, monthOf, secondsOf } from "./schema-raw.js";
+const sum = (c) => `${c} SimpleAggregateFunction(sum, Float64)`;
+const COUNTERS = ["c0", "c1", "c2", "c3", "c4", "c5"];
+/** Short tiers age out by bucket; the 1 h tier by retention. Day partitions for short tiers, month for 1 h. */
+function tierTail(tier) {
+    const ttl = tier.keepMs === null ? EXPIRE : `TTL ${secondsOf("bucket_ms")} + INTERVAL ${Math.ceil(tier.keepMs / 1000)} SECOND DELETE`;
+    return `PARTITION BY ${tier.keepMs === null ? monthOf("bucket_ms") : dayOf("bucket_ms")} ${ttl}`;
+}
+function rollupDdl(tier) {
+    const t = tier.table;
+    const cols = [
+        "project_id LowCardinality(String), env LowCardinality(String), kind LowCardinality(String), key String, bucket_ms Float64",
+        ...["n", "err", "sum_v", ...COUNTERS].map(sum),
+        "first_ms SimpleAggregateFunction(min, Float64), last_ms SimpleAggregateFunction(max, Float64), top_rank SimpleAggregateFunction(max, Float64)",
+        `top AggregateFunction(argMax, Nullable(String), Float64), ids AggregateFunction(groupArraySorted(${IDS_MAX}), Tuple(Float64, String))`,
+        "expire_ms SimpleAggregateFunction(max, Float64)",
+    ].join(", ");
+    const read = ["project_id", "env", "kind", "key", "bucket_ms", "n", "err", "sum_v", ...COUNTERS, "first_ms", "last_ms", "top_rank"].join(", ");
+    const finals = "finalizeAggregation(top) AS top, arrayStringConcat(arrayMap(x -> x.2, finalizeAggregation(ids)), ' ') AS ids";
+    return [
+        `CREATE TABLE IF NOT EXISTS ${t}_t (${cols}) ENGINE = AggregatingMergeTree ORDER BY (project_id, kind, bucket_ms, env, key) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${t} AS SELECT ${read}, ${finals} FROM ${t}_t FINAL`,
+        `CREATE TABLE IF NOT EXISTS ${tier.hist}_t (
+      project_id LowCardinality(String), env LowCardinality(String), kind LowCardinality(String), key String, bucket_ms Float64, bin Int32,
+      cnt SimpleAggregateFunction(sum, Float64), expire_ms SimpleAggregateFunction(max, Float64)
+    ) ENGINE = AggregatingMergeTree ORDER BY (project_id, kind, bucket_ms, env, key, bin) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${tier.hist} AS SELECT project_id, env, kind, key, bucket_ms, bin, cnt FROM ${tier.hist}_t FINAL`,
+    ];
+}
+function metricDdl(tier) {
+    const t = `metric_${tier.name}`;
+    const nmin = (c) => `${c} SimpleAggregateFunction(min, Nullable(Float64))`;
+    const nmax = (c) => `${c} SimpleAggregateFunction(max, Nullable(Float64))`;
+    const cols = [
+        "project_id LowCardinality(String), env LowCardinality(String), series_key String, name LowCardinality(String), bucket_ms Float64",
+        sum("cnt"), sum("vsum"), nmin("vmin"), nmax("vmax"), "first_ms SimpleAggregateFunction(min, Float64)",
+        "first_v AggregateFunction(argMin, Nullable(Float64), Float64)", "last_ms SimpleAggregateFunction(max, Float64)",
+        "last_v AggregateFunction(argMax, Nullable(Float64), Float64)", sum("rate_delta"), sum("rate_ms"), sum("h_count"), sum("h_sum"), nmin("h_min"), nmax("h_max"),
+        "h_bounds AggregateFunction(argMax, Nullable(String), UInt64), h_counts AggregateFunction(argMax, Nullable(String), UInt64)",
+        "expire_ms SimpleAggregateFunction(max, Float64)",
+    ].join(", ");
+    const read = "project_id, env, series_key, name, bucket_ms, cnt, vsum, vmin, vmax, first_ms, last_ms, rate_delta, rate_ms, h_count, h_sum, h_min, h_max";
+    const finals = ["first_v", "last_v", "h_bounds", "h_counts"].map((c) => `finalizeAggregation(${c}) AS ${c}`).join(", ");
+    return [
+        `CREATE TABLE IF NOT EXISTS ${t}_t (${cols}) ENGINE = AggregatingMergeTree ORDER BY (project_id, name, bucket_ms, env, series_key) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${t} AS SELECT ${read}, ${finals} FROM ${t}_t FINAL`,
+    ];
+}
+const SERIES_DDL = [
+    `CREATE TABLE IF NOT EXISTS metric_series_t (
+    project_id LowCardinality(String), env LowCardinality(String), series_key String, name LowCardinality(String), service LowCardinality(String),
+    kind LowCardinality(String), unit String, description String, monotonic Bool, temporality String, attributes String, last_ms Float64,
+    last_value Nullable(Float64), last_hist Nullable(String), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY (project_id, env, series_key) ${EXPIRE}`,
+    `CREATE OR REPLACE VIEW metric_series AS SELECT project_id, env, series_key, name, service, kind, unit, description, monotonic, temporality,
+    attributes, last_ms, last_value, last_hist FROM metric_series_t FINAL`,
+];
+export const ROLLUP_DDL = [...TIERS.flatMap(rollupDdl), ...TIERS.flatMap(metricDdl), ...SERIES_DDL];
+//# sourceMappingURL=schema-rollups.js.map

package/dist/telemetry/ch/schema-rollups.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-rollups.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-rollups.ts"],"names":[],"mappings":"AAAA,mIAAmI;AACnI,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEpE,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,wCAAwC,CAAC;AACxE,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEtD,+GAA+G;AAC/G,SAAS,QAAQ,CAAC,IAA4B;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,SAAS,CAAC,WAAW,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACtI,OAAO,gBAAgB,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,GAAG,EAAE,CAAC;AACnG,CAAC;AAED,SAAS,SAAS,CAAC,IAA4B;IAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;IACrB,MAAM,IAAI,GAAG;QACX,2HAA2H;QAC3H,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;QAC9C,+IAA+I;QAC/I,oGAAoG,OAAO,4BAA4B;QACvI,iDAAiD;KAClD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,IAAI,GAAG,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/I,MAAM,MAAM,GAAG,8GAA8G,CAAC;IAC9H,OAAO;QACL,8BAA8B,CAAC,OAAO,IAAI,oFAAoF,QAAQ,CAAC,IAAI,CAAC,EAAE;QAC9I,0BAA0B,CAAC,cAAc,IAAI,KAAK,MAAM,SAAS,CAAC,UAAU;QAC5E,8BAA8B,IAAI,CAAC,IAAI;;;4FAGiD,QAAQ,CAAC,IAAI,CAAC,EAAE;QACxG,0BAA0B,IAAI,CAAC,IAAI,mEAAmE,IAAI,CAAC,IAAI,UAAU;KAC1H,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,IAA4B;IAC7C,MAAM,CAAC,GAAG,UAAU,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,kDAAkD,CAAC;IACnF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,kDAAkD,CAAC;IACnF,MAAM,IAAI,GAAG;QACX,kIAAkI;QAClI,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,gDAAgD;QACrG,+DAA+D,EAAE,+CAA+C;QAChH,8DAA8D,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC;QAC7J,4HAA4H;QAC5H,iDAAiD;KAClD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,IAAI,GAAG,2IAA2I,CAAC;IACzJ,MAAM,MAAM,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxH,OAAO;QACL,8BAA8B,CAAC,OAAO,IAAI,2FAA2F,QAAQ,CAAC,IAAI,CAAC,EAAE;QACrJ,0BAA0B,CAAC,cAAc,IAAI,KAAK,MAAM,SAAS,CAAC,UAAU;KAC7E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,GAAG;IACjB;;;;8EAI4E,MAAM,EAAE;IACpF;0EACwE;CACzE,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAsB,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC"}

package/dist/telemetry/ch/schema.d.ts

@@ -0,0 +1,12 @@
+import { type ChClient } from "./http.js";
+/** Every statement, in order. A change to a shipped table needs an ALTER added here, never an edit to its CREATE. */
+export declare const CH_SCHEMA: readonly string[];
+/** Tables whose rows belong to one project, for clearing it. Views are left out. */
+export declare const PROJECT_TABLES: readonly string[];
+/** Every statement without its TTL clauses: for tests on a fixed clock, whose rows would expire on insert. */
+export declare const withoutTtl: (sql: string) => string;
+/** Create the database (through the default one) and every table and view. `ttl: false` keeps every row (tests only). */
+export declare function migrateClickHouse(url: string, client: ChClient, o?: {
+    ttl?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=schema.d.ts.map

package/dist/telemetry/ch/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,QAAQ,EAAY,MAAM,WAAW,CAAC;AAapD,qHAAqH;AACrH,eAAO,MAAM,SAAS,EAAE,SAAS,MAAM,EAAmG,CAAC;AAE3I,oFAAoF;AACpF,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EAAgD,CAAC;AAE7F,8GAA8G;AAC9G,eAAO,MAAM,UAAU,QAAS,MAAM,KAAG,MAAmD,CAAC;AAE7F,yHAAyH;AACzH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/G"}