@wizeworks/lens-cloud 0.3.0-alpha.1 → 0.3.0-alpha.3

package/dist/telemetry/ch/schema.js

@@ -0,0 +1,25 @@
+// ClickHouse schema setup: the database, then every table and view. Statements are idempotent, so each start runs them all.
+import { chClient } from "./http.js";
+import { FACT_ALTERS, FACT_DDL } from "./schema-facts.js";
+import { RAW_TABLES } from "./schema-raw.js";
+import { ROLLUP_DDL } from "./schema-rollups.js";
+const TABLES = [...RAW_TABLES, ...FACT_DDL, ...ROLLUP_DDL].map((s) => /^CREATE TABLE IF NOT EXISTS (\w+)/.exec(s)?.[1]).filter((t) => !!t);
+/** New parts write no column statistics (see http.ts); old parts lose theirs as they merge. */
+const NO_STATISTICS = TABLES.map((t) => `ALTER TABLE ${t} MODIFY SETTING auto_statistics_types = ''`);
+/** Tables only settling writes, always with a token: a retried insert lands once (see settle-rows.ts, Label). */
+const DEDUPLICATED = TABLES.filter((t) => /^(fact_|rollup_|hist_)|^settle_state$/.test(t)).map((t) => `ALTER TABLE ${t} MODIFY SETTING non_replicated_deduplication_window = 1000`);
+/** Every statement, in order. A change to a shipped table needs an ALTER added here, never an edit to its CREATE. */
+export const CH_SCHEMA = [...RAW_TABLES, ...FACT_DDL, ...FACT_ALTERS, ...ROLLUP_DDL, ...NO_STATISTICS, ...DEDUPLICATED];
+/** Tables whose rows belong to one project, for clearing it. Views are left out. */
+export const PROJECT_TABLES = TABLES.filter((t) => t !== "trace_updates");
+/** Every statement without its TTL clauses: for tests on a fixed clock, whose rows would expire on insert. */
+export const withoutTtl = (sql) => sql.replace(/\sTTL\s[\s\S]*?DELETE/g, "");
+/** Create the database (through the default one) and every table and view. `ttl: false` keeps every row (tests only). */
+export async function migrateClickHouse(url, client, o = {}) {
+    const u = new URL(url);
+    u.pathname = "/default";
+    await chClient({ url: u.toString() }).command(`CREATE DATABASE IF NOT EXISTS \`${client.database.replace(/`/g, "")}\``);
+    for (const sql of CH_SCHEMA)
+        await client.command(o.ttl === false ? withoutTtl(sql) : sql);
+}
+//# sourceMappingURL=schema.js.map

package/dist/telemetry/ch/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AAAA,4HAA4H;AAC5H,OAAO,EAAiB,QAAQ,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAExJ,+FAA+F;AAC/F,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4CAA4C,CAAC,CAAC;AAEtG,iHAAiH;AACjH,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4DAA4D,CAAC,CAAC;AAEpL,qHAAqH;AACrH,MAAM,CAAC,MAAM,SAAS,GAAsB,CAAC,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,EAAE,GAAG,UAAU,EAAE,GAAG,aAAa,EAAE,GAAG,YAAY,CAAC,CAAC;AAE3I,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAsB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC;AAE7F,8GAA8G;AAC9G,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;AAE7F,yHAAyH;AACzH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,MAAgB,EAAE,CAAC,GAAsB,EAAE;IAC9F,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC;IACxB,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mCAAmC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IACxH,KAAK,MAAM,GAAG,IAAI,SAAS;QAAE,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7F,CAAC"}

package/dist/telemetry/ch/settle-rows.d.ts

@@ -0,0 +1,13 @@
+import { type FactTable, type SettleDelta } from "@wizeworks/lens-core";
+import type { ChClient } from "./http.js";
+type Row = Record<string, unknown>;
+/** Fact rows as objects, with their version and expiry. `expire` maps a trace id to its row expiry base. */
+export declare function factObjects(rows: Record<FactTable, unknown[][]>, ver: number, expireOf: (r: Row) => number): Record<FactTable, Row[]>;
+/** Rows the re-settled traces had before and no longer have, as tombstones. Only traces settled before can have any. */
+export declare function tombstones(ch: ChClient, project: string, traceIds: readonly string[], fresh: Record<FactTable, Row[]>, ver: number): Promise<Record<FactTable, Row[]>>;
+/** Settings that make one batch's insert into a table safe to send again: ClickHouse keeps the first copy of a token. */
+export type Label = (table: string) => Record<string, string>;
+/** Write a merged settle delta into every tier. `retentionMs` is per project. */
+export declare function writeDeltas(ch: ChClient, delta: SettleDelta, retentionMs: (project: string) => number, label: Label): Promise<void>;
+export {};
+//# sourceMappingURL=settle-rows.d.ts.map

package/dist/telemetry/ch/settle-rows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-rows.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AACA,OAAO,EAA6B,KAAK,SAAS,EAAgC,KAAK,WAAW,EAAS,MAAM,sBAAsB,CAAC;AAExI,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAG1C,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAMnC,4GAA4G;AAC5G,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAUrI;AAED,wHAAwH;AACxH,wBAAsB,UAAU,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAa5K;AAoBD,yHAAyH;AACzH,MAAM,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D,iFAAiF;AACjF,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAczI"}

package/dist/telemetry/ch/settle-rows.js

@@ -0,0 +1,67 @@
+// What one settle batch writes to ClickHouse: fact rows, tombstones for rows a trace no longer has, and rollup deltas.
+import { FACT_COLUMNS, FACT_TABLES, IDS_MAX, NONE_HIGH, NONE_LOW, TIERS } from "@wizeworks/lens-core";
+import { FACT_KEYS } from "./schema-facts.js";
+import { rowOf } from "./util.js";
+const COUNTERS = ["c0", "c1", "c2", "c3", "c4", "c5"];
+const keyCols = (t) => FACT_KEYS[`${t}_t`];
+const keyOf = (t, r) => [r.trace_id, ...keyCols(t).map((c) => r[c])].map((v) => JSON.stringify(v ?? null)).join("|");
+/** Fact rows as objects, with their version and expiry. `expire` maps a trace id to its row expiry base. */
+export function factObjects(rows, ver, expireOf) {
+    const out = {};
+    for (const t of FACT_TABLES) {
+        const cols = ["project_id", ...FACT_COLUMNS[t]];
+        out[t] = rows[t].map((values) => {
+            const r = rowOf(cols, values);
+            return { ...r, ver, is_deleted: 0, expire_ms: expireOf(r) };
+        });
+    }
+    return out;
+}
+/** Rows the re-settled traces had before and no longer have, as tombstones. Only traces settled before can have any. */
+export async function tombstones(ch, project, traceIds, fresh, ver) {
+    const out = {};
+    for (const t of FACT_TABLES) {
+        out[t] = [];
+        if (!traceIds.length)
+            continue;
+        const cols = ["trace_id", ...keyCols(t)];
+        const sql = `SELECT ${cols.join(", ")} FROM ${t}_t FINAL WHERE project_id = {p1:String} AND has({p2:Array(String)}, trace_id)`;
+        const keep = new Set(fresh[t].map((r) => keyOf(t, r)));
+        for (const old of await ch.query(sql, [project, traceIds])) {
+            if (!keep.has(keyOf(t, old)))
+                out[t].push({ project_id: project, ...old, ver, is_deleted: 1, expire_ms: NONE_HIGH });
+        }
+    }
+    return out;
+}
+const ROLLUP_INPUT = [
+    "project_id String, env String, kind String, key String, bucket_ms Float64, n Float64, err Float64, sum_v Float64",
+    ...COUNTERS.map((c) => `${c} Float64`),
+    "first_ms Float64, last_ms Float64, top_rank Float64, top Nullable(String), ids String, expire_ms Float64",
+].join(", ");
+/** Deltas add into the merge tables; a batch's deltas for one bucket and key become one row. */
+function rollupInsert(table) {
+    const sums = ["n", "err", "sum_v", ...COUNTERS].map((c) => `sum(${c})`).join(", ");
+    const ids = `groupArraySortedArrayState(${IDS_MAX})(arrayMap(x -> (-last_ms, x), arrayFilter(x -> x != '', splitByChar(' ', ids))))`;
+    return `${table}_t SELECT project_id, env, kind, key, bucket_ms, ${sums}, min(first_ms), max(last_ms), max(top_rank), argMaxState(top, top_rank), ${ids}, max(expire_ms)
+    FROM input('${ROLLUP_INPUT}') GROUP BY project_id, env, kind, key, bucket_ms`;
+}
+const HIST_INPUT = "project_id String, env String, kind String, key String, bucket_ms Float64, bin Int32, cnt Float64, expire_ms Float64";
+const histInsert = (table) => `${table}_t SELECT project_id, env, kind, key, bucket_ms, bin, sum(cnt), max(expire_ms) FROM input('${HIST_INPUT}') GROUP BY project_id, env, kind, key, bucket_ms, bin`;
+/** Write a merged settle delta into every tier. `retentionMs` is per project. */
+export async function writeDeltas(ch, delta, retentionMs, label) {
+    const writes = [];
+    for (const tier of TIERS) {
+        const rollups = delta.rollup.filter((r) => r.tier === tier.name).map((r) => ({
+            project_id: r.project, env: r.env, kind: r.kind, key: r.key, bucket_ms: r.bucket, n: r.n, err: r.err, sum_v: r.sum,
+            ...Object.fromEntries(COUNTERS.map((c, i) => [c, r.c[i] ?? 0])), first_ms: r.firstMs ?? NONE_HIGH, last_ms: r.lastMs ?? NONE_LOW,
+            top_rank: r.topRank ?? NONE_LOW, top: r.topRank === null ? null : JSON.stringify(r.top ?? null), ids: r.ids, expire_ms: r.bucket + retentionMs(r.project),
+        }));
+        const hists = delta.hist.filter((h) => h.tier === tier.name).map((h) => ({
+            project_id: h.project, env: h.env, kind: h.kind, key: h.key, bucket_ms: h.bucket, bin: h.bin, cnt: h.cnt, expire_ms: h.bucket + retentionMs(h.project),
+        }));
+        writes.push(ch.insert(rollupInsert(tier.table), rollups, label(tier.table)), ch.insert(histInsert(tier.hist), hists, label(tier.hist)));
+    }
+    await Promise.all(writes);
+}
+//# sourceMappingURL=settle-rows.js.map

package/dist/telemetry/ch/settle-rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-rows.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AAAA,uHAAuH;AACvH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAkB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAoB,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACxI,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAIlC,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACtD,MAAM,OAAO,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;AACvD,MAAM,KAAK,GAAG,CAAC,CAAY,EAAE,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAErI,4GAA4G;AAC5G,MAAM,UAAU,WAAW,CAAC,IAAoC,EAAE,GAAW,EAAE,QAA4B;IACzG,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,CAAC,YAAY,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC9B,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC9B,OAAO,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wHAAwH;AACxH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAY,EAAE,OAAe,EAAE,QAA2B,EAAE,KAA+B,EAAE,GAAW;IACvI,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS;QAC/B,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,+EAA+E,CAAC;QAC/H,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC,KAAK,CAAM,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QACvH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,kHAAkH;IAClH,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;IACtC,0GAA0G;CAC3G,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,gGAAgG;AAChG,SAAS,YAAY,CAAC,KAAa;IACjC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,GAAG,GAAG,8BAA8B,OAAO,mFAAmF,CAAC;IACrI,OAAO,GAAG,KAAK,oDAAoD,IAAI,6EAA6E,GAAG;kBACvI,YAAY,mDAAmD,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,GAAG,sHAAsH,CAAC;AAC1I,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE,CACnC,GAAG,KAAK,8FAA8F,UAAU,wDAAwD,CAAC;AAK3K,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,WAAwC,EAAE,KAAY;IACxH,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3E,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG;YAClH,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,IAAI,QAAQ;YAChI,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;SAC1J,CAAC,CAAC,CAAC;QACJ,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvE,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;SACvJ,CAAC,CAAC,CAAC;QACJ,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1I,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}

package/dist/telemetry/ch/settle.d.ts

@@ -0,0 +1,7 @@
+import type { SettleOptions } from "../store.js";
+import type { ChClient } from "./http.js";
+import type { SettleQueue } from "./queue.js";
+import { type RetentionCache } from "./util.js";
+/** Settle due traces until none are left or the budget is spent. Batches overlap (`DEPTH` at once): one reads and computes while others wait on their writes. */
+export declare function settleClickHouse(ch: ChClient, queue: SettleQueue, retention: RetentionCache, now: number, o?: SettleOptions): Promise<number>;
+//# sourceMappingURL=settle.d.ts.map

package/dist/telemetry/ch/settle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAS,WAAW,EAAE,MAAM,YAAY,CAAC;AAGrD,OAAO,EAAsB,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AA6EpE,iKAAiK;AACjK,wBAAsB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,GAAE,aAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAuBvJ"}

package/dist/telemetry/ch/settle.js

@@ -0,0 +1,96 @@
+// Settling on ClickHouse: the same facts and deltas as Postgres, written as inserts. One process settles, so no lock is needed.
+import { compact, factRows, FACT_TABLES, mergeDeltas, settleDelta, storedOf, traceFacts, withoutTop } from "@wizeworks/lens-core";
+import { loadTraces } from "./read.js";
+import { factObjects, tombstones, writeDeltas } from "./settle-rows.js";
+import { byProject, nextVer } from "./util.js";
+const BATCH = 2_000;
+const BUDGET_MS = 2_000;
+const DEPTH = 3;
+const hasData = (t) => !!t && t.spanCount + t.logs.length > 0;
+/** Stored contributions of some traces, by trace id. */
+async function storedContribs(ch, project, ids) {
+    const rows = await ch.query("SELECT trace_id, contrib FROM settle_state FINAL WHERE project_id = {p1:String} AND has({p2:Array(String)}, trace_id)", [project, ids]);
+    return new Map(rows.map((r) => [r.trace_id, r.contrib]));
+}
+/** Facts, deltas, and new settle states for one project's dirty traces. */
+async function projectWork(ch, project, items, now, retentionMs, ver) {
+    const ids = items.map((d) => d.traceId);
+    const [traces, stored] = await Promise.all([loadTraces(ch, [project], ids), storedContribs(ch, project, ids)]);
+    const raw = Object.fromEntries(FACT_TABLES.map((t) => [t, []]));
+    const work = { facts: {}, deltas: [], states: [], resettled: [] };
+    items.forEach((d, i) => {
+        const trace = traces[i];
+        const scope = { project, env: d.env };
+        const facts = hasData(trace) ? traceFacts(trace) : null;
+        const next = facts ? compact(facts, scope) : null;
+        const old = stored.get(d.traceId);
+        if (old !== undefined)
+            work.resettled.push(d.traceId);
+        work.deltas.push(settleDelta(storedOf(old || null), next, now, retentionMs));
+        if (facts)
+            for (const [t, rows] of Object.entries(factRows(facts, scope, trace)))
+                raw[t].push(...rows);
+        const start = trace?.startMs ?? now;
+        work.states.push({ project_id: project, trace_id: d.traceId, settled_ms: d.updatedMs, contrib: next ? JSON.stringify(withoutTop(next)) : "", ver, expire_ms: start + retentionMs });
+    });
+    const timeOf = (r) => Number(r.start_ms ?? r.time_ms ?? now);
+    work.facts = factObjects(raw, ver, (r) => timeOf(r) + retentionMs);
+    return work;
+}
+/** Write one project's work: facts (with tombstones) and rollups at once, then the settle states, so a failed write settles again. */
+async function writeWork(ch, project, w, ver, retentionMs, label) {
+    const dead = await tombstones(ch, project, w.resettled, w.facts, ver);
+    const facts = FACT_TABLES.map((t) => ch.insert(`${t}_t`, [...dead[t], ...w.facts[t]], label(t)));
+    await Promise.all([...facts, writeDeltas(ch, mergeDeltas(w.deltas), () => retentionMs, label)]);
+    await ch.insert("settle_state", w.states, label("settle_state"));
+}
+/** Settle one batch of taken traces. A failure releases the ones not written yet. */
+async function settleItems(ch, queue, retention, now, due, o) {
+    let left = [...due];
+    try {
+        const days = await retention.daysOf(due.map((d) => d.project));
+        for (const [project, items] of byProject(due)) {
+            const retentionMs = days.get(project) * 86_400_000;
+            const ver = nextVer(now);
+            const label = (table) => ({ insert_deduplication_token: `${ver}:${project}:${table}` });
+            await writeWork(ch, project, await projectWork(ch, project, items, now, retentionMs, ver), ver, retentionMs, label);
+            for (const d of items)
+                queue.settled(d, d.updatedMs);
+            left = left.filter((d) => d.project !== project);
+            o.onSettled?.(project);
+        }
+        return due.length;
+    }
+    finally {
+        queue.release(left);
+    }
+}
+/** Settle due traces until none are left or the budget is spent. Batches overlap (`DEPTH` at once): one reads and computes while others wait on their writes. */
+export async function settleClickHouse(ch, queue, retention, now, o = {}) {
+    const start = Date.now();
+    const batch = o.batch ?? BATCH;
+    const running = [];
+    const started = [];
+    let total = 0;
+    try {
+        for (;;) {
+            const due = queue.take(now, batch);
+            if (due.length) {
+                const run = settleItems(ch, queue, retention, now, due, o);
+                // Handled now, awaited in turn: a batch that fails while another is awaited must not end the process.
+                run.catch(() => undefined);
+                running.push(run);
+                started.push(run);
+            }
+            const last = !due.length || due.length < batch || Date.now() - start >= (o.budgetMs ?? BUDGET_MS);
+            while (running.length && (last || running.length >= DEPTH))
+                total += await running.shift();
+            if (last)
+                return total;
+        }
+    }
+    finally {
+        await Promise.allSettled(started);
+    }
+}
+//# sourceMappingURL=settle.js.map

package/dist/telemetry/ch/settle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle.ts"],"names":[],"mappings":"AAAA,gIAAgI;AAChI,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAkB,WAAW,EAAoB,WAAW,EAAE,QAAQ,EAAc,UAAU,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAIhL,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,WAAW,EAAc,UAAU,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,OAAO,EAAuB,MAAM,WAAW,CAAC;AAEpE,MAAM,KAAK,GAAG,KAAK,CAAC;AACpB,MAAM,SAAS,GAAG,KAAK,CAAC;AACxB,MAAM,KAAK,GAAG,CAAC,CAAC;AAIhB,MAAM,OAAO,GAAG,CAAC,CAAoB,EAAc,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AAE7F,wDAAwD;AACxD,KAAK,UAAU,cAAc,CAAC,EAAY,EAAE,OAAe,EAAE,GAAsB;IACjF,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CACzB,uHAAuH,EACvH,CAAC,OAAO,EAAE,GAAG,CAAC,CACf,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AASD,2EAA2E;AAC3E,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,OAAe,EAAE,KAAuB,EAAE,GAAW,EAAE,WAAmB,EAAE,GAAW;IAC9H,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAiB,CAAC,CAAC,CAAmC,CAAC;IACjH,MAAM,IAAI,GAAgB,EAAE,KAAK,EAAE,EAA8B,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC3G,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC7E,IAAI,KAAK;YAAE,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAA+B;gBAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QACrI,MAAM,KAAK,GAAG,KAAK,EAAE,OAAO,IAAI,GAAG,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,GAAG,WAAW,EAAE,CAAC,CAAC;IACtL,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,CAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sIAAsI;AACtI,KAAK,UAAU,SAAS,CAAC,EAAY,EAAE,OAAe,EAAE,CAAc,EAAE,GAAW,EAAE,WAAmB,EAAE,KAAY;IACpH,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAChG,MAAM,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,qFAAqF;AACrF,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,SAAyB,EAAE,GAAW,EAAE,GAAqB,EAAE,CAAgB;IAC1I,IAAI,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/D,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAE,GAAG,UAAU,CAAC;YACpD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YACzB,MAAM,KAAK,GAAU,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,0BAA0B,EAAE,GAAG,GAAG,IAAI,OAAO,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;YAC/F,MAAM,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YACpH,KAAK,MAAM,CAAC,IAAI,KAAK;gBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;YACjD,CAAC,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QACD,OAAO,GAAG,CAAC,MAAM,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,iKAAiK;AACjK,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAY,EAAE,KAAkB,EAAE,SAAyB,EAAE,GAAW,EAAE,CAAC,GAAkB,EAAE;IACpI,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC;IAC/B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC;QACH,SAAS,CAAC;YACR,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC3D,sGAAsG;gBACtG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC3B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC;YAClG,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;gBAAE,KAAK,IAAI,MAAM,OAAO,CAAC,KAAK,EAAG,CAAC;YAC5F,IAAI,IAAI;gBAAE,OAAO,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;AACH,CAAC"}

package/dist/telemetry/ch/store.d.ts

@@ -0,0 +1,16 @@
+import { type SqlRunner } from "@wizeworks/lens-core";
+import type { RetentionOf, TelemetryStore } from "../store.js";
+import { type ChClient } from "./http.js";
+export interface ChStoreOptions {
+    url: string;
+    retentionOf: RetentionOf;
+    defaultRetentionDays: number;
+    /** The clock for recovery on start. */
+    now: () => number;
+    /** False leaves the TTLs out (tests on a fixed clock). Default true. */
+    ttl?: boolean;
+}
+export declare const chRunner: (ch: ChClient) => SqlRunner;
+/** Create the schema, rebuild the settle queue, and return the store. */
+export declare function openClickHouse(o: ChStoreOptions): Promise<TelemetryStore>;
+//# sourceMappingURL=store.d.ts.map

package/dist/telemetry/ch/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/store.ts"],"names":[],"mappings":"AACA,OAAO,EAAoH,KAAK,SAAS,EAAc,MAAM,sBAAsB,CAAC;AACpL,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE/D,OAAO,EAAE,KAAK,QAAQ,EAAY,MAAM,WAAW,CAAC;AAUpD,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,WAAW,CAAC;IACzB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uCAAuC;IACvC,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,wEAAwE;IACxE,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED,eAAO,MAAM,QAAQ,OAAQ,QAAQ,KAAG,SAAuF,CAAC;AAWhI,yEAAyE;AACzE,wBAAsB,cAAc,CAAC,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAO/E"}

package/dist/telemetry/ch/store.js

@@ -0,0 +1,65 @@
+// Scale mode: telemetry in ClickHouse. Reads go through views with the Postgres names, so the shared SQL answers pages and searches.
+import { alignNow, CH_DIALECT, metricAlertSteps, metricPageSteps, runMetricAsync, searchSql, sqlExecutor, suggestSql } from "@wizeworks/lens-core";
+import { checkFactsVersionCh, clearProjectCh, syncProjects } from "./admin.js";
+import { chClient } from "./http.js";
+import { MetricWriter } from "./metrics.js";
+import { activeHours, counts, lastTraceMs, slowestEndpoints, traceEnvs } from "./queries.js";
+import { SettleQueue } from "./queue.js";
+import { findTraceIds, getTrace, joinedTrace, loadTraces } from "./read.js";
+import { migrateClickHouse } from "./schema.js";
+import { settleClickHouse } from "./settle.js";
+import { RetentionCache } from "./util.js";
+import { writeRaw } from "./write.js";
+export const chRunner = (ch) => ({ all: (sql, params) => ch.query(sql, params) });
+/** ClickHouse may still be starting (a deploy starts both at once): wait up to two minutes. */
+async function waitReady(ch) {
+    for (let i = 0; i < 60; i++) {
+        if (await ch.ping())
+            return;
+        await new Promise((r) => setTimeout(r, 2_000));
+    }
+    throw new Error("ClickHouse did not answer /ping for two minutes. Check --clickhouse and that the server runs.");
+}
+/** Create the schema, rebuild the settle queue, and return the store. */
+export async function openClickHouse(o) {
+    const ch = chClient({ url: o.url });
+    await waitReady(ch);
+    await migrateClickHouse(o.url, ch, { ttl: o.ttl });
+    const queue = new SettleQueue();
+    await queue.recover(ch, o.now());
+    return chTelemetry(ch, queue, new RetentionCache(o.retentionOf, o.defaultRetentionDays), o.retentionOf);
+}
+function chTelemetry(ch, queue, retention, retentionOf) {
+    const src = (projectId) => ({ d: CH_DIALECT, run: chRunner(ch), scope: { project: projectId } });
+    const metrics = new MetricWriter(ch);
+    const runMetric = (s) => ch.query(s.sql, s.args);
+    return {
+        kind: "clickhouse",
+        async write(projectId, w, now) {
+            const retentionMs = await retention.msOf(projectId);
+            await writeRaw(ch, queue, projectId, w, now, retentionMs);
+            await metrics.write(projectId, w.sender.environment ?? "", w.metrics ?? [], now, retentionMs);
+        },
+        traces: async (t) => loadTraces(ch, [t.projectId], await findTraceIds(ch, t, Date.now())),
+        trace: (projectId, traceId) => getTrace(ch, [projectId], traceId),
+        joinedTrace: (primaryId, scope, traceId) => joinedTrace(ch, primaryId, scope, traceId),
+        counts: (projectId) => counts(ch, projectId),
+        traceEnvs: (projectId) => traceEnvs(ch, projectId),
+        activeHours: (projectId, from, to) => activeHours(ch, projectId, from, to),
+        lastTraceMs: () => lastTraceMs(ch, Date.now()),
+        executor: (projectId, retentionMs, env) => sqlExecutor({ ...src(projectId), scope: { project: projectId, env }, retentionMs, loadTrace: (id) => getTrace(ch, [projectId], id).then((t) => t ?? null) }),
+        settling: async (projectId, env) => queue.count(projectId, env) || null,
+        search: (projectId, q) => searchSql(src(projectId), q),
+        suggest: (projectId, r) => suggestSql(src(projectId), r),
+        slowestEndpoints: (projectId, w, minCount, limit) => slowestEndpoints(ch, projectId, w, minCount, limit),
+        metricData: (m, range, params, now) => runMetricAsync(metricPageSteps("ch", { project: m.projectId, env: m.env }, range, alignNow(now), params), runMetric),
+        metricAlertData: (m, name, w, now) => runMetricAsync(metricAlertSteps("ch", { project: m.projectId, env: m.env, service: m.service }, name, w, now), runMetric),
+        settle: (now, o) => settleClickHouse(ch, queue, retention, now, o),
+        prune: (now) => syncProjects(ch, queue, retentionOf, now),
+        pruneRollups: async () => undefined,
+        checkFactsVersion: () => checkFactsVersionCh(ch, queue),
+        clearProject: (projectId) => clearProjectCh(ch, queue, projectId),
+        close: async () => undefined,
+    };
+}
+//# sourceMappingURL=store.js.map

package/dist/telemetry/ch/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../../src/telemetry/ch/store.ts"],"names":[],"mappings":"AAAA,qIAAqI;AACrI,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,EAAmB,cAAc,EAAE,SAAS,EAAE,WAAW,EAAkB,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAEpL,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAiB,QAAQ,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAYtC,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAY,EAAa,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAA0B,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;AAEhI,+FAA+F;AAC/F,KAAK,UAAU,SAAS,CAAC,EAAY;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,MAAM,EAAE,CAAC,IAAI,EAAE;YAAE,OAAO;QAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,+FAA+F,CAAC,CAAC;AACnH,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,CAAiB;IACpD,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACpC,MAAM,SAAS,CAAC,EAAE,CAAC,CAAC;IACpB,MAAM,iBAAiB,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACjC,OAAO,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,cAAc,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;AAC1G,CAAC;AAED,SAAS,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,SAAyB,EAAE,WAAwB;IACxG,MAAM,GAAG,GAAG,CAAC,SAAiB,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IACzG,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,CAAC,CAAmC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAwB,CAAC;IAC1G,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,EAAE,GAAG;YAC3B,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACpD,MAAM,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;YAC1D,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;QAChG,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,MAAM,YAAY,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzF,KAAK,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;QACjE,WAAW,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC;QACtF,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC;QAC5C,SAAS,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC;QAClD,WAAW,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1E,WAAW,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9C,QAAQ,EAAE,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,CACxC,WAAW,CAAC,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QAC9J,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,IAAI;QACvE,MAAM,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACtD,OAAO,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACxD,gBAAgB,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC;QACxG,UAAU,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,SAAS,CAAC;QAC3J,eAAe,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC;QAC/J,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QAClE,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,CAAC;QACzD,YAAY,EAAE,KAAK,IAAI,EAAE,CAAC,SAAS;QACnC,iBAAiB,EAAE,GAAG,EAAE,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,CAAC;QACvD,YAAY,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC;QACjE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,SAAS;KAC7B,CAAC;AACJ,CAAC"}

package/dist/telemetry/ch/util.d.ts

@@ -0,0 +1,20 @@
+import type { RetentionOf } from "../store.js";
+declare const DAY_MS = 86400000;
+/** A row version that only grows: ms times 1000, plus a counter within the same ms. */
+export declare function nextVer(now: number): number;
+/** Retention days per project, from Postgres, cached for a minute. */
+export declare class RetentionCache {
+    private readonly lookup;
+    private readonly defaultDays;
+    private readonly cache;
+    constructor(lookup: RetentionOf, defaultDays: number);
+    daysOf(projectIds: readonly string[]): Promise<Map<string, number>>;
+    msOf(projectId: string): Promise<number>;
+}
+/** A row object from a column list and values in the same order. */
+export declare function rowOf(cols: readonly string[], values: readonly unknown[]): Record<string, unknown>;
+export declare const byProject: <T extends {
+    project: string;
+}>(items: readonly T[]) => Map<string, T[]>;
+export { DAY_MS };
+//# sourceMappingURL=util.d.ts.map

package/dist/telemetry/ch/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/util.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,QAAA,MAAM,MAAM,WAAa,CAAC;AAK1B,uFAAuF;AACvF,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAG3C;AAED,sEAAsE;AACtE,qBAAa,cAAc;IAIvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAJ9B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqD;IAE3E,YACmB,MAAM,EAAE,WAAW,EACnB,WAAW,EAAE,MAAM,EAClC;IAEE,MAAM,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAQxE;IAEK,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAE7C;CACF;AAED,oEAAoE;AACpE,wBAAgB,KAAK,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAIlG;AAED,eAAO,MAAM,SAAS,GAAI,CAAC,SAAS;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,SAAS,SAAS,CAAC,EAAE,KAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,CAI7F,CAAC;AAEF,OAAO,EAAE,MAAM,EAAE,CAAC"}

package/dist/telemetry/ch/util.js

@@ -0,0 +1,45 @@
+const DAY_MS = 86_400_000;
+const RETENTION_CACHE_MS = 60_000;
+let lastVer = 0;
+/** A row version that only grows: ms times 1000, plus a counter within the same ms. */
+export function nextVer(now) {
+    lastVer = Math.max(lastVer + 1, Math.floor(now) * 1000);
+    return lastVer;
+}
+/** Retention days per project, from Postgres, cached for a minute. */
+export class RetentionCache {
+    lookup;
+    defaultDays;
+    cache = new Map();
+    constructor(lookup, defaultDays) {
+        this.lookup = lookup;
+        this.defaultDays = defaultDays;
+    }
+    async daysOf(projectIds) {
+        const now = Date.now();
+        const missing = [...new Set(projectIds)].filter((id) => !this.cache.has(id) || now - this.cache.get(id).atMs > RETENTION_CACHE_MS);
+        if (missing.length) {
+            const found = await this.lookup(missing);
+            for (const id of missing)
+                this.cache.set(id, { days: found.get(id) ?? this.defaultDays, atMs: now });
+        }
+        return new Map(projectIds.map((id) => [id, this.cache.get(id).days]));
+    }
+    async msOf(projectId) {
+        return (await this.daysOf([projectId])).get(projectId) * DAY_MS;
+    }
+}
+/** A row object from a column list and values in the same order. */
+export function rowOf(cols, values) {
+    const out = {};
+    cols.forEach((c, i) => (out[c] = values[i] ?? null));
+    return out;
+}
+export const byProject = (items) => {
+    const out = new Map();
+    for (const it of items)
+        out.set(it.project, [...(out.get(it.project) ?? []), it]);
+    return out;
+};
+export { DAY_MS };
+//# sourceMappingURL=util.js.map

package/dist/telemetry/ch/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/telemetry/ch/util.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,GAAG,UAAU,CAAC;AAC1B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,IAAI,OAAO,GAAG,CAAC,CAAC;AAEhB,uFAAuF;AACvF,MAAM,UAAU,OAAO,CAAC,GAAW;IACjC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,sEAAsE;AACtE,MAAM,OAAO,cAAc;IAIN,MAAM;IACN,WAAW;IAJb,KAAK,GAAG,IAAI,GAAG,EAA0C,CAAC;IAE3E,YACmB,MAAmB,EACnB,WAAmB;sBADnB,MAAM;2BACN,WAAW;IAC3B,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,UAA6B;QACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,IAAI,GAAG,kBAAkB,CAAC,CAAC;QACpI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzC,KAAK,MAAM,EAAE,IAAI,OAAO;gBAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;QACvG,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,SAAiB;QAC1B,OAAO,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAE,GAAG,MAAM,CAAC;IACnE,CAAC;CACF;AAED,oEAAoE;AACpE,MAAM,UAAU,KAAK,CAAC,IAAuB,EAAE,MAA0B;IACvE,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IACrD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAG,CAAgC,KAAmB,EAAoB,EAAE;IAChG,MAAM,GAAG,GAAG,IAAI,GAAG,EAAe,CAAC;IACnC,KAAK,MAAM,EAAE,IAAI,KAAK;QAAE,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IAClF,OAAO,GAAG,CAAC;AACb,CAAC,CAAC;AAEF,OAAO,EAAE,MAAM,EAAE,CAAC"}

package/dist/telemetry/ch/write.d.ts

@@ -0,0 +1,11 @@
+import type { HubWrite } from "@wizeworks/lens-hub/kit";
+import type { ChClient } from "./http.js";
+import type { SettleQueue } from "./queue.js";
+/** Small batches from many add-ons: ClickHouse buffers them into one part, and answers once they are written. */
+export declare const ASYNC: {
+    async_insert: string;
+    wait_for_async_insert: string;
+};
+/** Save one batch's raw telemetry and queue its traces for settling. Metric rollups are written by the caller. */
+export declare function writeRaw(ch: ChClient, queue: SettleQueue, projectId: string, w: HubWrite, now: number, retentionMs: number): Promise<void>;
+//# sourceMappingURL=write.d.ts.map

package/dist/telemetry/ch/write.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"write.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/write.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACxD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9C,iHAAiH;AACjH,eAAO,MAAM,KAAK;IAAK,YAAY;IAAO,qBAAqB;CAAO,CAAC;AA0CvE,kHAAkH;AAClH,wBAAsB,QAAQ,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAahJ"}

package/dist/telemetry/ch/write.js

@@ -0,0 +1,59 @@
+// Ingest into ClickHouse: raw spans, logs, trace bounds, the update log, and metric points. Server-side async inserts batch the small writes.
+import { createHash } from "node:crypto";
+import { ATTR_BROWSER_SESSION, logKey, seriesKeyOf } from "@wizeworks/lens-core";
+import { nextVer } from "./util.js";
+/** Small batches from many add-ons: ClickHouse buffers them into one part, and answers once they are written. */
+export const ASYNC = { async_insert: "1", wait_for_async_insert: "1" };
+const sha1 = (text) => createHash("sha1").update(text).digest("hex");
+const sessionOf = (s) => {
+    const v = s.attributes[ATTR_BROWSER_SESSION];
+    return typeof v === "string" && v ? v : null;
+};
+/** One bounds row per trace in the batch: its earliest start and latest end here. */
+function headRows(w, projectId, now, expire) {
+    const bounds = new Map();
+    const touch = (id, start, end) => {
+        const b = bounds.get(id);
+        bounds.set(id, b ? [Math.min(b[0], start), Math.max(b[1], end)] : [start, end]);
+    };
+    for (const s of w.spans)
+        touch(s.traceId, s.startMs, s.startMs + s.durationMs);
+    for (const { traceId, log } of w.logs)
+        touch(traceId, log.timeMs, log.timeMs);
+    const env = w.sender.environment ?? "";
+    return [...bounds].map(([trace_id, [start, end]]) => ({ project_id: projectId, trace_id, start_ms: start, end_ms: end, updated_ms: now, env, expire_ms: expire(start) }));
+}
+function spanRows(w, projectId, ver, expire) {
+    return w.spans.map((s) => ({
+        project_id: projectId, trace_id: s.traceId, span_id: s.spanId, service: s.service, start_ms: s.startMs, duration_ms: s.durationMs,
+        session: sessionOf(s), json: JSON.stringify(s), ver, expire_ms: expire(s.startMs),
+    }));
+}
+function logRows(w, projectId, ver, expire) {
+    return w.logs.map(({ traceId, log }) => ({
+        project_id: projectId, trace_id: traceId, key: sha1(logKey(log)), time_ms: log.timeMs, json: JSON.stringify(log), ver, expire_ms: expire(log.timeMs),
+    }));
+}
+function pointRows(w, projectId, expire) {
+    const env = w.sender.environment ?? "";
+    return (w.metrics ?? []).map((p) => ({
+        project_id: projectId, time_ms: p.timeMs, env, service: p.service, name: p.name, series_key: sha1(seriesKeyOf(p)), json: JSON.stringify(p), expire_ms: expire(p.timeMs),
+    }));
+}
+/** Save one batch's raw telemetry and queue its traces for settling. Metric rollups are written by the caller. */
+export async function writeRaw(ch, queue, projectId, w, now, retentionMs) {
+    const expire = (ms) => ms + retentionMs;
+    const ver = nextVer(now);
+    const heads = headRows(w, projectId, now, expire);
+    await Promise.all([
+        ch.insert("spans_raw", spanRows(w, projectId, ver, expire), ASYNC),
+        ch.insert("logs_raw", logRows(w, projectId, ver, expire), ASYNC),
+        ch.insert("metric_points_raw", pointRows(w, projectId, expire), ASYNC),
+    ]);
+    // Bounds and the update log go last, so a settle never sees a trace before its parts.
+    const updates = heads.map((h) => ({ project_id: h.project_id, trace_id: h.trace_id, updated_ms: now, env: h.env }));
+    await Promise.all([ch.insert("trace_heads", heads, ASYNC), ch.insert("trace_updates", updates, ASYNC)]);
+    for (const h of heads)
+        queue.touch(projectId, h.trace_id, h.env, now);
+}
+//# sourceMappingURL=write.js.map

package/dist/telemetry/ch/write.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"write.js","sourceRoot":"","sources":["../../../src/telemetry/ch/write.ts"],"names":[],"mappings":"AAAA,8IAA8I;AAC9I,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,MAAM,EAAE,WAAW,EAAa,MAAM,sBAAsB,CAAC;AAI5F,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,iHAAiH;AACjH,MAAM,CAAC,MAAM,KAAK,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,qBAAqB,EAAE,GAAG,EAAE,CAAC;AAEvE,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAE7E,MAAM,SAAS,GAAG,CAAC,CAAO,EAAE,EAAE;IAC5B,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC;IAC7C,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC,CAAC;AAEF,qFAAqF;AACrF,SAAS,QAAQ,CAAC,CAAW,EAAE,SAAiB,EAAE,GAAW,EAAE,MAA8B;IAC3F,MAAM,MAAM,GAAG,IAAI,GAAG,EAA4B,CAAC;IACnD,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,KAAa,EAAE,GAAW,EAAE,EAAE;QACvD,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAClF,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK;QAAE,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAC/E,KAAK,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,IAAI;QAAE,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC9E,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC;IACvC,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;AAC5K,CAAC;AAED,SAAS,QAAQ,CAAC,CAAW,EAAE,SAAiB,EAAE,GAAW,EAAE,MAA8B;IAC3F,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACzB,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,UAAU;QACjI,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;KAClF,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,OAAO,CAAC,CAAW,EAAE,SAAiB,EAAE,GAAW,EAAE,MAA8B;IAC1F,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;QACvC,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;KACrJ,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,SAAS,CAAC,CAAW,EAAE,SAAiB,EAAE,MAA8B;IAC/E,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC;IACvC,OAAO,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACnC,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;KACxK,CAAC,CAAC,CAAC;AACN,CAAC;AAED,kHAAkH;AAClH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAY,EAAE,KAAkB,EAAE,SAAiB,EAAE,CAAW,EAAE,GAAW,EAAE,WAAmB;IAC/H,MAAM,MAAM,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,EAAE,GAAG,WAAW,CAAC;IAChD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IAClD,MAAM,OAAO,CAAC,GAAG,CAAC;QAChB,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,KAAK,CAAC;QAClE,EAAE,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,KAAK,CAAC;QAChE,EAAE,CAAC,MAAM,CAAC,mBAAmB,EAAE,SAAS,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,EAAE,KAAK,CAAC;KACvE,CAAC,CAAC;IACH,sFAAsF;IACtF,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACpH,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACxG,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;AACxE,CAAC"}

package/dist/telemetry/copy.d.ts

@@ -0,0 +1,9 @@
+import type { Db } from "../db/db.js";
+import type { TelemetryStore } from "./store.js";
+export interface CopyResult {
+    traces: number;
+    points: number;
+}
+/** Copy everything (the target settles it later); projects the target has are skipped, else rollups count twice; `force` copies them too, after the switch. */
+export declare function copyTelemetry(db: Db, to: TelemetryStore, now: number, log?: (line: string) => void, force?: boolean): Promise<CopyResult>;
+//# sourceMappingURL=copy.d.ts.map

package/dist/telemetry/copy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"copy.d.ts","sourceRoot":"","sources":["../../src/telemetry/copy.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAIjD,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAoDD,+JAA+J;AAC/J,wBAAsB,aAAa,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAe,EAAE,KAAK,UAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAcvJ"}

package/dist/telemetry/copy.js

@@ -0,0 +1,60 @@
+const PAGE = 500;
+/** One page of a project's traces, newest first, after the (start, id) cursor. */
+async function tracePage(db, projectId, after) {
+    const cursor = after ? "AND (start_ms < $2 OR (start_ms = $2 AND trace_id > $3))" : "";
+    return db.query(`SELECT trace_id, start_ms, env FROM traces WHERE project_id = $1 ${cursor} ORDER BY start_ms DESC, trace_id LIMIT ${PAGE}`, after ? [projectId, after[0], after[1]] : [projectId]);
+}
+async function partsOf(db, projectId, ids) {
+    const spans = await db.query("SELECT trace_id, json FROM spans WHERE project_id = $1 AND trace_id = ANY($2)", [projectId, [...ids]]);
+    const logs = await db.query("SELECT trace_id, json FROM logs WHERE project_id = $1 AND trace_id = ANY($2)", [projectId, [...ids]]);
+    return { spans, logs };
+}
+/** Every trace of a project, written one trace at a time so each keeps its environment. */
+async function copyTraces(db, to, projectId, now) {
+    let after = null;
+    let n = 0;
+    for (;;) {
+        const page = await tracePage(db, projectId, after);
+        if (!page.length)
+            return n;
+        const { spans, logs } = await partsOf(db, projectId, page.map((t) => t.trace_id));
+        for (const t of page) {
+            const s = spans.filter((r) => r.trace_id === t.trace_id).map((r) => JSON.parse(r.json));
+            const l = logs.filter((r) => r.trace_id === t.trace_id).map((r) => ({ traceId: t.trace_id, log: JSON.parse(r.json) }));
+            await to.write(projectId, { sender: { service: s[0]?.service ?? "copy", environment: t.env }, via: "addon", spans: s, logs: l, remotes: [] }, now);
+        }
+        n += page.length;
+        after = [page[page.length - 1].start_ms, page[page.length - 1].trace_id];
+    }
+}
+/** Every metric point of a project, oldest first, in batches of one environment, so counters and rates come out as at ingest. */
+async function copyPoints(db, to, projectId, now) {
+    let n = 0;
+    for (let offset = 0;; offset += PAGE * 10) {
+        const rows = await db.query(`SELECT env, json FROM metric_points WHERE project_id = $1 ORDER BY time_ms, series_key LIMIT ${PAGE * 10} OFFSET ${offset}`, [projectId]);
+        if (!rows.length)
+            return n;
+        for (const env of [...new Set(rows.map((r) => r.env))]) {
+            const metrics = rows.filter((r) => r.env === env).map((r) => JSON.parse(r.json));
+            await to.write(projectId, { sender: { service: "copy", environment: env }, via: "otlp", spans: [], logs: [], remotes: [], metrics }, now);
+        }
+        n += rows.length;
+    }
+}
+/** Copy everything (the target settles it later); projects the target has are skipped, else rollups count twice; `force` copies them too, after the switch. */
+export async function copyTelemetry(db, to, now, log = () => { }, force = false) {
+    const out = { traces: 0, points: 0 };
+    for (const { id } of await db.query("SELECT id FROM projects ORDER BY id")) {
+        if (!force && (await to.counts(id)).traces > 0) {
+            log(`  ${id}: already copied, skipped`);
+            continue;
+        }
+        const traces = await copyTraces(db, to, id, now);
+        const points = await copyPoints(db, to, id, now);
+        log(`  ${id}: ${traces} traces, ${points} metric points`);
+        out.traces += traces;
+        out.points += points;
+    }
+    return out;
+}
+//# sourceMappingURL=copy.js.map

package/dist/telemetry/copy.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"copy.js","sourceRoot":"","sources":["../../src/telemetry/copy.ts"],"names":[],"mappings":"AAKA,MAAM,IAAI,GAAG,GAAG,CAAC;AAOjB,kFAAkF;AAClF,KAAK,UAAU,SAAS,CAAC,EAAM,EAAE,SAAiB,EAAE,KAA8B;IAChF,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,0DAA0D,CAAC,CAAC,CAAC,EAAE,CAAC;IACvF,OAAO,EAAE,CAAC,KAAK,CACb,oEAAoE,MAAM,2CAA2C,IAAI,EAAE,EAC3H,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CACtD,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,EAAM,EAAE,SAAiB,EAAE,GAAsB;IACtE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,KAAK,CAAqC,+EAA+E,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACzK,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CAAqC,8EAA8E,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IACvK,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC;AAED,2FAA2F;AAC3F,KAAK,UAAU,UAAU,CAAC,EAAM,EAAE,EAAkB,EAAE,SAAiB,EAAE,GAAW;IAClF,IAAI,KAAK,GAA4B,IAAI,CAAC;IAC1C,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;QAClF,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAS,CAAC,CAAC;YAChG,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAc,EAAE,CAAC,CAAC,CAAC;YACpI,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,IAAI,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACrJ,CAAC;QACD,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC;QACjB,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,QAAQ,CAAC,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,iIAAiI;AACjI,KAAK,UAAU,UAAU,CAAC,EAAM,EAAE,EAAkB,EAAE,SAAiB,EAAE,GAAW;IAClF,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,MAAM,GAAG,CAAC,GAAI,MAAM,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CACzB,gGAAgG,IAAI,GAAG,EAAE,WAAW,MAAM,EAAE,EAC5H,CAAC,SAAS,CAAC,CACZ,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACvD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAgB,CAAC,CAAC;YAChG,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,GAAG,CAAC,CAAC;QAC5I,CAAC;QACD,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC;IACnB,CAAC;AACH,CAAC;AAED,+JAA+J;AAC/J,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAAM,EAAE,EAAkB,EAAE,GAAW,EAAE,GAAG,GAA2B,GAAG,EAAE,GAAE,CAAC,EAAE,KAAK,GAAG,KAAK;IAChI,MAAM,GAAG,GAAe,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACjD,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,MAAM,EAAE,CAAC,KAAK,CAAiB,qCAAqC,CAAC,EAAE,CAAC;QAC3F,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,GAAG,CAAC,KAAK,EAAE,2BAA2B,CAAC,CAAC;YACxC,SAAS;QACX,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QACjD,GAAG,CAAC,KAAK,EAAE,KAAK,MAAM,YAAY,MAAM,gBAAgB,CAAC,CAAC;QAC1D,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC;QACrB,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC;IACvB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}