@wizeworks/lens-cloud 0.3.0-alpha.2 → 0.3.0-alpha.4

package/dist/telemetry/ch/rebuild.d.ts

@@ -0,0 +1,23 @@
+import type { ChClient } from "./http.js";
+import type { SettleQueue } from "./queue.js";
+export declare const ensureMeta: (ch: ChClient) => Promise<void>;
+export declare class Rebuild {
+    private readonly ch;
+    private readonly queue;
+    private at;
+    private readonly size;
+    /** Keys of the page in the queue now; the next page loads when they are all settled. */
+    private page;
+    /** The current page is the last one. */
+    private ended;
+    private loading;
+    private constructor();
+    /** Picks up a rebuild a restart cut short, from its last settled page. */
+    static open(ch: ChClient, queue: SettleQueue, size?: number): Promise<Rebuild>;
+    /** Every trace settles again, from the first. */
+    start(): Promise<void>;
+    /** Before each settle run: when the current page is done, save the place and queue the next page. */
+    refill(): Promise<void>;
+    private next;
+}
+//# sourceMappingURL=rebuild.d.ts.map

package/dist/telemetry/ch/rebuild.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rebuild.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/rebuild.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAU9C,eAAO,MAAM,UAAU,OAAQ,QAAQ,kBACgG,CAAC;AAIxI,qBAAa,OAAO;IAQhB,OAAO,CAAC,QAAQ,CAAC,EAAE;IACnB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,EAAE;IACV,OAAO,CAAC,QAAQ,CAAC,IAAI;IAVvB,wFAAwF;IACxF,OAAO,CAAC,IAAI,CAAgB;IAC5B,wCAAwC;IACxC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,eAKH;IAEJ,0EAA0E;IAC1E,OAAa,IAAI,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,SAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAIjF;IAED,iDAAiD;IAC3C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAK3B;IAED,qGAAqG;IAC/F,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAY5B;YAEa,IAAI;CAWnB"}

package/dist/telemetry/ch/rebuild.js

@@ -0,0 +1,65 @@
+import { nextVer } from "./util.js";
+const PAGE = 5_000;
+const KEY = "rebuild";
+const DONE = "done";
+export const ensureMeta = (ch) => ch.command("CREATE TABLE IF NOT EXISTS ch_meta (key String, value String, ver UInt64) ENGINE = ReplacingMergeTree(ver) ORDER BY key");
+const save = (ch, at) => ch.insert("ch_meta", [{ key: KEY, value: at ? JSON.stringify(at) : DONE, ver: nextVer(Date.now()) }]);
+export class Rebuild {
+    ch;
+    queue;
+    at;
+    size;
+    /** Keys of the page in the queue now; the next page loads when they are all settled. */
+    page = [];
+    /** The current page is the last one. */
+    ended = false;
+    loading = false;
+    constructor(ch, queue, at, size) {
+        this.ch = ch;
+        this.queue = queue;
+        this.at = at;
+        this.size = size;
+    }
+    /** Picks up a rebuild a restart cut short, from its last settled page. */
+    static async open(ch, queue, size = PAGE) {
+        await ensureMeta(ch);
+        const [row] = await ch.query(`SELECT value FROM ch_meta FINAL WHERE key = '${KEY}'`);
+        return new Rebuild(ch, queue, row && row.value !== DONE ? JSON.parse(row.value) : null, size);
+    }
+    /** Every trace settles again, from the first. */
+    async start() {
+        this.at = ["", ""];
+        this.page = [];
+        this.ended = false;
+        await save(this.ch, this.at);
+    }
+    /** Before each settle run: when the current page is done, save the place and queue the next page. */
+    async refill() {
+        if (!this.at || this.loading)
+            return;
+        this.page = this.page.filter((k) => this.queue.has(k));
+        if (this.page.length)
+            return;
+        this.loading = true;
+        try {
+            await save(this.ch, this.ended ? null : this.at);
+            if (this.ended)
+                this.at = null;
+            else
+                await this.next(this.at);
+        }
+        finally {
+            this.loading = false;
+        }
+    }
+    async next(after) {
+        const rows = await this.ch.query(`SELECT project_id, trace_id, max(env) AS env, max(updated_ms) AS u FROM trace_heads WHERE (project_id, trace_id) > ({p1:String}, {p2:String})
+       GROUP BY project_id, trace_id ORDER BY project_id, trace_id LIMIT ${this.size}`, after);
+        this.page = rows.map((r) => this.queue.requeue(r.project_id, r.trace_id, r.env, r.u));
+        const last = rows.at(-1);
+        if (last)
+            this.at = [last.project_id, last.trace_id];
+        this.ended = rows.length < this.size;
+    }
+}
+//# sourceMappingURL=rebuild.js.map

package/dist/telemetry/ch/rebuild.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rebuild.js","sourceRoot":"","sources":["../../../src/telemetry/ch/rebuild.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,IAAI,GAAG,KAAK,CAAC;AACnB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,IAAI,GAAG,MAAM,CAAC;AAKpB,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAY,EAAE,EAAE,CACzC,EAAE,CAAC,OAAO,CAAC,yHAAyH,CAAC,CAAC;AAExI,MAAM,IAAI,GAAG,CAAC,EAAY,EAAE,EAAU,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAEjJ,MAAM,OAAO,OAAO;IAQC,EAAE;IACF,KAAK;IACd,EAAE;IACO,IAAI;IAVvB,wFAAwF;IAChF,IAAI,GAAa,EAAE,CAAC;IAC5B,wCAAwC;IAChC,KAAK,GAAG,KAAK,CAAC;IACd,OAAO,GAAG,KAAK,CAAC;IAExB,YACmB,EAAY,EACZ,KAAkB,EAC3B,EAAU,EACD,IAAY;kBAHZ,EAAE;qBACF,KAAK;kBACd,EAAE;oBACO,IAAI;IACpB,CAAC;IAEJ,0EAA0E;IAC1E,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAY,EAAE,KAAkB,EAAE,IAAI,GAAG,IAAI;QAC7D,MAAM,UAAU,CAAC,EAAE,CAAC,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAoB,gDAAgD,GAAG,GAAG,CAAC,CAAC;QACxG,OAAO,IAAI,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAY,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5G,CAAC;IAED,iDAAiD;IACjD,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,qGAAqG;IACrG,KAAK,CAAC,MAAM;QACV,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjD,IAAI,IAAI,CAAC,KAAK;gBAAE,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;;gBAC1B,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,KAAuB;QACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;2EACqE,IAAI,CAAC,IAAI,EAAE,EAChF,KAAK,CACN,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,IAAI;YAAE,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;IACvC,CAAC;CACF"}

package/dist/telemetry/ch/schema-facts.d.ts

@@ -0,0 +1,6 @@
+export declare const FACT_DDL: readonly string[];
+/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. Log facts gained their pattern id. */
+export declare const FACT_ALTERS: readonly string[];
+/** Fact tables (storage names) and the columns that identify a row within a trace, for tombstones. */
+export declare const FACT_KEYS: Record<string, readonly string[]>;
+//# sourceMappingURL=schema-facts.d.ts.map

package/dist/telemetry/ch/schema-facts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-facts.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AAmCA,eAAO,MAAM,QAAQ,EAAE,SAAS,MAAM,EASrC,CAAC;AAEF,oMAAoM;AACpM,eAAO,MAAM,WAAW,EAAE,SAAS,MAAM,EAMxC,CAAC;AAEF,sGAAsG;AACtG,eAAO,MAAM,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAOvD,CAAC"}

package/dist/telemetry/ch/schema-facts.js

@@ -0,0 +1,55 @@
+// ClickHouse fact tables (one row per request, trace, log line, slow span, search document, attribute), and views with the Postgres names.
+import { EXPIRE, monthOf } from "./schema-raw.js";
+const P = "project_id LowCardinality(String), trace_id String";
+const VERSIONED = "ver UInt64, is_deleted UInt8, expire_ms Float64";
+/** A fact table: replaced by version, deleted by tombstone; the view reads only live rows, with the Postgres columns. */
+function fact(name, cols, order, time, indexes = "") {
+    const names = cols.split(/,\s*/).map((c) => c.trim().split(/\s+/)[0]);
+    return [
+        `CREATE TABLE IF NOT EXISTS ${name}_t (${P}, ${cols}, ${VERSIONED}${indexes ? `, ${indexes}` : ""})
+     ENGINE = ReplacingMergeTree(ver, is_deleted) PARTITION BY ${monthOf(time)} ORDER BY (${order}) ${EXPIRE}`,
+        `CREATE OR REPLACE VIEW ${name} AS SELECT project_id, trace_id, ${names.join(", ")} FROM ${name}_t FINAL`,
+    ];
+}
+const REQUEST_COLS = [
+    "env LowCardinality(String), start_ms Float64, method LowCardinality(String), path String, endpoint String, status Nullable(Int32)",
+    "status_class LowCardinality(String), failed Bool, dur_class LowCardinality(String), aborted Bool, duration_ms Float64, server_ms Nullable(Float64)",
+    "db_ms Float64, db_count Int32, fetch_count Int32, fetch_ms Float64, render_ms Nullable(Float64), network_ms Nullable(Float64)",
+    "service LowCardinality(String), services String, errors Int32, insights Int32, health LowCardinality(String), user_agent Nullable(String)",
+    "client_address Nullable(String), end_user Nullable(String), request_id Nullable(String), tags String, search String",
+].join(", ");
+/** The log fact columns as shipped; `pattern` came later (FACT_ALTERS). */
+const LOG_COLS = "idx Int32, env LowCardinality(String), time_ms Float64, level LowCardinality(String), level_rank Int16, service LowCardinality(String), category String, message String, trace_name String, span_id Nullable(String), attributes String, search String";
+const LOG_NAMES = LOG_COLS.split(/,\s*/).map((c) => c.split(/\s+/)[0]).join(", ");
+const SEARCH_COLS = [
+    "kind LowCardinality(String), owner String, env LowCardinality(String), time_ms Float64, span_id Nullable(String), service String",
+    "name Nullable(String), method Nullable(String), path Nullable(String), route Nullable(String), status Nullable(Int32), outcome Nullable(String)",
+    "duration_ms Nullable(Float64), level_rank Nullable(Int16), category Nullable(String), span_kind Nullable(String), end_user Nullable(String)",
+    "error Bool, message Nullable(String), text String",
+].join(", ");
+export const FACT_DDL = [
+    ...fact("fact_requests", REQUEST_COLS, "project_id, start_ms, trace_id", "start_ms"),
+    ...fact("fact_traces", "env LowCardinality(String), start_ms Float64, path String, duration_ms Float64, health Nullable(String), status String, services String, span_count Int32, slow Bool, search String", "project_id, start_ms, trace_id", "start_ms"),
+    ...fact("fact_logs", LOG_COLS, "project_id, time_ms, trace_id, idx", "time_ms"),
+    ...fact("fact_critical", "span_id String, env LowCardinality(String), start_ms Float64, duration_ms Float64, name String, service LowCardinality(String), layer LowCardinality(String)", "project_id, start_ms, trace_id, span_id", "start_ms"),
+    ...fact("fact_search", SEARCH_COLS, "project_id, kind, time_ms, trace_id, owner", "time_ms", "INDEX ix_text text TYPE ngrambf_v1(3, 65536, 2, 0) GRANULARITY 1, INDEX ix_trace trace_id TYPE bloom_filter GRANULARITY 1"),
+    ...fact("fact_attrs", "src LowCardinality(String), owner String, key LowCardinality(String), value String, num Nullable(Float64), time_ms Float64", "project_id, key, value, trace_id, src, owner", "time_ms", "INDEX ix_trace trace_id TYPE bloom_filter GRANULARITY 1"),
+];
+/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. Log facts gained their pattern id. */
+export const FACT_ALTERS = [
+    "ALTER TABLE fact_search_t ADD INDEX IF NOT EXISTS ix_text_ngrams text TYPE text(tokenizer = ngrams(3)) GRANULARITY 1",
+    "ALTER TABLE fact_search_t DROP INDEX IF EXISTS ix_text",
+    "ALTER TABLE fact_logs_t ADD COLUMN IF NOT EXISTS pattern String DEFAULT ''",
+    "ALTER TABLE fact_logs_t ADD INDEX IF NOT EXISTS ix_pattern pattern TYPE bloom_filter GRANULARITY 1",
+    `CREATE OR REPLACE VIEW fact_logs AS SELECT project_id, trace_id, ${LOG_NAMES}, pattern FROM fact_logs_t FINAL`,
+];
+/** Fact tables (storage names) and the columns that identify a row within a trace, for tombstones. */
+export const FACT_KEYS = {
+    fact_requests_t: ["start_ms"],
+    fact_traces_t: ["start_ms"],
+    fact_logs_t: ["time_ms", "idx"],
+    fact_critical_t: ["start_ms", "span_id"],
+    fact_search_t: ["kind", "time_ms", "owner"],
+    fact_attrs_t: ["key", "value", "src", "owner"],
+};
+//# sourceMappingURL=schema-facts.js.map

package/dist/telemetry/ch/schema-facts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-facts.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AAAA,2IAA2I;AAC3I,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,CAAC,GAAG,oDAAoD,CAAC;AAC/D,MAAM,SAAS,GAAG,iDAAiD,CAAC;AAEpE,yHAAyH;AACzH,SAAS,IAAI,CAAC,IAAY,EAAE,IAAY,EAAE,KAAa,EAAE,IAAY,EAAE,OAAO,GAAG,EAAE;IACjF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IACvE,OAAO;QACL,8BAA8B,IAAI,OAAO,CAAC,KAAK,IAAI,KAAK,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;iEACpC,OAAO,CAAC,IAAI,CAAC,cAAc,KAAK,KAAK,MAAM,EAAE;QAC1G,0BAA0B,IAAI,oCAAoC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,UAAU;KAC1G,CAAC;AACJ,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,mIAAmI;IACnI,oJAAoJ;IACpJ,+HAA+H;IAC/H,2IAA2I;IAC3I,qHAAqH;CACtH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,2EAA2E;AAC3E,MAAM,QAAQ,GAAG,wPAAwP,CAAC;AAC1Q,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnF,MAAM,WAAW,GAAG;IAClB,kIAAkI;IAClI,iJAAiJ;IACjJ,6IAA6I;IAC7I,mDAAmD;CACpD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,CAAC,MAAM,QAAQ,GAAsB;IACzC,GAAG,IAAI,CAAC,eAAe,EAAE,YAAY,EAAE,gCAAgC,EAAE,UAAU,CAAC;IACpF,GAAG,IAAI,CAAC,aAAa,EAAE,qLAAqL,EAAE,gCAAgC,EAAE,UAAU,CAAC;IAC3P,GAAG,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,oCAAoC,EAAE,SAAS,CAAC;IAC/E,GAAG,IAAI,CAAC,eAAe,EAAE,8JAA8J,EAAE,yCAAyC,EAAE,UAAU,CAAC;IAC/O,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,4CAA4C,EAAE,SAAS,EACzF,2HAA2H,CAAC;IAC9H,GAAG,IAAI,CAAC,YAAY,EAAE,4HAA4H,EAAE,8CAA8C,EAAE,SAAS,EAC3M,yDAAyD,CAAC;CAC7D,CAAC;AAEF,oMAAoM;AACpM,MAAM,CAAC,MAAM,WAAW,GAAsB;IAC5C,sHAAsH;IACtH,wDAAwD;IACxD,4EAA4E;IAC5E,oGAAoG;IACpG,oEAAoE,SAAS,kCAAkC;CAChH,CAAC;AAEF,sGAAsG;AACtG,MAAM,CAAC,MAAM,SAAS,GAAsC;IAC1D,eAAe,EAAE,CAAC,UAAU,CAAC;IAC7B,aAAa,EAAE,CAAC,UAAU,CAAC;IAC3B,WAAW,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC;IAC/B,eAAe,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;IACxC,aAAa,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC;IAC3C,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;CAC/C,CAAC"}

package/dist/telemetry/ch/schema-raw.d.ts

@@ -0,0 +1,10 @@
+/** Epoch ms in a column as a DateTime, for partitions and TTLs. */
+export declare const secondsOf: (col: string) => string;
+export declare const monthOf: (col: string) => string;
+export declare const dayOf: (col: string) => string;
+/** Rows go when their organization's retention runs out, set when they are written. */
+export declare const EXPIRE: string;
+export declare const RAW_TABLES: readonly string[];
+/** Changes to shipped raw tables, in order. Log retention per project: 0 means the trace retention. */
+export declare const RAW_ALTERS: readonly string[];
+//# sourceMappingURL=schema-raw.d.ts.map

package/dist/telemetry/ch/schema-raw.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-raw.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAEA,mEAAmE;AACnE,eAAO,MAAM,SAAS,QAAS,MAAM,KAAG,MAA2F,CAAC;AACpI,eAAO,MAAM,OAAO,QAAS,MAAM,KAAG,MAAuC,CAAC;AAC9E,eAAO,MAAM,KAAK,QAAS,MAAM,KAAG,MAAyC,CAAC;AAC9E,uFAAuF;AACvF,eAAO,MAAM,MAAM,QAAyC,CAAC;AAE7D,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EA0BvC,CAAC;AAEF,uGAAuG;AACvG,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAAyF,CAAC"}

package/dist/telemetry/ch/schema-raw.js

@@ -0,0 +1,37 @@
+// ClickHouse tables for what add-ons send: spans, logs, trace bounds, metric points, and the settle state.
+/** Epoch ms in a column as a DateTime, for partitions and TTLs. */
+export const secondsOf = (col) => `toDateTime(intDiv(toUInt64(least(greatest(${col}, 0.), 4102444800000.)), 1000))`;
+export const monthOf = (col) => `toYYYYMM(${secondsOf(col)})`;
+export const dayOf = (col) => `toYYYYMMDD(${secondsOf(col)})`;
+/** Rows go when their organization's retention runs out, set when they are written. */
+export const EXPIRE = `TTL ${secondsOf("expire_ms")} DELETE`;
+export const RAW_TABLES = [
+    `CREATE TABLE IF NOT EXISTS spans_raw (
+    project_id LowCardinality(String), trace_id String, span_id String, service LowCardinality(String), start_ms Float64, duration_ms Float64,
+    session Nullable(String), json String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64,
+    INDEX ix_session session TYPE bloom_filter GRANULARITY 4
+  ) ENGINE = ReplacingMergeTree(ver) PARTITION BY ${monthOf("start_ms")} ORDER BY (project_id, trace_id, span_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS logs_raw (
+    project_id LowCardinality(String), trace_id String, key String, time_ms Float64, json String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) PARTITION BY ${monthOf("time_ms")} ORDER BY (project_id, trace_id, key) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS trace_heads (
+    project_id LowCardinality(String), trace_id String, start_ms SimpleAggregateFunction(min, Float64), end_ms SimpleAggregateFunction(max, Float64),
+    updated_ms SimpleAggregateFunction(max, Float64), env SimpleAggregateFunction(max, String), expire_ms SimpleAggregateFunction(max, Float64)
+  ) ENGINE = AggregatingMergeTree ORDER BY (project_id, trace_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS trace_updates (
+    project_id LowCardinality(String), trace_id String, updated_ms Float64, env String
+  ) ENGINE = MergeTree ORDER BY updated_ms TTL ${secondsOf("updated_ms")} + INTERVAL 2 DAY DELETE`,
+    `CREATE TABLE IF NOT EXISTS settle_state (
+    project_id LowCardinality(String), trace_id String, settled_ms Float64, contrib String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY (project_id, trace_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS metric_points_raw (
+    project_id LowCardinality(String), time_ms Float64, env LowCardinality(String), service LowCardinality(String), name LowCardinality(String),
+    series_key String, json String CODEC(ZSTD(3)), expire_ms Float64
+  ) ENGINE = MergeTree PARTITION BY ${monthOf("time_ms")} ORDER BY (project_id, name, time_ms) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS project_retention (
+    project_id String, days UInt32, ver UInt64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY project_id`,
+];
+/** Changes to shipped raw tables, in order. Log retention per project: 0 means the trace retention. */
+export const RAW_ALTERS = ["ALTER TABLE project_retention ADD COLUMN IF NOT EXISTS log_days UInt32 DEFAULT 0"];
+//# sourceMappingURL=schema-raw.js.map

package/dist/telemetry/ch/schema-raw.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-raw.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAAA,2GAA2G;AAE3G,mEAAmE;AACnE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,6CAA6C,GAAG,iCAAiC,CAAC;AACpI,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,YAAY,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,cAAc,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,uFAAuF;AACvF,MAAM,CAAC,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC;AAE7D,MAAM,CAAC,MAAM,UAAU,GAAsB;IAC3C;;;;oDAIkD,OAAO,CAAC,UAAU,CAAC,6CAA6C,MAAM,EAAE;IAC1H;;oDAEkD,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACrH;;;oEAGkE,MAAM,EAAE;IAC1E;;iDAE+C,SAAS,CAAC,YAAY,CAAC,0BAA0B;IAChG;;uEAEqE,MAAM,EAAE;IAC7E;;;sCAGoC,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACvG;;yDAEuD;CACxD,CAAC;AAEF,uGAAuG;AACvG,MAAM,CAAC,MAAM,UAAU,GAAsB,CAAC,kFAAkF,CAAC,CAAC"}

package/dist/telemetry/ch/schema-rollups.d.ts

@@ -0,0 +1,2 @@
+export declare const ROLLUP_DDL: readonly string[];
+//# sourceMappingURL=schema-rollups.d.ts.map

package/dist/telemetry/ch/schema-rollups.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-rollups.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-rollups.ts"],"names":[],"mappings":"AAiEA,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAA8E,CAAC"}

package/dist/telemetry/ch/schema-rollups.js

@@ -0,0 +1,61 @@
+// ClickHouse rollup, histogram, and metric tiers: rows add up when they merge, and views read them merged with the Postgres names.
+import { IDS_MAX, TIERS } from "@wizeworks/lens-core";
+import { dayOf, EXPIRE, monthOf, secondsOf } from "./schema-raw.js";
+const sum = (c) => `${c} SimpleAggregateFunction(sum, Float64)`;
+const COUNTERS = ["c0", "c1", "c2", "c3", "c4", "c5"];
+/** Short tiers age out by bucket; the 1 h tier by retention. Day partitions for short tiers, month for 1 h. */
+function tierTail(tier) {
+    const ttl = tier.keepMs === null ? EXPIRE : `TTL ${secondsOf("bucket_ms")} + INTERVAL ${Math.ceil(tier.keepMs / 1000)} SECOND DELETE`;
+    return `PARTITION BY ${tier.keepMs === null ? monthOf("bucket_ms") : dayOf("bucket_ms")} ${ttl}`;
+}
+function rollupDdl(tier) {
+    const t = tier.table;
+    const cols = [
+        "project_id LowCardinality(String), env LowCardinality(String), kind LowCardinality(String), key String, bucket_ms Float64",
+        ...["n", "err", "sum_v", ...COUNTERS].map(sum),
+        "first_ms SimpleAggregateFunction(min, Float64), last_ms SimpleAggregateFunction(max, Float64), top_rank SimpleAggregateFunction(max, Float64)",
+        `top AggregateFunction(argMax, Nullable(String), Float64), ids AggregateFunction(groupArraySorted(${IDS_MAX}), Tuple(Float64, String))`,
+        "expire_ms SimpleAggregateFunction(max, Float64)",
+    ].join(", ");
+    const read = ["project_id", "env", "kind", "key", "bucket_ms", "n", "err", "sum_v", ...COUNTERS, "first_ms", "last_ms", "top_rank"].join(", ");
+    const finals = "finalizeAggregation(top) AS top, arrayStringConcat(arrayMap(x -> x.2, finalizeAggregation(ids)), ' ') AS ids";
+    return [
+        `CREATE TABLE IF NOT EXISTS ${t}_t (${cols}) ENGINE = AggregatingMergeTree ORDER BY (project_id, kind, bucket_ms, env, key) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${t} AS SELECT ${read}, ${finals} FROM ${t}_t FINAL`,
+        `CREATE TABLE IF NOT EXISTS ${tier.hist}_t (
+      project_id LowCardinality(String), env LowCardinality(String), kind LowCardinality(String), key String, bucket_ms Float64, bin Int32,
+      cnt SimpleAggregateFunction(sum, Float64), expire_ms SimpleAggregateFunction(max, Float64)
+    ) ENGINE = AggregatingMergeTree ORDER BY (project_id, kind, bucket_ms, env, key, bin) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${tier.hist} AS SELECT project_id, env, kind, key, bucket_ms, bin, cnt FROM ${tier.hist}_t FINAL`,
+    ];
+}
+function metricDdl(tier) {
+    const t = `metric_${tier.name}`;
+    const nmin = (c) => `${c} SimpleAggregateFunction(min, Nullable(Float64))`;
+    const nmax = (c) => `${c} SimpleAggregateFunction(max, Nullable(Float64))`;
+    const cols = [
+        "project_id LowCardinality(String), env LowCardinality(String), series_key String, name LowCardinality(String), bucket_ms Float64",
+        sum("cnt"), sum("vsum"), nmin("vmin"), nmax("vmax"), "first_ms SimpleAggregateFunction(min, Float64)",
+        "first_v AggregateFunction(argMin, Nullable(Float64), Float64)", "last_ms SimpleAggregateFunction(max, Float64)",
+        "last_v AggregateFunction(argMax, Nullable(Float64), Float64)", sum("rate_delta"), sum("rate_ms"), sum("h_count"), sum("h_sum"), nmin("h_min"), nmax("h_max"),
+        "h_bounds AggregateFunction(argMax, Nullable(String), UInt64), h_counts AggregateFunction(argMax, Nullable(String), UInt64)",
+        "expire_ms SimpleAggregateFunction(max, Float64)",
+    ].join(", ");
+    const read = "project_id, env, series_key, name, bucket_ms, cnt, vsum, vmin, vmax, first_ms, last_ms, rate_delta, rate_ms, h_count, h_sum, h_min, h_max";
+    const finals = ["first_v", "last_v", "h_bounds", "h_counts"].map((c) => `finalizeAggregation(${c}) AS ${c}`).join(", ");
+    return [
+        `CREATE TABLE IF NOT EXISTS ${t}_t (${cols}) ENGINE = AggregatingMergeTree ORDER BY (project_id, name, bucket_ms, env, series_key) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${t} AS SELECT ${read}, ${finals} FROM ${t}_t FINAL`,
+    ];
+}
+const SERIES_DDL = [
+    `CREATE TABLE IF NOT EXISTS metric_series_t (
+    project_id LowCardinality(String), env LowCardinality(String), series_key String, name LowCardinality(String), service LowCardinality(String),
+    kind LowCardinality(String), unit String, description String, monotonic Bool, temporality String, attributes String, last_ms Float64,
+    last_value Nullable(Float64), last_hist Nullable(String), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY (project_id, env, series_key) ${EXPIRE}`,
+    `CREATE OR REPLACE VIEW metric_series AS SELECT project_id, env, series_key, name, service, kind, unit, description, monotonic, temporality,
+    attributes, last_ms, last_value, last_hist FROM metric_series_t FINAL`,
+];
+export const ROLLUP_DDL = [...TIERS.flatMap(rollupDdl), ...TIERS.flatMap(metricDdl), ...SERIES_DDL];
+//# sourceMappingURL=schema-rollups.js.map

package/dist/telemetry/ch/schema-rollups.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-rollups.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-rollups.ts"],"names":[],"mappings":"AAAA,mIAAmI;AACnI,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEpE,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,wCAAwC,CAAC;AACxE,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEtD,+GAA+G;AAC/G,SAAS,QAAQ,CAAC,IAA4B;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,SAAS,CAAC,WAAW,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACtI,OAAO,gBAAgB,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,GAAG,EAAE,CAAC;AACnG,CAAC;AAED,SAAS,SAAS,CAAC,IAA4B;IAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;IACrB,MAAM,IAAI,GAAG;QACX,2HAA2H;QAC3H,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;QAC9C,+IAA+I;QAC/I,oGAAoG,OAAO,4BAA4B;QACvI,iDAAiD;KAClD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,IAAI,GAAG,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/I,MAAM,MAAM,GAAG,8GAA8G,CAAC;IAC9H,OAAO;QACL,8BAA8B,CAAC,OAAO,IAAI,oFAAoF,QAAQ,CAAC,IAAI,CAAC,EAAE;QAC9I,0BAA0B,CAAC,cAAc,IAAI,KAAK,MAAM,SAAS,CAAC,UAAU;QAC5E,8BAA8B,IAAI,CAAC,IAAI;;;4FAGiD,QAAQ,CAAC,IAAI,CAAC,EAAE;QACxG,0BAA0B,IAAI,CAAC,IAAI,mEAAmE,IAAI,CAAC,IAAI,UAAU;KAC1H,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,IAA4B;IAC7C,MAAM,CAAC,GAAG,UAAU,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,kDAAkD,CAAC;IACnF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,kDAAkD,CAAC;IACnF,MAAM,IAAI,GAAG;QACX,kIAAkI;QAClI,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,gDAAgD;QACrG,+DAA+D,EAAE,+CAA+C;QAChH,8DAA8D,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC;QAC7J,4HAA4H;QAC5H,iDAAiD;KAClD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,IAAI,GAAG,2IAA2I,CAAC;IACzJ,MAAM,MAAM,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxH,OAAO;QACL,8BAA8B,CAAC,OAAO,IAAI,2FAA2F,QAAQ,CAAC,IAAI,CAAC,EAAE;QACrJ,0BAA0B,CAAC,cAAc,IAAI,KAAK,MAAM,SAAS,CAAC,UAAU;KAC7E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,GAAG;IACjB;;;;8EAI4E,MAAM,EAAE;IACpF;0EACwE;CACzE,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAsB,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC"}

package/dist/telemetry/ch/schema.d.ts

@@ -0,0 +1,12 @@
+import { type ChClient } from "./http.js";
+/** Every statement, in order. A change to a shipped table needs an ALTER added here, never an edit to its CREATE. */
+export declare const CH_SCHEMA: readonly string[];
+/** Tables whose rows belong to one project, for clearing it. Views are left out. */
+export declare const PROJECT_TABLES: readonly string[];
+/** Every statement without its TTL clauses: for tests on a fixed clock, whose rows would expire on insert. */
+export declare const withoutTtl: (sql: string) => string;
+/** Create the database (through the default one) and every table and view. `ttl: false` keeps every row (tests only). */
+export declare function migrateClickHouse(url: string, client: ChClient, o?: {
+    ttl?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=schema.d.ts.map

package/dist/telemetry/ch/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,QAAQ,EAAY,MAAM,WAAW,CAAC;AAapD,qHAAqH;AACrH,eAAO,MAAM,SAAS,EAAE,SAAS,MAAM,EAAkH,CAAC;AAE1J,oFAAoF;AACpF,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EAAgD,CAAC;AAE7F,8GAA8G;AAC9G,eAAO,MAAM,UAAU,QAAS,MAAM,KAAG,MAAmD,CAAC;AAE7F,yHAAyH;AACzH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/G"}

package/dist/telemetry/ch/schema.js

@@ -0,0 +1,25 @@
+// ClickHouse schema setup: the database, then every table and view. Statements are idempotent, so each start runs them all.
+import { chClient } from "./http.js";
+import { FACT_ALTERS, FACT_DDL } from "./schema-facts.js";
+import { RAW_ALTERS, RAW_TABLES } from "./schema-raw.js";
+import { ROLLUP_DDL } from "./schema-rollups.js";
+const TABLES = [...RAW_TABLES, ...FACT_DDL, ...ROLLUP_DDL].map((s) => /^CREATE TABLE IF NOT EXISTS (\w+)/.exec(s)?.[1]).filter((t) => !!t);
+/** New parts write no column statistics (see http.ts); old parts lose theirs as they merge. */
+const NO_STATISTICS = TABLES.map((t) => `ALTER TABLE ${t} MODIFY SETTING auto_statistics_types = ''`);
+/** Tables only settling writes, always with a token: a retried insert lands once (see settle-rows.ts, Label). */
+const DEDUPLICATED = TABLES.filter((t) => /^(fact_|rollup_|hist_)|^settle_state$/.test(t)).map((t) => `ALTER TABLE ${t} MODIFY SETTING non_replicated_deduplication_window = 1000`);
+/** Every statement, in order. A change to a shipped table needs an ALTER added here, never an edit to its CREATE. */
+export const CH_SCHEMA = [...RAW_TABLES, ...RAW_ALTERS, ...FACT_DDL, ...FACT_ALTERS, ...ROLLUP_DDL, ...NO_STATISTICS, ...DEDUPLICATED];
+/** Tables whose rows belong to one project, for clearing it. Views are left out. */
+export const PROJECT_TABLES = TABLES.filter((t) => t !== "trace_updates");
+/** Every statement without its TTL clauses: for tests on a fixed clock, whose rows would expire on insert. */
+export const withoutTtl = (sql) => sql.replace(/\sTTL\s[\s\S]*?DELETE/g, "");
+/** Create the database (through the default one) and every table and view. `ttl: false` keeps every row (tests only). */
+export async function migrateClickHouse(url, client, o = {}) {
+    const u = new URL(url);
+    u.pathname = "/default";
+    await chClient({ url: u.toString() }).command(`CREATE DATABASE IF NOT EXISTS \`${client.database.replace(/`/g, "")}\``);
+    for (const sql of CH_SCHEMA)
+        await client.command(o.ttl === false ? withoutTtl(sql) : sql);
+}
+//# sourceMappingURL=schema.js.map

package/dist/telemetry/ch/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AAAA,4HAA4H;AAC5H,OAAO,EAAiB,QAAQ,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAExJ,+FAA+F;AAC/F,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4CAA4C,CAAC,CAAC;AAEtG,iHAAiH;AACjH,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4DAA4D,CAAC,CAAC;AAEpL,qHAAqH;AACrH,MAAM,CAAC,MAAM,SAAS,GAAsB,CAAC,GAAG,UAAU,EAAE,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,EAAE,GAAG,UAAU,EAAE,GAAG,aAAa,EAAE,GAAG,YAAY,CAAC,CAAC;AAE1J,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAsB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC;AAE7F,8GAA8G;AAC9G,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;AAE7F,yHAAyH;AACzH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,MAAgB,EAAE,CAAC,GAAsB,EAAE;IAC9F,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC;IACxB,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mCAAmC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IACxH,KAAK,MAAM,GAAG,IAAI,SAAS;QAAE,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7F,CAAC"}

package/dist/telemetry/ch/settle-pool.d.ts

@@ -0,0 +1,24 @@
+import type { ProjectRunner } from "./settle-work.js";
+export interface SettlePoolOptions {
+    /** The ClickHouse URL; each worker opens its own client. */
+    url: string;
+    size: number;
+    /** The worker file. Default: the built `settle-worker.js` next to this file. */
+    entry?: URL;
+    execArgv?: string[];
+}
+export declare class SettlePool {
+    private readonly slots;
+    private readonly o;
+    private lastId;
+    private closed;
+    constructor(o: SettlePoolOptions);
+    get size(): number;
+    private spawn;
+    /** A worker died: its tasks fail (their traces settle again later), and a new worker takes its place. */
+    private lost;
+    /** Settle one project task on the least busy worker. */
+    readonly run: ProjectRunner;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=settle-pool.d.ts.map

package/dist/telemetry/ch/settle-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-pool.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-pool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAe,MAAM,kBAAkB,CAAC;AAEnE,MAAM,WAAW,iBAAiB;IAChC,4DAA4D;IAC5D,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,gFAAgF;IAChF,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAYD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IACpC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAoB;IACtC,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,MAAM,CAAS;IAEvB,YAAY,CAAC,EAAE,iBAAiB,EAG/B;IAED,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,OAAO,CAAC,KAAK;IAcb,yGAAyG;IACzG,OAAO,CAAC,IAAI;IASZ,wDAAwD;IACxD,QAAQ,CAAC,GAAG,EAAE,aAAa,CAQzB;IAEI,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAG3B;CACF"}

package/dist/telemetry/ch/settle-pool.js

@@ -0,0 +1,58 @@
+// Worker threads for settle: the CPU work of a batch runs off the main thread, so ingest and pages stay fast and settle uses more cores.
+import { Worker } from "node:worker_threads";
+export class SettlePool {
+    slots = [];
+    o;
+    lastId = 0;
+    closed = false;
+    constructor(o) {
+        this.o = o;
+        for (let i = 0; i < o.size; i++)
+            this.slots.push(this.spawn());
+    }
+    get size() {
+        return this.slots.length;
+    }
+    spawn() {
+        const entry = this.o.entry ?? new URL("./settle-worker.js", import.meta.url);
+        const slot = { worker: new Worker(entry, { workerData: { url: this.o.url }, execArgv: this.o.execArgv }), pending: new Map() };
+        slot.worker.on("message", (m) => {
+            const w = slot.pending.get(m.id);
+            slot.pending.delete(m.id);
+            if (m.error === undefined)
+                w?.resolve();
+            else
+                w?.reject(new Error(m.error));
+        });
+        slot.worker.on("error", (err) => this.lost(slot, err));
+        slot.worker.on("exit", (code) => this.lost(slot, new Error(`A settle worker stopped (exit code ${code}).`)));
+        return slot;
+    }
+    /** A worker died: its tasks fail (their traces settle again later), and a new worker takes its place. */
+    lost(slot, err) {
+        for (const w of slot.pending.values())
+            w.reject(err);
+        slot.pending.clear();
+        const i = this.slots.indexOf(slot);
+        if (i < 0 || this.closed)
+            return;
+        console.error("[wizelens-cloud] A settle worker failed and was started again:", err.message);
+        this.slots[i] = this.spawn();
+    }
+    /** Settle one project task on the least busy worker. */
+    run = (task) => {
+        if (this.closed)
+            return Promise.reject(new Error("The settle workers are closed."));
+        const slot = this.slots.reduce((a, b) => (b.pending.size < a.pending.size ? b : a));
+        const id = ++this.lastId;
+        return new Promise((resolve, reject) => {
+            slot.pending.set(id, { resolve, reject });
+            slot.worker.postMessage({ id, task });
+        });
+    };
+    async close() {
+        this.closed = true;
+        await Promise.all(this.slots.map((s) => s.worker.terminate()));
+    }
+}
+//# sourceMappingURL=settle-pool.js.map

package/dist/telemetry/ch/settle-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-pool.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-pool.ts"],"names":[],"mappings":"AAAA,yIAAyI;AACzI,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAsB7C,MAAM,OAAO,UAAU;IACJ,KAAK,GAAW,EAAE,CAAC;IACnB,CAAC,CAAoB;IAC9B,MAAM,GAAG,CAAC,CAAC;IACX,MAAM,GAAG,KAAK,CAAC;IAEvB,YAAY,CAAoB;QAC9B,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAEO,KAAK;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,oBAAoB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7E,MAAM,IAAI,GAAS,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;QACrI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,CAAiC,EAAE,EAAE;YAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1B,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;gBAAE,CAAC,EAAE,OAAO,EAAE,CAAC;;gBACnC,CAAC,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,sCAAsC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7G,OAAO,IAAI,CAAC;IACd,CAAC;IAED,yGAAyG;IACjG,IAAI,CAAC,IAAU,EAAE,GAAU;QACjC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACjC,OAAO,CAAC,KAAK,CAAC,gEAAgE,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7F,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAED,wDAAwD;IAC/C,GAAG,GAAkB,CAAC,IAAiB,EAAE,EAAE;QAClD,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC,CAAC;QACpF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpF,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;QACzB,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACjE,CAAC;CACF"}

package/dist/telemetry/ch/settle-rows.d.ts

@@ -0,0 +1,16 @@
+import { type FactTable, type SettleDelta } from "@wizeworks/lens-core";
+import type { ChClient } from "./http.js";
+import { type Keep } from "./util.js";
+type Row = Record<string, unknown>;
+/** How long a fact row is kept: log rows (lines, their search rows and attributes) by the log retention, the rest by the trace retention. */
+export declare function factKeepOf(table: FactTable, r: Row, keep: Keep): number;
+/** Fact rows as objects, with their version and expiry. */
+export declare function factObjects(rows: Record<FactTable, unknown[][]>, ver: number, expireOf: (r: Row, table: FactTable) => number): Record<FactTable, Row[]>;
+/** Rows the re-settled traces had before and no longer have, as tombstones. Only traces settled before can have any. */
+export declare function tombstones(ch: ChClient, project: string, traceIds: readonly string[], fresh: Record<FactTable, Row[]>, ver: number): Promise<Record<FactTable, Row[]>>;
+/** Settings that make one batch's insert into a table safe to send again: ClickHouse keeps the first copy of a token. */
+export type Label = (table: string) => Record<string, string>;
+/** Write a merged settle delta into every tier. Rows expire by their project's retention; log kinds by its log retention. */
+export declare function writeDeltas(ch: ChClient, delta: SettleDelta, keepOf: (project: string) => Keep, label: Label): Promise<void>;
+export {};
+//# sourceMappingURL=settle-rows.d.ts.map

package/dist/telemetry/ch/settle-rows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-rows.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AACA,OAAO,EAA6B,KAAK,SAAS,EAA4E,KAAK,WAAW,EAAS,MAAM,sBAAsB,CAAC;AAEpL,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,IAAI,EAAoB,MAAM,WAAW,CAAC;AAExD,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAMnC,6IAA6I;AAC7I,wBAAgB,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,GAAG,MAAM,CAIvE;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,KAAK,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAUvJ;AAED,wHAAwH;AACxH,wBAAsB,UAAU,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAa5K;AAoBD,yHAAyH;AACzH,MAAM,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D,6HAA6H;AAC7H,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAelI"}

package/dist/telemetry/ch/settle-rows.js

@@ -0,0 +1,74 @@
+// What one settle batch writes to ClickHouse: fact rows, tombstones for rows a trace no longer has, and rollup deltas.
+import { FACT_COLUMNS, FACT_TABLES, IDS_MAX, isLooseTraceId, LOG_KINDS, NONE_HIGH, NONE_LOW, TIERS } from "@wizeworks/lens-core";
+import { FACT_KEYS } from "./schema-facts.js";
+import { logKeepOf, rowOf } from "./util.js";
+const COUNTERS = ["c0", "c1", "c2", "c3", "c4", "c5"];
+const keyCols = (t) => FACT_KEYS[`${t}_t`];
+const keyOf = (t, r) => [r.trace_id, ...keyCols(t).map((c) => r[c])].map((v) => JSON.stringify(v ?? null)).join("|");
+/** How long a fact row is kept: log rows (lines, their search rows and attributes) by the log retention, the rest by the trace retention. */
+export function factKeepOf(table, r, keep) {
+    const id = String(r.trace_id);
+    const logRow = table === "fact_logs" || (table === "fact_search" && r.kind === "logs") || (table === "fact_attrs" && r.src === "l");
+    return logRow ? logKeepOf(id, keep) : isLooseTraceId(id) ? keep.logMs : keep.ms;
+}
+/** Fact rows as objects, with their version and expiry. */
+export function factObjects(rows, ver, expireOf) {
+    const out = {};
+    for (const t of FACT_TABLES) {
+        const cols = ["project_id", ...FACT_COLUMNS[t]];
+        out[t] = rows[t].map((values) => {
+            const r = rowOf(cols, values);
+            return { ...r, ver, is_deleted: 0, expire_ms: expireOf(r, t) };
+        });
+    }
+    return out;
+}
+/** Rows the re-settled traces had before and no longer have, as tombstones. Only traces settled before can have any. */
+export async function tombstones(ch, project, traceIds, fresh, ver) {
+    const out = {};
+    for (const t of FACT_TABLES) {
+        out[t] = [];
+        if (!traceIds.length)
+            continue;
+        const cols = ["trace_id", ...keyCols(t)];
+        const sql = `SELECT ${cols.join(", ")} FROM ${t}_t FINAL WHERE project_id = {p1:String} AND has({p2:Array(String)}, trace_id)`;
+        const keep = new Set(fresh[t].map((r) => keyOf(t, r)));
+        for (const old of await ch.query(sql, [project, traceIds])) {
+            if (!keep.has(keyOf(t, old)))
+                out[t].push({ project_id: project, ...old, ver, is_deleted: 1, expire_ms: NONE_HIGH });
+        }
+    }
+    return out;
+}
+const ROLLUP_INPUT = [
+    "project_id String, env String, kind String, key String, bucket_ms Float64, n Float64, err Float64, sum_v Float64",
+    ...COUNTERS.map((c) => `${c} Float64`),
+    "first_ms Float64, last_ms Float64, top_rank Float64, top Nullable(String), ids String, expire_ms Float64",
+].join(", ");
+/** Deltas add into the merge tables; a batch's deltas for one bucket and key become one row. */
+function rollupInsert(table) {
+    const sums = ["n", "err", "sum_v", ...COUNTERS].map((c) => `sum(${c})`).join(", ");
+    const ids = `groupArraySortedArrayState(${IDS_MAX})(arrayMap(x -> (-last_ms, x), arrayFilter(x -> x != '', splitByChar(' ', ids))))`;
+    return `${table}_t SELECT project_id, env, kind, key, bucket_ms, ${sums}, min(first_ms), max(last_ms), max(top_rank), argMaxState(top, top_rank), ${ids}, max(expire_ms)
+    FROM input('${ROLLUP_INPUT}') GROUP BY project_id, env, kind, key, bucket_ms`;
+}
+const HIST_INPUT = "project_id String, env String, kind String, key String, bucket_ms Float64, bin Int32, cnt Float64, expire_ms Float64";
+const histInsert = (table) => `${table}_t SELECT project_id, env, kind, key, bucket_ms, bin, sum(cnt), max(expire_ms) FROM input('${HIST_INPUT}') GROUP BY project_id, env, kind, key, bucket_ms, bin`;
+/** Write a merged settle delta into every tier. Rows expire by their project's retention; log kinds by its log retention. */
+export async function writeDeltas(ch, delta, keepOf, label) {
+    const ms = (project, kind) => (LOG_KINDS.has(kind) ? keepOf(project).logMs : keepOf(project).ms);
+    const writes = [];
+    for (const tier of TIERS) {
+        const rollups = delta.rollup.filter((r) => r.tier === tier.name).map((r) => ({
+            project_id: r.project, env: r.env, kind: r.kind, key: r.key, bucket_ms: r.bucket, n: r.n, err: r.err, sum_v: r.sum,
+            ...Object.fromEntries(COUNTERS.map((c, i) => [c, r.c[i] ?? 0])), first_ms: r.firstMs ?? NONE_HIGH, last_ms: r.lastMs ?? NONE_LOW,
+            top_rank: r.topRank ?? NONE_LOW, top: r.topRank === null ? null : JSON.stringify(r.top ?? null), ids: r.ids, expire_ms: r.bucket + ms(r.project, r.kind),
+        }));
+        const hists = delta.hist.filter((h) => h.tier === tier.name).map((h) => ({
+            project_id: h.project, env: h.env, kind: h.kind, key: h.key, bucket_ms: h.bucket, bin: h.bin, cnt: h.cnt, expire_ms: h.bucket + ms(h.project, h.kind),
+        }));
+        writes.push(ch.insert(rollupInsert(tier.table), rollups, label(tier.table)), ch.insert(histInsert(tier.hist), hists, label(tier.hist)));
+    }
+    await Promise.all(writes);
+}
+//# sourceMappingURL=settle-rows.js.map

package/dist/telemetry/ch/settle-rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-rows.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AAAA,uHAAuH;AACvH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAkB,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAqC,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACpL,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,OAAO,EAAa,SAAS,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAIxD,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACtD,MAAM,OAAO,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;AACvD,MAAM,KAAK,GAAG,CAAC,CAAY,EAAE,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAErI,6IAA6I;AAC7I,MAAM,UAAU,UAAU,CAAC,KAAgB,EAAE,CAAM,EAAE,IAAU;IAC7D,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,KAAK,KAAK,WAAW,IAAI,CAAC,KAAK,KAAK,aAAa,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,KAAK,YAAY,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;AAClF,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,IAAoC,EAAE,GAAW,EAAE,QAA8C;IAC3H,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,CAAC,YAAY,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC9B,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC9B,OAAO,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wHAAwH;AACxH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAY,EAAE,OAAe,EAAE,QAA2B,EAAE,KAA+B,EAAE,GAAW;IACvI,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS;QAC/B,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,+EAA+E,CAAC;QAC/H,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC,KAAK,CAAM,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QACvH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,kHAAkH;IAClH,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;IACtC,0GAA0G;CAC3G,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,gGAAgG;AAChG,SAAS,YAAY,CAAC,KAAa;IACjC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,GAAG,GAAG,8BAA8B,OAAO,mFAAmF,CAAC;IACrI,OAAO,GAAG,KAAK,oDAAoD,IAAI,6EAA6E,GAAG;kBACvI,YAAY,mDAAmD,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,GAAG,sHAAsH,CAAC;AAC1I,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE,CACnC,GAAG,KAAK,8FAA8F,UAAU,wDAAwD,CAAC;AAK3K,6HAA6H;AAC7H,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,MAAiC,EAAE,KAAY;IACjH,MAAM,EAAE,GAAG,CAAC,OAAe,EAAE,IAAgB,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACrH,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3E,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG;YAClH,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,IAAI,QAAQ;YAChI,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC;SACzJ,CAAC,CAAC,CAAC;QACJ,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvE,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC;SACtJ,CAAC,CAAC,CAAC;QACJ,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1I,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}

package/dist/telemetry/ch/settle-work.d.ts

@@ -0,0 +1,21 @@
+import { type FactOptions } from "@wizeworks/lens-core";
+import type { ChClient } from "./http.js";
+import type { Dirty } from "./queue.js";
+/** One project's part of a batch. Plain data, so it can go to a worker thread. */
+export interface ProjectTask {
+    project: string;
+    items: Dirty[];
+    now: number;
+    retentionMs: number;
+    /** How long log rows are kept for this project. */
+    logRetentionMs: number;
+    /** The project's thresholds. */
+    facts?: Partial<FactOptions>;
+    /** The row version, from the main thread, so versions only grow. */
+    ver: number;
+}
+/** Settles one project's part of a batch. Resolves once every row is written. */
+export type ProjectRunner = (task: ProjectTask) => Promise<void>;
+/** Settle one project's part of a batch. */
+export declare function settleProject(ch: ChClient, t: ProjectTask): Promise<void>;
+//# sourceMappingURL=settle-work.d.ts.map

package/dist/telemetry/ch/settle-work.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-work.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-work.ts"],"names":[],"mappings":"AACA,OAAO,EAAW,KAAK,WAAW,EAAmJ,MAAM,sBAAsB,CAAC;AAClN,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,gCAAgC;IAChC,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7B,oEAAoE;IACpE,GAAG,EAAE,MAAM,CAAC;CACb;AAED,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAwDjE,4CAA4C;AAC5C,wBAAsB,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/E"}