@wizeworks/lens-cloud 0.3.0-alpha.3 → 0.3.0-alpha.5

package/dist/telemetry/ch/rebuild.js

@@ -0,0 +1,65 @@
+import { nextVer } from "./util.js";
+const PAGE = 5_000;
+const KEY = "rebuild";
+const DONE = "done";
+export const ensureMeta = (ch) => ch.command("CREATE TABLE IF NOT EXISTS ch_meta (key String, value String, ver UInt64) ENGINE = ReplacingMergeTree(ver) ORDER BY key");
+const save = (ch, at) => ch.insert("ch_meta", [{ key: KEY, value: at ? JSON.stringify(at) : DONE, ver: nextVer(Date.now()) }]);
+export class Rebuild {
+    ch;
+    queue;
+    at;
+    size;
+    /** Keys of the page in the queue now; the next page loads when they are all settled. */
+    page = [];
+    /** The current page is the last one. */
+    ended = false;
+    loading = false;
+    constructor(ch, queue, at, size) {
+        this.ch = ch;
+        this.queue = queue;
+        this.at = at;
+        this.size = size;
+    }
+    /** Picks up a rebuild a restart cut short, from its last settled page. */
+    static async open(ch, queue, size = PAGE) {
+        await ensureMeta(ch);
+        const [row] = await ch.query(`SELECT value FROM ch_meta FINAL WHERE key = '${KEY}'`);
+        return new Rebuild(ch, queue, row && row.value !== DONE ? JSON.parse(row.value) : null, size);
+    }
+    /** Every trace settles again, from the first. */
+    async start() {
+        this.at = ["", ""];
+        this.page = [];
+        this.ended = false;
+        await save(this.ch, this.at);
+    }
+    /** Before each settle run: when the current page is done, save the place and queue the next page. */
+    async refill() {
+        if (!this.at || this.loading)
+            return;
+        this.page = this.page.filter((k) => this.queue.has(k));
+        if (this.page.length)
+            return;
+        this.loading = true;
+        try {
+            await save(this.ch, this.ended ? null : this.at);
+            if (this.ended)
+                this.at = null;
+            else
+                await this.next(this.at);
+        }
+        finally {
+            this.loading = false;
+        }
+    }
+    async next(after) {
+        const rows = await this.ch.query(`SELECT project_id, trace_id, max(env) AS env, max(updated_ms) AS u FROM trace_heads WHERE (project_id, trace_id) > ({p1:String}, {p2:String})
+       GROUP BY project_id, trace_id ORDER BY project_id, trace_id LIMIT ${this.size}`, after);
+        this.page = rows.map((r) => this.queue.requeue(r.project_id, r.trace_id, r.env, r.u));
+        const last = rows.at(-1);
+        if (last)
+            this.at = [last.project_id, last.trace_id];
+        this.ended = rows.length < this.size;
+    }
+}
+//# sourceMappingURL=rebuild.js.map

package/dist/telemetry/ch/rebuild.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"rebuild.js","sourceRoot":"","sources":["../../../src/telemetry/ch/rebuild.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,IAAI,GAAG,KAAK,CAAC;AACnB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,IAAI,GAAG,MAAM,CAAC;AAKpB,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,EAAY,EAAE,EAAE,CACzC,EAAE,CAAC,OAAO,CAAC,yHAAyH,CAAC,CAAC;AAExI,MAAM,IAAI,GAAG,CAAC,EAAY,EAAE,EAAU,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;AAEjJ,MAAM,OAAO,OAAO;IAQC,EAAE;IACF,KAAK;IACd,EAAE;IACO,IAAI;IAVvB,wFAAwF;IAChF,IAAI,GAAa,EAAE,CAAC;IAC5B,wCAAwC;IAChC,KAAK,GAAG,KAAK,CAAC;IACd,OAAO,GAAG,KAAK,CAAC;IAExB,YACmB,EAAY,EACZ,KAAkB,EAC3B,EAAU,EACD,IAAY;kBAHZ,EAAE;qBACF,KAAK;kBACd,EAAE;oBACO,IAAI;IACpB,CAAC;IAEJ,0EAA0E;IAC1E,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAY,EAAE,KAAkB,EAAE,IAAI,GAAG,IAAI;QAC7D,MAAM,UAAU,CAAC,EAAE,CAAC,CAAC;QACrB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC,KAAK,CAAoB,gDAAgD,GAAG,GAAG,CAAC,CAAC;QACxG,OAAO,IAAI,OAAO,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAY,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC5G,CAAC;IAED,iDAAiD;IACjD,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QACnB,IAAI,CAAC,IAAI,GAAG,EAAE,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAC/B,CAAC;IAED,qGAAqG;IACrG,KAAK,CAAC,MAAM;QACV,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACjD,IAAI,IAAI,CAAC,KAAK;gBAAE,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;;gBAC1B,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAChC,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,IAAI,CAAC,KAAuB;QACxC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;2EACqE,IAAI,CAAC,IAAI,EAAE,EAChF,KAAK,CACN,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtF,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,IAAI;YAAE,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;IACvC,CAAC;CACF"}

package/dist/telemetry/ch/schema-facts.d.ts

@@ -1,5 +1,5 @@
 export declare const FACT_DDL: readonly string[];
-/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. */
+/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. Log facts gained their pattern id. */
 export declare const FACT_ALTERS: readonly string[];
 /** Fact tables (storage names) and the columns that identify a row within a trace, for tombstones. */
 export declare const FACT_KEYS: Record<string, readonly string[]>;

package/dist/telemetry/ch/schema-facts.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema-facts.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AA+BA,eAAO,MAAM,QAAQ,EAAE,SAAS,MAAM,EASrC,CAAC;AAEF,iKAAiK;AACjK,eAAO,MAAM,WAAW,EAAE,SAAS,MAAM,EAGxC,CAAC;AAEF,sGAAsG;AACtG,eAAO,MAAM,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAOvD,CAAC"}
+{"version":3,"file":"schema-facts.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AAmCA,eAAO,MAAM,QAAQ,EAAE,SAAS,MAAM,EASrC,CAAC;AAEF,oMAAoM;AACpM,eAAO,MAAM,WAAW,EAAE,SAAS,MAAM,EAMxC,CAAC;AAEF,sGAAsG;AACtG,eAAO,MAAM,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAOvD,CAAC"}

package/dist/telemetry/ch/schema-facts.js

@@ -18,6 +18,9 @@ const REQUEST_COLS = [
     "service LowCardinality(String), services String, errors Int32, insights Int32, health LowCardinality(String), user_agent Nullable(String)",
     "client_address Nullable(String), end_user Nullable(String), request_id Nullable(String), tags String, search String",
 ].join(", ");
+/** The log fact columns as shipped; `pattern` came later (FACT_ALTERS). */
+const LOG_COLS = "idx Int32, env LowCardinality(String), time_ms Float64, level LowCardinality(String), level_rank Int16, service LowCardinality(String), category String, message String, trace_name String, span_id Nullable(String), attributes String, search String";
+const LOG_NAMES = LOG_COLS.split(/,\s*/).map((c) => c.split(/\s+/)[0]).join(", ");
 const SEARCH_COLS = [
     "kind LowCardinality(String), owner String, env LowCardinality(String), time_ms Float64, span_id Nullable(String), service String",
     "name Nullable(String), method Nullable(String), path Nullable(String), route Nullable(String), status Nullable(Int32), outcome Nullable(String)",
@@ -27,15 +30,18 @@ const SEARCH_COLS = [
 export const FACT_DDL = [
     ...fact("fact_requests", REQUEST_COLS, "project_id, start_ms, trace_id", "start_ms"),
     ...fact("fact_traces", "env LowCardinality(String), start_ms Float64, path String, duration_ms Float64, health Nullable(String), status String, services String, span_count Int32, slow Bool, search String", "project_id, start_ms, trace_id", "start_ms"),
-    ...fact("fact_logs", "idx Int32, env LowCardinality(String), time_ms Float64, level LowCardinality(String), level_rank Int16, service LowCardinality(String), category String, message String, trace_name String, span_id Nullable(String), attributes String, search String", "project_id, time_ms, trace_id, idx", "time_ms"),
+    ...fact("fact_logs", LOG_COLS, "project_id, time_ms, trace_id, idx", "time_ms"),
     ...fact("fact_critical", "span_id String, env LowCardinality(String), start_ms Float64, duration_ms Float64, name String, service LowCardinality(String), layer LowCardinality(String)", "project_id, start_ms, trace_id, span_id", "start_ms"),
     ...fact("fact_search", SEARCH_COLS, "project_id, kind, time_ms, trace_id, owner", "time_ms", "INDEX ix_text text TYPE ngrambf_v1(3, 65536, 2, 0) GRANULARITY 1, INDEX ix_trace trace_id TYPE bloom_filter GRANULARITY 1"),
     ...fact("fact_attrs", "src LowCardinality(String), owner String, key LowCardinality(String), value String, num Nullable(Float64), time_ms Float64", "project_id, key, value, trace_id, src, owner", "time_ms", "INDEX ix_trace trace_id TYPE bloom_filter GRANULARITY 1"),
 ];
-/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. */
+/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. Log facts gained their pattern id. */
 export const FACT_ALTERS = [
     "ALTER TABLE fact_search_t ADD INDEX IF NOT EXISTS ix_text_ngrams text TYPE text(tokenizer = ngrams(3)) GRANULARITY 1",
     "ALTER TABLE fact_search_t DROP INDEX IF EXISTS ix_text",
+    "ALTER TABLE fact_logs_t ADD COLUMN IF NOT EXISTS pattern String DEFAULT ''",
+    "ALTER TABLE fact_logs_t ADD INDEX IF NOT EXISTS ix_pattern pattern TYPE bloom_filter GRANULARITY 1",
+    `CREATE OR REPLACE VIEW fact_logs AS SELECT project_id, trace_id, ${LOG_NAMES}, pattern FROM fact_logs_t FINAL`,
 ];
 /** Fact tables (storage names) and the columns that identify a row within a trace, for tombstones. */
 export const FACT_KEYS = {

package/dist/telemetry/ch/schema-facts.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema-facts.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AAAA,2IAA2I;AAC3I,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,CAAC,GAAG,oDAAoD,CAAC;AAC/D,MAAM,SAAS,GAAG,iDAAiD,CAAC;AAEpE,yHAAyH;AACzH,SAAS,IAAI,CAAC,IAAY,EAAE,IAAY,EAAE,KAAa,EAAE,IAAY,EAAE,OAAO,GAAG,EAAE;IACjF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IACvE,OAAO;QACL,8BAA8B,IAAI,OAAO,CAAC,KAAK,IAAI,KAAK,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;iEACpC,OAAO,CAAC,IAAI,CAAC,cAAc,KAAK,KAAK,MAAM,EAAE;QAC1G,0BAA0B,IAAI,oCAAoC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,UAAU;KAC1G,CAAC;AACJ,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,mIAAmI;IACnI,oJAAoJ;IACpJ,+HAA+H;IAC/H,2IAA2I;IAC3I,qHAAqH;CACtH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,WAAW,GAAG;IAClB,kIAAkI;IAClI,iJAAiJ;IACjJ,6IAA6I;IAC7I,mDAAmD;CACpD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,CAAC,MAAM,QAAQ,GAAsB;IACzC,GAAG,IAAI,CAAC,eAAe,EAAE,YAAY,EAAE,gCAAgC,EAAE,UAAU,CAAC;IACpF,GAAG,IAAI,CAAC,aAAa,EAAE,qLAAqL,EAAE,gCAAgC,EAAE,UAAU,CAAC;IAC3P,GAAG,IAAI,CAAC,WAAW,EAAE,wPAAwP,EAAE,oCAAoC,EAAE,SAAS,CAAC;IAC/T,GAAG,IAAI,CAAC,eAAe,EAAE,8JAA8J,EAAE,yCAAyC,EAAE,UAAU,CAAC;IAC/O,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,4CAA4C,EAAE,SAAS,EACzF,2HAA2H,CAAC;IAC9H,GAAG,IAAI,CAAC,YAAY,EAAE,4HAA4H,EAAE,8CAA8C,EAAE,SAAS,EAC3M,yDAAyD,CAAC;CAC7D,CAAC;AAEF,iKAAiK;AACjK,MAAM,CAAC,MAAM,WAAW,GAAsB;IAC5C,sHAAsH;IACtH,wDAAwD;CACzD,CAAC;AAEF,sGAAsG;AACtG,MAAM,CAAC,MAAM,SAAS,GAAsC;IAC1D,eAAe,EAAE,CAAC,UAAU,CAAC;IAC7B,aAAa,EAAE,CAAC,UAAU,CAAC;IAC3B,WAAW,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC;IAC/B,eAAe,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;IACxC,aAAa,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC;IAC3C,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;CAC/C,CAAC"}
+{"version":3,"file":"schema-facts.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AAAA,2IAA2I;AAC3I,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,CAAC,GAAG,oDAAoD,CAAC;AAC/D,MAAM,SAAS,GAAG,iDAAiD,CAAC;AAEpE,yHAAyH;AACzH,SAAS,IAAI,CAAC,IAAY,EAAE,IAAY,EAAE,KAAa,EAAE,IAAY,EAAE,OAAO,GAAG,EAAE;IACjF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IACvE,OAAO;QACL,8BAA8B,IAAI,OAAO,CAAC,KAAK,IAAI,KAAK,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;iEACpC,OAAO,CAAC,IAAI,CAAC,cAAc,KAAK,KAAK,MAAM,EAAE;QAC1G,0BAA0B,IAAI,oCAAoC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,UAAU;KAC1G,CAAC;AACJ,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,mIAAmI;IACnI,oJAAoJ;IACpJ,+HAA+H;IAC/H,2IAA2I;IAC3I,qHAAqH;CACtH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,2EAA2E;AAC3E,MAAM,QAAQ,GAAG,wPAAwP,CAAC;AAC1Q,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEnF,MAAM,WAAW,GAAG;IAClB,kIAAkI;IAClI,iJAAiJ;IACjJ,6IAA6I;IAC7I,mDAAmD;CACpD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,CAAC,MAAM,QAAQ,GAAsB;IACzC,GAAG,IAAI,CAAC,eAAe,EAAE,YAAY,EAAE,gCAAgC,EAAE,UAAU,CAAC;IACpF,GAAG,IAAI,CAAC,aAAa,EAAE,qLAAqL,EAAE,gCAAgC,EAAE,UAAU,CAAC;IAC3P,GAAG,IAAI,CAAC,WAAW,EAAE,QAAQ,EAAE,oCAAoC,EAAE,SAAS,CAAC;IAC/E,GAAG,IAAI,CAAC,eAAe,EAAE,8JAA8J,EAAE,yCAAyC,EAAE,UAAU,CAAC;IAC/O,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,4CAA4C,EAAE,SAAS,EACzF,2HAA2H,CAAC;IAC9H,GAAG,IAAI,CAAC,YAAY,EAAE,4HAA4H,EAAE,8CAA8C,EAAE,SAAS,EAC3M,yDAAyD,CAAC;CAC7D,CAAC;AAEF,oMAAoM;AACpM,MAAM,CAAC,MAAM,WAAW,GAAsB;IAC5C,sHAAsH;IACtH,wDAAwD;IACxD,4EAA4E;IAC5E,oGAAoG;IACpG,oEAAoE,SAAS,kCAAkC;CAChH,CAAC;AAEF,sGAAsG;AACtG,MAAM,CAAC,MAAM,SAAS,GAAsC;IAC1D,eAAe,EAAE,CAAC,UAAU,CAAC;IAC7B,aAAa,EAAE,CAAC,UAAU,CAAC;IAC3B,WAAW,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC;IAC/B,eAAe,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;IACxC,aAAa,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC;IAC3C,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;CAC/C,CAAC"}

package/dist/telemetry/ch/schema-raw.d.ts

@@ -5,4 +5,6 @@ export declare const dayOf: (col: string) => string;
 /** Rows go when their organization's retention runs out, set when they are written. */
 export declare const EXPIRE: string;
 export declare const RAW_TABLES: readonly string[];
+/** Changes to shipped raw tables, in order. Log retention per project: 0 means the trace retention. */
+export declare const RAW_ALTERS: readonly string[];
 //# sourceMappingURL=schema-raw.d.ts.map

package/dist/telemetry/ch/schema-raw.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema-raw.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAEA,mEAAmE;AACnE,eAAO,MAAM,SAAS,QAAS,MAAM,KAAG,MAA2F,CAAC;AACpI,eAAO,MAAM,OAAO,QAAS,MAAM,KAAG,MAAuC,CAAC;AAC9E,eAAO,MAAM,KAAK,QAAS,MAAM,KAAG,MAAyC,CAAC;AAC9E,uFAAuF;AACvF,eAAO,MAAM,MAAM,QAAyC,CAAC;AAE7D,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EA0BvC,CAAC"}
+{"version":3,"file":"schema-raw.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAEA,mEAAmE;AACnE,eAAO,MAAM,SAAS,QAAS,MAAM,KAAG,MAA2F,CAAC;AACpI,eAAO,MAAM,OAAO,QAAS,MAAM,KAAG,MAAuC,CAAC;AAC9E,eAAO,MAAM,KAAK,QAAS,MAAM,KAAG,MAAyC,CAAC;AAC9E,uFAAuF;AACvF,eAAO,MAAM,MAAM,QAAyC,CAAC;AAE7D,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EA0BvC,CAAC;AAEF,uGAAuG;AACvG,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAAyF,CAAC"}

package/dist/telemetry/ch/schema-raw.js

@@ -32,4 +32,6 @@ export const RAW_TABLES = [
     project_id String, days UInt32, ver UInt64
   ) ENGINE = ReplacingMergeTree(ver) ORDER BY project_id`,
 ];
+/** Changes to shipped raw tables, in order. Log retention per project: 0 means the trace retention. */
+export const RAW_ALTERS = ["ALTER TABLE project_retention ADD COLUMN IF NOT EXISTS log_days UInt32 DEFAULT 0"];
 //# sourceMappingURL=schema-raw.js.map

package/dist/telemetry/ch/schema-raw.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema-raw.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAAA,2GAA2G;AAE3G,mEAAmE;AACnE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,6CAA6C,GAAG,iCAAiC,CAAC;AACpI,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,YAAY,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,cAAc,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,uFAAuF;AACvF,MAAM,CAAC,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC;AAE7D,MAAM,CAAC,MAAM,UAAU,GAAsB;IAC3C;;;;oDAIkD,OAAO,CAAC,UAAU,CAAC,6CAA6C,MAAM,EAAE;IAC1H;;oDAEkD,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACrH;;;oEAGkE,MAAM,EAAE;IAC1E;;iDAE+C,SAAS,CAAC,YAAY,CAAC,0BAA0B;IAChG;;uEAEqE,MAAM,EAAE;IAC7E;;;sCAGoC,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACvG;;yDAEuD;CACxD,CAAC"}
+{"version":3,"file":"schema-raw.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAAA,2GAA2G;AAE3G,mEAAmE;AACnE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,6CAA6C,GAAG,iCAAiC,CAAC;AACpI,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,YAAY,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,cAAc,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,uFAAuF;AACvF,MAAM,CAAC,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC;AAE7D,MAAM,CAAC,MAAM,UAAU,GAAsB;IAC3C;;;;oDAIkD,OAAO,CAAC,UAAU,CAAC,6CAA6C,MAAM,EAAE;IAC1H;;oDAEkD,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACrH;;;oEAGkE,MAAM,EAAE;IAC1E;;iDAE+C,SAAS,CAAC,YAAY,CAAC,0BAA0B;IAChG;;uEAEqE,MAAM,EAAE;IAC7E;;;sCAGoC,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACvG;;yDAEuD;CACxD,CAAC;AAEF,uGAAuG;AACvG,MAAM,CAAC,MAAM,UAAU,GAAsB,CAAC,kFAAkF,CAAC,CAAC"}

package/dist/telemetry/ch/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,QAAQ,EAAY,MAAM,WAAW,CAAC;AAapD,qHAAqH;AACrH,eAAO,MAAM,SAAS,EAAE,SAAS,MAAM,EAAmG,CAAC;AAE3I,oFAAoF;AACpF,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EAAgD,CAAC;AAE7F,8GAA8G;AAC9G,eAAO,MAAM,UAAU,QAAS,MAAM,KAAG,MAAmD,CAAC;AAE7F,yHAAyH;AACzH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/G"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,QAAQ,EAAY,MAAM,WAAW,CAAC;AAapD,qHAAqH;AACrH,eAAO,MAAM,SAAS,EAAE,SAAS,MAAM,EAAkH,CAAC;AAE1J,oFAAoF;AACpF,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EAAgD,CAAC;AAE7F,8GAA8G;AAC9G,eAAO,MAAM,UAAU,QAAS,MAAM,KAAG,MAAmD,CAAC;AAE7F,yHAAyH;AACzH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/G"}

package/dist/telemetry/ch/schema.js

@@ -1,7 +1,7 @@
 // ClickHouse schema setup: the database, then every table and view. Statements are idempotent, so each start runs them all.
 import { chClient } from "./http.js";
 import { FACT_ALTERS, FACT_DDL } from "./schema-facts.js";
-import { RAW_TABLES } from "./schema-raw.js";
+import { RAW_ALTERS, RAW_TABLES } from "./schema-raw.js";
 import { ROLLUP_DDL } from "./schema-rollups.js";
 const TABLES = [...RAW_TABLES, ...FACT_DDL, ...ROLLUP_DDL].map((s) => /^CREATE TABLE IF NOT EXISTS (\w+)/.exec(s)?.[1]).filter((t) => !!t);
 /** New parts write no column statistics (see http.ts); old parts lose theirs as they merge. */
@@ -9,7 +9,7 @@ const NO_STATISTICS = TABLES.map((t) => `ALTER TABLE ${t} MODIFY SETTING auto_st
 /** Tables only settling writes, always with a token: a retried insert lands once (see settle-rows.ts, Label). */
 const DEDUPLICATED = TABLES.filter((t) => /^(fact_|rollup_|hist_)|^settle_state$/.test(t)).map((t) => `ALTER TABLE ${t} MODIFY SETTING non_replicated_deduplication_window = 1000`);
 /** Every statement, in order. A change to a shipped table needs an ALTER added here, never an edit to its CREATE. */
-export const CH_SCHEMA = [...RAW_TABLES, ...FACT_DDL, ...FACT_ALTERS, ...ROLLUP_DDL, ...NO_STATISTICS, ...DEDUPLICATED];
+export const CH_SCHEMA = [...RAW_TABLES, ...RAW_ALTERS, ...FACT_DDL, ...FACT_ALTERS, ...ROLLUP_DDL, ...NO_STATISTICS, ...DEDUPLICATED];
 /** Tables whose rows belong to one project, for clearing it. Views are left out. */
 export const PROJECT_TABLES = TABLES.filter((t) => t !== "trace_updates");
 /** Every statement without its TTL clauses: for tests on a fixed clock, whose rows would expire on insert. */

package/dist/telemetry/ch/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AAAA,4HAA4H;AAC5H,OAAO,EAAiB,QAAQ,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAExJ,+FAA+F;AAC/F,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4CAA4C,CAAC,CAAC;AAEtG,iHAAiH;AACjH,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4DAA4D,CAAC,CAAC;AAEpL,qHAAqH;AACrH,MAAM,CAAC,MAAM,SAAS,GAAsB,CAAC,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,EAAE,GAAG,UAAU,EAAE,GAAG,aAAa,EAAE,GAAG,YAAY,CAAC,CAAC;AAE3I,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAsB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC;AAE7F,8GAA8G;AAC9G,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;AAE7F,yHAAyH;AACzH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,MAAgB,EAAE,CAAC,GAAsB,EAAE;IAC9F,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC;IACxB,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mCAAmC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IACxH,KAAK,MAAM,GAAG,IAAI,SAAS;QAAE,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7F,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AAAA,4HAA4H;AAC5H,OAAO,EAAiB,QAAQ,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAExJ,+FAA+F;AAC/F,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4CAA4C,CAAC,CAAC;AAEtG,iHAAiH;AACjH,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4DAA4D,CAAC,CAAC;AAEpL,qHAAqH;AACrH,MAAM,CAAC,MAAM,SAAS,GAAsB,CAAC,GAAG,UAAU,EAAE,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,EAAE,GAAG,UAAU,EAAE,GAAG,aAAa,EAAE,GAAG,YAAY,CAAC,CAAC;AAE1J,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAsB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC;AAE7F,8GAA8G;AAC9G,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;AAE7F,yHAAyH;AACzH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,MAAgB,EAAE,CAAC,GAAsB,EAAE;IAC9F,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC;IACxB,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mCAAmC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IACxH,KAAK,MAAM,GAAG,IAAI,SAAS;QAAE,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7F,CAAC"}

package/dist/telemetry/ch/settle-pool.d.ts

@@ -0,0 +1,24 @@
+import type { ProjectRunner } from "./settle-work.js";
+export interface SettlePoolOptions {
+    /** The ClickHouse URL; each worker opens its own client. */
+    url: string;
+    size: number;
+    /** The worker file. Default: the built `settle-worker.js` next to this file. */
+    entry?: URL;
+    execArgv?: string[];
+}
+export declare class SettlePool {
+    private readonly slots;
+    private readonly o;
+    private lastId;
+    private closed;
+    constructor(o: SettlePoolOptions);
+    get size(): number;
+    private spawn;
+    /** A worker died: its tasks fail (their traces settle again later), and a new worker takes its place. */
+    private lost;
+    /** Settle one project task on the least busy worker. */
+    readonly run: ProjectRunner;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=settle-pool.d.ts.map

package/dist/telemetry/ch/settle-pool.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-pool.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-pool.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAe,MAAM,kBAAkB,CAAC;AAEnE,MAAM,WAAW,iBAAiB;IAChC,4DAA4D;IAC5D,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;IACb,gFAAgF;IAChF,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;CACrB;AAYD,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IACpC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAoB;IACtC,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,MAAM,CAAS;IAEvB,YAAY,CAAC,EAAE,iBAAiB,EAG/B;IAED,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,OAAO,CAAC,KAAK;IAcb,yGAAyG;IACzG,OAAO,CAAC,IAAI;IASZ,wDAAwD;IACxD,QAAQ,CAAC,GAAG,EAAE,aAAa,CAQzB;IAEI,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAG3B;CACF"}

package/dist/telemetry/ch/settle-pool.js

@@ -0,0 +1,58 @@
+// Worker threads for settle: the CPU work of a batch runs off the main thread, so ingest and pages stay fast and settle uses more cores.
+import { Worker } from "node:worker_threads";
+export class SettlePool {
+    slots = [];
+    o;
+    lastId = 0;
+    closed = false;
+    constructor(o) {
+        this.o = o;
+        for (let i = 0; i < o.size; i++)
+            this.slots.push(this.spawn());
+    }
+    get size() {
+        return this.slots.length;
+    }
+    spawn() {
+        const entry = this.o.entry ?? new URL("./settle-worker.js", import.meta.url);
+        const slot = { worker: new Worker(entry, { workerData: { url: this.o.url }, execArgv: this.o.execArgv }), pending: new Map() };
+        slot.worker.on("message", (m) => {
+            const w = slot.pending.get(m.id);
+            slot.pending.delete(m.id);
+            if (m.error === undefined)
+                w?.resolve();
+            else
+                w?.reject(new Error(m.error));
+        });
+        slot.worker.on("error", (err) => this.lost(slot, err));
+        slot.worker.on("exit", (code) => this.lost(slot, new Error(`A settle worker stopped (exit code ${code}).`)));
+        return slot;
+    }
+    /** A worker died: its tasks fail (their traces settle again later), and a new worker takes its place. */
+    lost(slot, err) {
+        for (const w of slot.pending.values())
+            w.reject(err);
+        slot.pending.clear();
+        const i = this.slots.indexOf(slot);
+        if (i < 0 || this.closed)
+            return;
+        console.error("[wizelens-cloud] A settle worker failed and was started again:", err.message);
+        this.slots[i] = this.spawn();
+    }
+    /** Settle one project task on the least busy worker. */
+    run = (task) => {
+        if (this.closed)
+            return Promise.reject(new Error("The settle workers are closed."));
+        const slot = this.slots.reduce((a, b) => (b.pending.size < a.pending.size ? b : a));
+        const id = ++this.lastId;
+        return new Promise((resolve, reject) => {
+            slot.pending.set(id, { resolve, reject });
+            slot.worker.postMessage({ id, task });
+        });
+    };
+    async close() {
+        this.closed = true;
+        await Promise.all(this.slots.map((s) => s.worker.terminate()));
+    }
+}
+//# sourceMappingURL=settle-pool.js.map

package/dist/telemetry/ch/settle-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-pool.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-pool.ts"],"names":[],"mappings":"AAAA,yIAAyI;AACzI,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAsB7C,MAAM,OAAO,UAAU;IACJ,KAAK,GAAW,EAAE,CAAC;IACnB,CAAC,CAAoB;IAC9B,MAAM,GAAG,CAAC,CAAC;IACX,MAAM,GAAG,KAAK,CAAC;IAEvB,YAAY,CAAoB;QAC9B,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAEO,KAAK;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,oBAAoB,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7E,MAAM,IAAI,GAAS,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAC;QACrI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,CAAiC,EAAE,EAAE;YAC9D,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACjC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC1B,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;gBAAE,CAAC,EAAE,OAAO,EAAE,CAAC;;gBACnC,CAAC,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,KAAK,CAAC,sCAAsC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7G,OAAO,IAAI,CAAC;IACd,CAAC;IAED,yGAAyG;IACjG,IAAI,CAAC,IAAU,EAAE,GAAU;QACjC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACrB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACjC,OAAO,CAAC,KAAK,CAAC,gEAAgE,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC;QAC7F,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAED,wDAAwD;IAC/C,GAAG,GAAkB,CAAC,IAAiB,EAAE,EAAE;QAClD,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC,CAAC;QACpF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpF,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC;QACzB,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACjE,CAAC;CACF"}

package/dist/telemetry/ch/settle-rows.d.ts

@@ -1,13 +1,16 @@
 import { type FactTable, type SettleDelta } from "@wizeworks/lens-core";
 import type { ChClient } from "./http.js";
+import { type Keep } from "./util.js";
 type Row = Record<string, unknown>;
-/** Fact rows as objects, with their version and expiry. `expire` maps a trace id to its row expiry base. */
-export declare function factObjects(rows: Record<FactTable, unknown[][]>, ver: number, expireOf: (r: Row) => number): Record<FactTable, Row[]>;
+/** How long a fact row is kept: log rows (lines, their search rows and attributes) by the log retention, the rest by the trace retention. */
+export declare function factKeepOf(table: FactTable, r: Row, keep: Keep): number;
+/** Fact rows as objects, with their version and expiry. */
+export declare function factObjects(rows: Record<FactTable, unknown[][]>, ver: number, expireOf: (r: Row, table: FactTable) => number): Record<FactTable, Row[]>;
 /** Rows the re-settled traces had before and no longer have, as tombstones. Only traces settled before can have any. */
 export declare function tombstones(ch: ChClient, project: string, traceIds: readonly string[], fresh: Record<FactTable, Row[]>, ver: number): Promise<Record<FactTable, Row[]>>;
 /** Settings that make one batch's insert into a table safe to send again: ClickHouse keeps the first copy of a token. */
 export type Label = (table: string) => Record<string, string>;
-/** Write a merged settle delta into every tier. `retentionMs` is per project. */
-export declare function writeDeltas(ch: ChClient, delta: SettleDelta, retentionMs: (project: string) => number, label: Label): Promise<void>;
+/** Write a merged settle delta into every tier. Rows expire by their project's retention; log kinds by its log retention. */
+export declare function writeDeltas(ch: ChClient, delta: SettleDelta, keepOf: (project: string) => Keep, label: Label): Promise<void>;
 export {};
 //# sourceMappingURL=settle-rows.d.ts.map

package/dist/telemetry/ch/settle-rows.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"settle-rows.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AACA,OAAO,EAA6B,KAAK,SAAS,EAAgC,KAAK,WAAW,EAAS,MAAM,sBAAsB,CAAC;AAExI,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAG1C,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAMnC,4GAA4G;AAC5G,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAUrI;AAED,wHAAwH;AACxH,wBAAsB,UAAU,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAa5K;AAoBD,yHAAyH;AACzH,MAAM,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D,iFAAiF;AACjF,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAczI"}
+{"version":3,"file":"settle-rows.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AACA,OAAO,EAA6B,KAAK,SAAS,EAA4E,KAAK,WAAW,EAAS,MAAM,sBAAsB,CAAC;AAEpL,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,KAAK,IAAI,EAAoB,MAAM,WAAW,CAAC;AAExD,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAMnC,6IAA6I;AAC7I,wBAAgB,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,GAAG,MAAM,CAIvE;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,SAAS,KAAK,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAUvJ;AAED,wHAAwH;AACxH,wBAAsB,UAAU,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAa5K;AAoBD,yHAAyH;AACzH,MAAM,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D,6HAA6H;AAC7H,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAelI"}

package/dist/telemetry/ch/settle-rows.js

@@ -1,18 +1,24 @@
 // What one settle batch writes to ClickHouse: fact rows, tombstones for rows a trace no longer has, and rollup deltas.
-import { FACT_COLUMNS, FACT_TABLES, IDS_MAX, NONE_HIGH, NONE_LOW, TIERS } from "@wizeworks/lens-core";
+import { FACT_COLUMNS, FACT_TABLES, IDS_MAX, isLooseTraceId, LOG_KINDS, NONE_HIGH, NONE_LOW, TIERS } from "@wizeworks/lens-core";
 import { FACT_KEYS } from "./schema-facts.js";
-import { rowOf } from "./util.js";
+import { logKeepOf, rowOf } from "./util.js";
 const COUNTERS = ["c0", "c1", "c2", "c3", "c4", "c5"];
 const keyCols = (t) => FACT_KEYS[`${t}_t`];
 const keyOf = (t, r) => [r.trace_id, ...keyCols(t).map((c) => r[c])].map((v) => JSON.stringify(v ?? null)).join("|");
-/** Fact rows as objects, with their version and expiry. `expire` maps a trace id to its row expiry base. */
+/** How long a fact row is kept: log rows (lines, their search rows and attributes) by the log retention, the rest by the trace retention. */
+export function factKeepOf(table, r, keep) {
+    const id = String(r.trace_id);
+    const logRow = table === "fact_logs" || (table === "fact_search" && r.kind === "logs") || (table === "fact_attrs" && r.src === "l");
+    return logRow ? logKeepOf(id, keep) : isLooseTraceId(id) ? keep.logMs : keep.ms;
+}
+/** Fact rows as objects, with their version and expiry. */
 export function factObjects(rows, ver, expireOf) {
     const out = {};
     for (const t of FACT_TABLES) {
         const cols = ["project_id", ...FACT_COLUMNS[t]];
         out[t] = rows[t].map((values) => {
             const r = rowOf(cols, values);
-            return { ...r, ver, is_deleted: 0, expire_ms: expireOf(r) };
+            return { ...r, ver, is_deleted: 0, expire_ms: expireOf(r, t) };
         });
     }
     return out;
@@ -48,17 +54,18 @@ function rollupInsert(table) {
 }
 const HIST_INPUT = "project_id String, env String, kind String, key String, bucket_ms Float64, bin Int32, cnt Float64, expire_ms Float64";
 const histInsert = (table) => `${table}_t SELECT project_id, env, kind, key, bucket_ms, bin, sum(cnt), max(expire_ms) FROM input('${HIST_INPUT}') GROUP BY project_id, env, kind, key, bucket_ms, bin`;
-/** Write a merged settle delta into every tier. `retentionMs` is per project. */
-export async function writeDeltas(ch, delta, retentionMs, label) {
+/** Write a merged settle delta into every tier. Rows expire by their project's retention; log kinds by its log retention. */
+export async function writeDeltas(ch, delta, keepOf, label) {
+    const ms = (project, kind) => (LOG_KINDS.has(kind) ? keepOf(project).logMs : keepOf(project).ms);
     const writes = [];
     for (const tier of TIERS) {
         const rollups = delta.rollup.filter((r) => r.tier === tier.name).map((r) => ({
             project_id: r.project, env: r.env, kind: r.kind, key: r.key, bucket_ms: r.bucket, n: r.n, err: r.err, sum_v: r.sum,
             ...Object.fromEntries(COUNTERS.map((c, i) => [c, r.c[i] ?? 0])), first_ms: r.firstMs ?? NONE_HIGH, last_ms: r.lastMs ?? NONE_LOW,
-            top_rank: r.topRank ?? NONE_LOW, top: r.topRank === null ? null : JSON.stringify(r.top ?? null), ids: r.ids, expire_ms: r.bucket + retentionMs(r.project),
+            top_rank: r.topRank ?? NONE_LOW, top: r.topRank === null ? null : JSON.stringify(r.top ?? null), ids: r.ids, expire_ms: r.bucket + ms(r.project, r.kind),
         }));
         const hists = delta.hist.filter((h) => h.tier === tier.name).map((h) => ({
-            project_id: h.project, env: h.env, kind: h.kind, key: h.key, bucket_ms: h.bucket, bin: h.bin, cnt: h.cnt, expire_ms: h.bucket + retentionMs(h.project),
+            project_id: h.project, env: h.env, kind: h.kind, key: h.key, bucket_ms: h.bucket, bin: h.bin, cnt: h.cnt, expire_ms: h.bucket + ms(h.project, h.kind),
         }));
         writes.push(ch.insert(rollupInsert(tier.table), rollups, label(tier.table)), ch.insert(histInsert(tier.hist), hists, label(tier.hist)));
     }

package/dist/telemetry/ch/settle-rows.js.map

@@ -1 +1 @@
-{"version":3,"file":"settle-rows.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AAAA,uHAAuH;AACvH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAkB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAoB,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACxI,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAIlC,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACtD,MAAM,OAAO,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;AACvD,MAAM,KAAK,GAAG,CAAC,CAAY,EAAE,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAErI,4GAA4G;AAC5G,MAAM,UAAU,WAAW,CAAC,IAAoC,EAAE,GAAW,EAAE,QAA4B;IACzG,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,CAAC,YAAY,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC9B,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC9B,OAAO,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wHAAwH;AACxH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAY,EAAE,OAAe,EAAE,QAA2B,EAAE,KAA+B,EAAE,GAAW;IACvI,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS;QAC/B,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,+EAA+E,CAAC;QAC/H,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC,KAAK,CAAM,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QACvH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,kHAAkH;IAClH,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;IACtC,0GAA0G;CAC3G,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,gGAAgG;AAChG,SAAS,YAAY,CAAC,KAAa;IACjC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,GAAG,GAAG,8BAA8B,OAAO,mFAAmF,CAAC;IACrI,OAAO,GAAG,KAAK,oDAAoD,IAAI,6EAA6E,GAAG;kBACvI,YAAY,mDAAmD,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,GAAG,sHAAsH,CAAC;AAC1I,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE,CACnC,GAAG,KAAK,8FAA8F,UAAU,wDAAwD,CAAC;AAK3K,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,WAAwC,EAAE,KAAY;IACxH,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3E,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG;YAClH,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,IAAI,QAAQ;YAChI,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;SAC1J,CAAC,CAAC,CAAC;QACJ,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvE,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;SACvJ,CAAC,CAAC,CAAC;QACJ,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1I,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}
+{"version":3,"file":"settle-rows.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AAAA,uHAAuH;AACvH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAkB,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAqC,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACpL,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,OAAO,EAAa,SAAS,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAIxD,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACtD,MAAM,OAAO,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;AACvD,MAAM,KAAK,GAAG,CAAC,CAAY,EAAE,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAErI,6IAA6I;AAC7I,MAAM,UAAU,UAAU,CAAC,KAAgB,EAAE,CAAM,EAAE,IAAU;IAC7D,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,KAAK,KAAK,WAAW,IAAI,CAAC,KAAK,KAAK,aAAa,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,KAAK,YAAY,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;IACpI,OAAO,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;AAClF,CAAC;AAED,2DAA2D;AAC3D,MAAM,UAAU,WAAW,CAAC,IAAoC,EAAE,GAAW,EAAE,QAA8C;IAC3H,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,CAAC,YAAY,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC9B,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC9B,OAAO,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACjE,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wHAAwH;AACxH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAY,EAAE,OAAe,EAAE,QAA2B,EAAE,KAA+B,EAAE,GAAW;IACvI,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS;QAC/B,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,+EAA+E,CAAC;QAC/H,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC,KAAK,CAAM,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QACvH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,kHAAkH;IAClH,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;IACtC,0GAA0G;CAC3G,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,gGAAgG;AAChG,SAAS,YAAY,CAAC,KAAa;IACjC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,GAAG,GAAG,8BAA8B,OAAO,mFAAmF,CAAC;IACrI,OAAO,GAAG,KAAK,oDAAoD,IAAI,6EAA6E,GAAG;kBACvI,YAAY,mDAAmD,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,GAAG,sHAAsH,CAAC;AAC1I,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE,CACnC,GAAG,KAAK,8FAA8F,UAAU,wDAAwD,CAAC;AAK3K,6HAA6H;AAC7H,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,MAAiC,EAAE,KAAY;IACjH,MAAM,EAAE,GAAG,CAAC,OAAe,EAAE,IAAgB,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACrH,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3E,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG;YAClH,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,IAAI,QAAQ;YAChI,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC;SACzJ,CAAC,CAAC,CAAC;QACJ,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvE,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC;SACtJ,CAAC,CAAC,CAAC;QACJ,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1I,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}

package/dist/telemetry/ch/settle-work.d.ts

@@ -0,0 +1,21 @@
+import { type FactOptions } from "@wizeworks/lens-core";
+import type { ChClient } from "./http.js";
+import type { Dirty } from "./queue.js";
+/** One project's part of a batch. Plain data, so it can go to a worker thread. */
+export interface ProjectTask {
+    project: string;
+    items: Dirty[];
+    now: number;
+    retentionMs: number;
+    /** How long log rows are kept for this project. */
+    logRetentionMs: number;
+    /** The project's thresholds. */
+    facts?: Partial<FactOptions>;
+    /** The row version, from the main thread, so versions only grow. */
+    ver: number;
+}
+/** Settles one project's part of a batch. Resolves once every row is written. */
+export type ProjectRunner = (task: ProjectTask) => Promise<void>;
+/** Settle one project's part of a batch. */
+export declare function settleProject(ch: ChClient, t: ProjectTask): Promise<void>;
+//# sourceMappingURL=settle-work.d.ts.map

package/dist/telemetry/ch/settle-work.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-work.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-work.ts"],"names":[],"mappings":"AACA,OAAO,EAAW,KAAK,WAAW,EAAmJ,MAAM,sBAAsB,CAAC;AAClN,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAMxC,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,KAAK,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,cAAc,EAAE,MAAM,CAAC;IACvB,gCAAgC;IAChC,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7B,oEAAoE;IACpE,GAAG,EAAE,MAAM,CAAC;CACb;AAED,iFAAiF;AACjF,MAAM,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;AAwDjE,4CAA4C;AAC5C,wBAAsB,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/E"}

package/dist/telemetry/ch/settle-work.js

@@ -0,0 +1,52 @@
+// The heavy part of a settle batch for one project: read its traces, build facts and deltas, and write them. Runs in a worker thread or in the main thread.
+import { compact, factRows, FACT_TABLES, isLooseTraceId, mergeDeltas, settleDelta, storedOf, traceFacts, withoutTop } from "@wizeworks/lens-core";
+import { loadTraces } from "./read.js";
+import { factKeepOf, factObjects, tombstones, writeDeltas } from "./settle-rows.js";
+const hasData = (t) => !!t && t.spanCount + t.logs.length > 0;
+/** Stored contributions of some traces, by trace id. */
+async function storedContribs(ch, project, ids) {
+    const rows = await ch.query("SELECT trace_id, contrib FROM settle_state FINAL WHERE project_id = {p1:String} AND has({p2:Array(String)}, trace_id)", [project, ids]);
+    return new Map(rows.map((r) => [r.trace_id, r.contrib]));
+}
+/** Facts, deltas, and new settle states for one project's dirty traces. */
+async function projectWork(ch, t) {
+    const { project, items, now, retentionMs, logRetentionMs, ver } = t;
+    const keep = { ms: retentionMs, logMs: logRetentionMs };
+    const ids = items.map((d) => d.traceId);
+    const [traces, stored] = await Promise.all([loadTraces(ch, [project], ids), storedContribs(ch, project, ids)]);
+    const raw = Object.fromEntries(FACT_TABLES.map((t) => [t, []]));
+    const work = { facts: {}, deltas: [], states: [], resettled: [] };
+    items.forEach((d, i) => {
+        const trace = traces[i];
+        const scope = { project, env: d.env };
+        const facts = hasData(trace) ? traceFacts(trace, t.facts) : null;
+        const next = facts ? compact(facts, scope) : null;
+        const old = stored.get(d.traceId);
+        if (old !== undefined)
+            work.resettled.push(d.traceId);
+        work.deltas.push(settleDelta(storedOf(old || null), next, now, retentionMs, logRetentionMs));
+        if (facts)
+            for (const [t, rows] of Object.entries(factRows(facts, scope, trace)))
+                raw[t].push(...rows);
+        const start = trace?.startMs ?? now;
+        const stateKeep = isLooseTraceId(d.traceId) ? logRetentionMs : retentionMs;
+        work.states.push({ project_id: project, trace_id: d.traceId, settled_ms: d.updatedMs, contrib: next ? JSON.stringify(withoutTop(next)) : "", ver, expire_ms: start + stateKeep });
+    });
+    const timeOf = (r) => Number(r.start_ms ?? r.time_ms ?? now);
+    work.facts = factObjects(raw, ver, (r, table) => timeOf(r) + factKeepOf(table, r, keep));
+    return work;
+}
+/** Write one project's work: facts (with tombstones) and rollups at once, then the settle states, so a failed write settles again. */
+async function writeWork(ch, t, w) {
+    // The same token on a retry lets ClickHouse keep one copy.
+    const label = (table) => ({ insert_deduplication_token: `${t.ver}:${t.project}:${table}` });
+    const dead = await tombstones(ch, t.project, w.resettled, w.facts, t.ver);
+    const facts = FACT_TABLES.map((f) => ch.insert(`${f}_t`, [...dead[f], ...w.facts[f]], label(f)));
+    await Promise.all([...facts, writeDeltas(ch, mergeDeltas(w.deltas), () => ({ ms: t.retentionMs, logMs: t.logRetentionMs }), label)]);
+    await ch.insert("settle_state", w.states, label("settle_state"));
+}
+/** Settle one project's part of a batch. */
+export async function settleProject(ch, t) {
+    await writeWork(ch, t, await projectWork(ch, t));
+}
+//# sourceMappingURL=settle-work.js.map

package/dist/telemetry/ch/settle-work.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-work.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-work.ts"],"names":[],"mappings":"AAAA,4JAA4J;AAC5J,OAAO,EAAE,OAAO,EAAoB,QAAQ,EAAE,WAAW,EAAkB,cAAc,EAAE,WAAW,EAAoB,WAAW,EAAE,QAAQ,EAAc,UAAU,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGlN,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAc,UAAU,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAqBhG,MAAM,OAAO,GAAG,CAAC,CAAoB,EAAc,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AAE7F,wDAAwD;AACxD,KAAK,UAAU,cAAc,CAAC,EAAY,EAAE,OAAe,EAAE,GAAsB;IACjF,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CACzB,uHAAuH,EACvH,CAAC,OAAO,EAAE,GAAG,CAAC,CACf,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AASD,2EAA2E;AAC3E,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,CAAc;IACrD,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,cAAc,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IACpE,MAAM,IAAI,GAAG,EAAE,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC;IACxD,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAiB,CAAC,CAAC,CAAmC,CAAC;IACjH,MAAM,IAAI,GAAgB,EAAE,KAAK,EAAE,EAA8B,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC3G,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACjE,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,EAAE,cAAc,CAAC,CAAC,CAAC;QAC7F,IAAI,KAAK;YAAE,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAA+B;gBAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QACrI,MAAM,KAAK,GAAG,KAAK,EAAE,OAAO,IAAI,GAAG,CAAC;QACpC,MAAM,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3E,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,GAAG,SAAS,EAAE,CAAC,CAAC;IACpL,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,CAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACzF,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sIAAsI;AACtI,KAAK,UAAU,SAAS,CAAC,EAAY,EAAE,CAAc,EAAE,CAAc;IACnE,2DAA2D;IAC3D,MAAM,KAAK,GAAU,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,0BAA0B,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;IACnG,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,CAAC,cAAc,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACrI,MAAM,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,4CAA4C;AAC5C,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,EAAY,EAAE,CAAc;IAC9D,MAAM,SAAS,CAAC,EAAE,EAAE,CAAC,EAAE,MAAM,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;AACnD,CAAC"}

package/dist/telemetry/ch/settle-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=settle-worker.d.ts.map

package/dist/telemetry/ch/settle-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-worker.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-worker.ts"],"names":[],"mappings":""}

package/dist/telemetry/ch/settle-worker.js

@@ -0,0 +1,10 @@
+// A settle worker thread: settles the project tasks the main thread sends, on its own ClickHouse client.
+import { parentPort, workerData } from "node:worker_threads";
+import { chClient } from "./http.js";
+import { settleProject } from "./settle-work.js";
+const port = parentPort;
+const ch = chClient({ url: workerData.url });
+port.on("message", (job) => {
+    settleProject(ch, job.task).then(() => port.postMessage({ id: job.id }), (err) => port.postMessage({ id: job.id, error: err instanceof Error ? err.message : String(err) }));
+});
+//# sourceMappingURL=settle-worker.js.map

package/dist/telemetry/ch/settle-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-worker.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-worker.ts"],"names":[],"mappings":"AAAA,yGAAyG;AACzG,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAoB,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAOnE,MAAM,IAAI,GAAG,UAAW,CAAC;AACzB,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAG,UAA8B,CAAC,GAAG,EAAE,CAAC,CAAC;AAElE,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAQ,EAAE,EAAE;IAC9B,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,CAC9B,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,EACtC,CAAC,GAAY,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAC5G,CAAC;AACJ,CAAC,CAAC,CAAC"}

package/dist/telemetry/ch/settle.d.ts

@@ -1,7 +1,21 @@
+import { type ProjectSettings } from "@wizeworks/lens-core";
 import type { SettleOptions } from "../store.js";
-import type { ChClient } from "./http.js";
 import type { SettleQueue } from "./queue.js";
+import type { ProjectRunner } from "./settle-work.js";
 import { type RetentionCache } from "./util.js";
-/** Settle due traces until none are left or the budget is spent. Batches overlap (`DEPTH` at once): one reads and computes while others wait on their writes. */
-export declare function settleClickHouse(ch: ChClient, queue: SettleQueue, retention: RetentionCache, now: number, o?: SettleOptions): Promise<number>;
+export interface Settler {
+    queue: SettleQueue;
+    retention: RetentionCache;
+    logRetention: RetentionCache;
+    /** A project's own settings, for its thresholds. */
+    settingsOf?: (projectId: string) => Promise<ProjectSettings>;
+    /** Settles one project task: in a worker thread, or in this thread. */
+    run: ProjectRunner;
+    /** Batches at once: one reads and computes while others wait on their writes. */
+    depth: number;
+    /** Queues the next page of a stats rebuild, when one runs. */
+    refill?: () => Promise<void>;
+}
+/** Settle due traces until none are left or the budget is spent, with up to `depth` batches at once. */
+export declare function settleClickHouse(s: Settler, now: number, o?: SettleOptions): Promise<number>;
 //# sourceMappingURL=settle.d.ts.map

package/dist/telemetry/ch/settle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"settle.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAS,WAAW,EAAE,MAAM,YAAY,CAAC;AAGrD,OAAO,EAAsB,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AA6EpE,iKAAiK;AACjK,wBAAsB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,GAAE,aAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAuBvJ"}
+{"version":3,"file":"settle.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle.ts"],"names":[],"mappings":"AACA,OAAO,EAAiB,KAAK,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAS,WAAW,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAsB,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAKpE,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,WAAW,CAAC;IACnB,SAAS,EAAE,cAAc,CAAC;IAC1B,YAAY,EAAE,cAAc,CAAC;IAC7B,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,eAAe,CAAC,CAAC;IAC7D,uEAAuE;IACvE,GAAG,EAAE,aAAa,CAAC;IACnB,iFAAiF;IACjF,KAAK,EAAE,MAAM,CAAC;IACd,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9B;AAqBD,wGAAwG;AACxG,wBAAsB,gBAAgB,CAAC,CAAC,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,GAAE,aAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAwBtG"}