@wizeworks/lens-cloud 0.3.0-alpha.2 → 0.3.0-alpha.3

package/dist/telemetry/ch/schema-facts.js

@@ -0,0 +1,49 @@
+// ClickHouse fact tables (one row per request, trace, log line, slow span, search document, attribute), and views with the Postgres names.
+import { EXPIRE, monthOf } from "./schema-raw.js";
+const P = "project_id LowCardinality(String), trace_id String";
+const VERSIONED = "ver UInt64, is_deleted UInt8, expire_ms Float64";
+/** A fact table: replaced by version, deleted by tombstone; the view reads only live rows, with the Postgres columns. */
+function fact(name, cols, order, time, indexes = "") {
+    const names = cols.split(/,\s*/).map((c) => c.trim().split(/\s+/)[0]);
+    return [
+        `CREATE TABLE IF NOT EXISTS ${name}_t (${P}, ${cols}, ${VERSIONED}${indexes ? `, ${indexes}` : ""})
+     ENGINE = ReplacingMergeTree(ver, is_deleted) PARTITION BY ${monthOf(time)} ORDER BY (${order}) ${EXPIRE}`,
+        `CREATE OR REPLACE VIEW ${name} AS SELECT project_id, trace_id, ${names.join(", ")} FROM ${name}_t FINAL`,
+    ];
+}
+const REQUEST_COLS = [
+    "env LowCardinality(String), start_ms Float64, method LowCardinality(String), path String, endpoint String, status Nullable(Int32)",
+    "status_class LowCardinality(String), failed Bool, dur_class LowCardinality(String), aborted Bool, duration_ms Float64, server_ms Nullable(Float64)",
+    "db_ms Float64, db_count Int32, fetch_count Int32, fetch_ms Float64, render_ms Nullable(Float64), network_ms Nullable(Float64)",
+    "service LowCardinality(String), services String, errors Int32, insights Int32, health LowCardinality(String), user_agent Nullable(String)",
+    "client_address Nullable(String), end_user Nullable(String), request_id Nullable(String), tags String, search String",
+].join(", ");
+const SEARCH_COLS = [
+    "kind LowCardinality(String), owner String, env LowCardinality(String), time_ms Float64, span_id Nullable(String), service String",
+    "name Nullable(String), method Nullable(String), path Nullable(String), route Nullable(String), status Nullable(Int32), outcome Nullable(String)",
+    "duration_ms Nullable(Float64), level_rank Nullable(Int16), category Nullable(String), span_kind Nullable(String), end_user Nullable(String)",
+    "error Bool, message Nullable(String), text String",
+].join(", ");
+export const FACT_DDL = [
+    ...fact("fact_requests", REQUEST_COLS, "project_id, start_ms, trace_id", "start_ms"),
+    ...fact("fact_traces", "env LowCardinality(String), start_ms Float64, path String, duration_ms Float64, health Nullable(String), status String, services String, span_count Int32, slow Bool, search String", "project_id, start_ms, trace_id", "start_ms"),
+    ...fact("fact_logs", "idx Int32, env LowCardinality(String), time_ms Float64, level LowCardinality(String), level_rank Int16, service LowCardinality(String), category String, message String, trace_name String, span_id Nullable(String), attributes String, search String", "project_id, time_ms, trace_id, idx", "time_ms"),
+    ...fact("fact_critical", "span_id String, env LowCardinality(String), start_ms Float64, duration_ms Float64, name String, service LowCardinality(String), layer LowCardinality(String)", "project_id, start_ms, trace_id, span_id", "start_ms"),
+    ...fact("fact_search", SEARCH_COLS, "project_id, kind, time_ms, trace_id, owner", "time_ms", "INDEX ix_text text TYPE ngrambf_v1(3, 65536, 2, 0) GRANULARITY 1, INDEX ix_trace trace_id TYPE bloom_filter GRANULARITY 1"),
+    ...fact("fact_attrs", "src LowCardinality(String), owner String, key LowCardinality(String), value String, num Nullable(Float64), time_ms Float64", "project_id, key, value, trace_id, src, owner", "time_ms", "INDEX ix_trace trace_id TYPE bloom_filter GRANULARITY 1"),
+];
+/** Changes to shipped fact tables, in order. The n-gram bloom filled up at 8,192 rows a block and skipped nothing; the text index lists each block's n-grams. */
+export const FACT_ALTERS = [
+    "ALTER TABLE fact_search_t ADD INDEX IF NOT EXISTS ix_text_ngrams text TYPE text(tokenizer = ngrams(3)) GRANULARITY 1",
+    "ALTER TABLE fact_search_t DROP INDEX IF EXISTS ix_text",
+];
+/** Fact tables (storage names) and the columns that identify a row within a trace, for tombstones. */
+export const FACT_KEYS = {
+    fact_requests_t: ["start_ms"],
+    fact_traces_t: ["start_ms"],
+    fact_logs_t: ["time_ms", "idx"],
+    fact_critical_t: ["start_ms", "span_id"],
+    fact_search_t: ["kind", "time_ms", "owner"],
+    fact_attrs_t: ["key", "value", "src", "owner"],
+};
+//# sourceMappingURL=schema-facts.js.map

package/dist/telemetry/ch/schema-facts.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-facts.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-facts.ts"],"names":[],"mappings":"AAAA,2IAA2I;AAC3I,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,CAAC,GAAG,oDAAoD,CAAC;AAC/D,MAAM,SAAS,GAAG,iDAAiD,CAAC;AAEpE,yHAAyH;AACzH,SAAS,IAAI,CAAC,IAAY,EAAE,IAAY,EAAE,KAAa,EAAE,IAAY,EAAE,OAAO,GAAG,EAAE;IACjF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;IACvE,OAAO;QACL,8BAA8B,IAAI,OAAO,CAAC,KAAK,IAAI,KAAK,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,KAAK,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE;iEACpC,OAAO,CAAC,IAAI,CAAC,cAAc,KAAK,KAAK,MAAM,EAAE;QAC1G,0BAA0B,IAAI,oCAAoC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,UAAU;KAC1G,CAAC;AACJ,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,mIAAmI;IACnI,oJAAoJ;IACpJ,+HAA+H;IAC/H,2IAA2I;IAC3I,qHAAqH;CACtH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,WAAW,GAAG;IAClB,kIAAkI;IAClI,iJAAiJ;IACjJ,6IAA6I;IAC7I,mDAAmD;CACpD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,MAAM,CAAC,MAAM,QAAQ,GAAsB;IACzC,GAAG,IAAI,CAAC,eAAe,EAAE,YAAY,EAAE,gCAAgC,EAAE,UAAU,CAAC;IACpF,GAAG,IAAI,CAAC,aAAa,EAAE,qLAAqL,EAAE,gCAAgC,EAAE,UAAU,CAAC;IAC3P,GAAG,IAAI,CAAC,WAAW,EAAE,wPAAwP,EAAE,oCAAoC,EAAE,SAAS,CAAC;IAC/T,GAAG,IAAI,CAAC,eAAe,EAAE,8JAA8J,EAAE,yCAAyC,EAAE,UAAU,CAAC;IAC/O,GAAG,IAAI,CAAC,aAAa,EAAE,WAAW,EAAE,4CAA4C,EAAE,SAAS,EACzF,2HAA2H,CAAC;IAC9H,GAAG,IAAI,CAAC,YAAY,EAAE,4HAA4H,EAAE,8CAA8C,EAAE,SAAS,EAC3M,yDAAyD,CAAC;CAC7D,CAAC;AAEF,iKAAiK;AACjK,MAAM,CAAC,MAAM,WAAW,GAAsB;IAC5C,sHAAsH;IACtH,wDAAwD;CACzD,CAAC;AAEF,sGAAsG;AACtG,MAAM,CAAC,MAAM,SAAS,GAAsC;IAC1D,eAAe,EAAE,CAAC,UAAU,CAAC;IAC7B,aAAa,EAAE,CAAC,UAAU,CAAC;IAC3B,WAAW,EAAE,CAAC,SAAS,EAAE,KAAK,CAAC;IAC/B,eAAe,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;IACxC,aAAa,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC;IAC3C,YAAY,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC;CAC/C,CAAC"}

package/dist/telemetry/ch/schema-raw.d.ts

@@ -0,0 +1,8 @@
+/** Epoch ms in a column as a DateTime, for partitions and TTLs. */
+export declare const secondsOf: (col: string) => string;
+export declare const monthOf: (col: string) => string;
+export declare const dayOf: (col: string) => string;
+/** Rows go when their organization's retention runs out, set when they are written. */
+export declare const EXPIRE: string;
+export declare const RAW_TABLES: readonly string[];
+//# sourceMappingURL=schema-raw.d.ts.map

package/dist/telemetry/ch/schema-raw.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-raw.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAEA,mEAAmE;AACnE,eAAO,MAAM,SAAS,QAAS,MAAM,KAAG,MAA2F,CAAC;AACpI,eAAO,MAAM,OAAO,QAAS,MAAM,KAAG,MAAuC,CAAC;AAC9E,eAAO,MAAM,KAAK,QAAS,MAAM,KAAG,MAAyC,CAAC;AAC9E,uFAAuF;AACvF,eAAO,MAAM,MAAM,QAAyC,CAAC;AAE7D,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EA0BvC,CAAC"}

package/dist/telemetry/ch/schema-raw.js

@@ -0,0 +1,35 @@
+// ClickHouse tables for what add-ons send: spans, logs, trace bounds, metric points, and the settle state.
+/** Epoch ms in a column as a DateTime, for partitions and TTLs. */
+export const secondsOf = (col) => `toDateTime(intDiv(toUInt64(least(greatest(${col}, 0.), 4102444800000.)), 1000))`;
+export const monthOf = (col) => `toYYYYMM(${secondsOf(col)})`;
+export const dayOf = (col) => `toYYYYMMDD(${secondsOf(col)})`;
+/** Rows go when their organization's retention runs out, set when they are written. */
+export const EXPIRE = `TTL ${secondsOf("expire_ms")} DELETE`;
+export const RAW_TABLES = [
+    `CREATE TABLE IF NOT EXISTS spans_raw (
+    project_id LowCardinality(String), trace_id String, span_id String, service LowCardinality(String), start_ms Float64, duration_ms Float64,
+    session Nullable(String), json String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64,
+    INDEX ix_session session TYPE bloom_filter GRANULARITY 4
+  ) ENGINE = ReplacingMergeTree(ver) PARTITION BY ${monthOf("start_ms")} ORDER BY (project_id, trace_id, span_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS logs_raw (
+    project_id LowCardinality(String), trace_id String, key String, time_ms Float64, json String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) PARTITION BY ${monthOf("time_ms")} ORDER BY (project_id, trace_id, key) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS trace_heads (
+    project_id LowCardinality(String), trace_id String, start_ms SimpleAggregateFunction(min, Float64), end_ms SimpleAggregateFunction(max, Float64),
+    updated_ms SimpleAggregateFunction(max, Float64), env SimpleAggregateFunction(max, String), expire_ms SimpleAggregateFunction(max, Float64)
+  ) ENGINE = AggregatingMergeTree ORDER BY (project_id, trace_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS trace_updates (
+    project_id LowCardinality(String), trace_id String, updated_ms Float64, env String
+  ) ENGINE = MergeTree ORDER BY updated_ms TTL ${secondsOf("updated_ms")} + INTERVAL 2 DAY DELETE`,
+    `CREATE TABLE IF NOT EXISTS settle_state (
+    project_id LowCardinality(String), trace_id String, settled_ms Float64, contrib String CODEC(ZSTD(3)), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY (project_id, trace_id) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS metric_points_raw (
+    project_id LowCardinality(String), time_ms Float64, env LowCardinality(String), service LowCardinality(String), name LowCardinality(String),
+    series_key String, json String CODEC(ZSTD(3)), expire_ms Float64
+  ) ENGINE = MergeTree PARTITION BY ${monthOf("time_ms")} ORDER BY (project_id, name, time_ms) ${EXPIRE}`,
+    `CREATE TABLE IF NOT EXISTS project_retention (
+    project_id String, days UInt32, ver UInt64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY project_id`,
+];
+//# sourceMappingURL=schema-raw.js.map

package/dist/telemetry/ch/schema-raw.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-raw.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-raw.ts"],"names":[],"mappings":"AAAA,2GAA2G;AAE3G,mEAAmE;AACnE,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,6CAA6C,GAAG,iCAAiC,CAAC;AACpI,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,YAAY,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,cAAc,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC;AAC9E,uFAAuF;AACvF,MAAM,CAAC,MAAM,MAAM,GAAG,OAAO,SAAS,CAAC,WAAW,CAAC,SAAS,CAAC;AAE7D,MAAM,CAAC,MAAM,UAAU,GAAsB;IAC3C;;;;oDAIkD,OAAO,CAAC,UAAU,CAAC,6CAA6C,MAAM,EAAE;IAC1H;;oDAEkD,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACrH;;;oEAGkE,MAAM,EAAE;IAC1E;;iDAE+C,SAAS,CAAC,YAAY,CAAC,0BAA0B;IAChG;;uEAEqE,MAAM,EAAE;IAC7E;;;sCAGoC,OAAO,CAAC,SAAS,CAAC,yCAAyC,MAAM,EAAE;IACvG;;yDAEuD;CACxD,CAAC"}

package/dist/telemetry/ch/schema-rollups.d.ts

@@ -0,0 +1,2 @@
+export declare const ROLLUP_DDL: readonly string[];
+//# sourceMappingURL=schema-rollups.d.ts.map

package/dist/telemetry/ch/schema-rollups.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-rollups.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-rollups.ts"],"names":[],"mappings":"AAiEA,eAAO,MAAM,UAAU,EAAE,SAAS,MAAM,EAA8E,CAAC"}

package/dist/telemetry/ch/schema-rollups.js

@@ -0,0 +1,61 @@
+// ClickHouse rollup, histogram, and metric tiers: rows add up when they merge, and views read them merged with the Postgres names.
+import { IDS_MAX, TIERS } from "@wizeworks/lens-core";
+import { dayOf, EXPIRE, monthOf, secondsOf } from "./schema-raw.js";
+const sum = (c) => `${c} SimpleAggregateFunction(sum, Float64)`;
+const COUNTERS = ["c0", "c1", "c2", "c3", "c4", "c5"];
+/** Short tiers age out by bucket; the 1 h tier by retention. Day partitions for short tiers, month for 1 h. */
+function tierTail(tier) {
+    const ttl = tier.keepMs === null ? EXPIRE : `TTL ${secondsOf("bucket_ms")} + INTERVAL ${Math.ceil(tier.keepMs / 1000)} SECOND DELETE`;
+    return `PARTITION BY ${tier.keepMs === null ? monthOf("bucket_ms") : dayOf("bucket_ms")} ${ttl}`;
+}
+function rollupDdl(tier) {
+    const t = tier.table;
+    const cols = [
+        "project_id LowCardinality(String), env LowCardinality(String), kind LowCardinality(String), key String, bucket_ms Float64",
+        ...["n", "err", "sum_v", ...COUNTERS].map(sum),
+        "first_ms SimpleAggregateFunction(min, Float64), last_ms SimpleAggregateFunction(max, Float64), top_rank SimpleAggregateFunction(max, Float64)",
+        `top AggregateFunction(argMax, Nullable(String), Float64), ids AggregateFunction(groupArraySorted(${IDS_MAX}), Tuple(Float64, String))`,
+        "expire_ms SimpleAggregateFunction(max, Float64)",
+    ].join(", ");
+    const read = ["project_id", "env", "kind", "key", "bucket_ms", "n", "err", "sum_v", ...COUNTERS, "first_ms", "last_ms", "top_rank"].join(", ");
+    const finals = "finalizeAggregation(top) AS top, arrayStringConcat(arrayMap(x -> x.2, finalizeAggregation(ids)), ' ') AS ids";
+    return [
+        `CREATE TABLE IF NOT EXISTS ${t}_t (${cols}) ENGINE = AggregatingMergeTree ORDER BY (project_id, kind, bucket_ms, env, key) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${t} AS SELECT ${read}, ${finals} FROM ${t}_t FINAL`,
+        `CREATE TABLE IF NOT EXISTS ${tier.hist}_t (
+      project_id LowCardinality(String), env LowCardinality(String), kind LowCardinality(String), key String, bucket_ms Float64, bin Int32,
+      cnt SimpleAggregateFunction(sum, Float64), expire_ms SimpleAggregateFunction(max, Float64)
+    ) ENGINE = AggregatingMergeTree ORDER BY (project_id, kind, bucket_ms, env, key, bin) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${tier.hist} AS SELECT project_id, env, kind, key, bucket_ms, bin, cnt FROM ${tier.hist}_t FINAL`,
+    ];
+}
+function metricDdl(tier) {
+    const t = `metric_${tier.name}`;
+    const nmin = (c) => `${c} SimpleAggregateFunction(min, Nullable(Float64))`;
+    const nmax = (c) => `${c} SimpleAggregateFunction(max, Nullable(Float64))`;
+    const cols = [
+        "project_id LowCardinality(String), env LowCardinality(String), series_key String, name LowCardinality(String), bucket_ms Float64",
+        sum("cnt"), sum("vsum"), nmin("vmin"), nmax("vmax"), "first_ms SimpleAggregateFunction(min, Float64)",
+        "first_v AggregateFunction(argMin, Nullable(Float64), Float64)", "last_ms SimpleAggregateFunction(max, Float64)",
+        "last_v AggregateFunction(argMax, Nullable(Float64), Float64)", sum("rate_delta"), sum("rate_ms"), sum("h_count"), sum("h_sum"), nmin("h_min"), nmax("h_max"),
+        "h_bounds AggregateFunction(argMax, Nullable(String), UInt64), h_counts AggregateFunction(argMax, Nullable(String), UInt64)",
+        "expire_ms SimpleAggregateFunction(max, Float64)",
+    ].join(", ");
+    const read = "project_id, env, series_key, name, bucket_ms, cnt, vsum, vmin, vmax, first_ms, last_ms, rate_delta, rate_ms, h_count, h_sum, h_min, h_max";
+    const finals = ["first_v", "last_v", "h_bounds", "h_counts"].map((c) => `finalizeAggregation(${c}) AS ${c}`).join(", ");
+    return [
+        `CREATE TABLE IF NOT EXISTS ${t}_t (${cols}) ENGINE = AggregatingMergeTree ORDER BY (project_id, name, bucket_ms, env, series_key) ${tierTail(tier)}`,
+        `CREATE OR REPLACE VIEW ${t} AS SELECT ${read}, ${finals} FROM ${t}_t FINAL`,
+    ];
+}
+const SERIES_DDL = [
+    `CREATE TABLE IF NOT EXISTS metric_series_t (
+    project_id LowCardinality(String), env LowCardinality(String), series_key String, name LowCardinality(String), service LowCardinality(String),
+    kind LowCardinality(String), unit String, description String, monotonic Bool, temporality String, attributes String, last_ms Float64,
+    last_value Nullable(Float64), last_hist Nullable(String), ver UInt64, expire_ms Float64
+  ) ENGINE = ReplacingMergeTree(ver) ORDER BY (project_id, env, series_key) ${EXPIRE}`,
+    `CREATE OR REPLACE VIEW metric_series AS SELECT project_id, env, series_key, name, service, kind, unit, description, monotonic, temporality,
+    attributes, last_ms, last_value, last_hist FROM metric_series_t FINAL`,
+];
+export const ROLLUP_DDL = [...TIERS.flatMap(rollupDdl), ...TIERS.flatMap(metricDdl), ...SERIES_DDL];
+//# sourceMappingURL=schema-rollups.js.map

package/dist/telemetry/ch/schema-rollups.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-rollups.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema-rollups.ts"],"names":[],"mappings":"AAAA,mIAAmI;AACnI,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAEpE,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,wCAAwC,CAAC;AACxE,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AAEtD,+GAA+G;AAC/G,SAAS,QAAQ,CAAC,IAA4B;IAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,SAAS,CAAC,WAAW,CAAC,eAAe,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACtI,OAAO,gBAAgB,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,CAAC,IAAI,GAAG,EAAE,CAAC;AACnG,CAAC;AAED,SAAS,SAAS,CAAC,IAA4B;IAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;IACrB,MAAM,IAAI,GAAG;QACX,2HAA2H;QAC3H,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC;QAC9C,+IAA+I;QAC/I,oGAAoG,OAAO,4BAA4B;QACvI,iDAAiD;KAClD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,IAAI,GAAG,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/I,MAAM,MAAM,GAAG,8GAA8G,CAAC;IAC9H,OAAO;QACL,8BAA8B,CAAC,OAAO,IAAI,oFAAoF,QAAQ,CAAC,IAAI,CAAC,EAAE;QAC9I,0BAA0B,CAAC,cAAc,IAAI,KAAK,MAAM,SAAS,CAAC,UAAU;QAC5E,8BAA8B,IAAI,CAAC,IAAI;;;4FAGiD,QAAQ,CAAC,IAAI,CAAC,EAAE;QACxG,0BAA0B,IAAI,CAAC,IAAI,mEAAmE,IAAI,CAAC,IAAI,UAAU;KAC1H,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,IAA4B;IAC7C,MAAM,CAAC,GAAG,UAAU,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,kDAAkD,CAAC;IACnF,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,kDAAkD,CAAC;IACnF,MAAM,IAAI,GAAG;QACX,kIAAkI;QAClI,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,gDAAgD;QACrG,+DAA+D,EAAE,+CAA+C;QAChH,8DAA8D,EAAE,GAAG,CAAC,YAAY,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC;QAC7J,4HAA4H;QAC5H,iDAAiD;KAClD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,MAAM,IAAI,GAAG,2IAA2I,CAAC;IACzJ,MAAM,MAAM,GAAG,CAAC,SAAS,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACxH,OAAO;QACL,8BAA8B,CAAC,OAAO,IAAI,2FAA2F,QAAQ,CAAC,IAAI,CAAC,EAAE;QACrJ,0BAA0B,CAAC,cAAc,IAAI,KAAK,MAAM,SAAS,CAAC,UAAU;KAC7E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,GAAG;IACjB;;;;8EAI4E,MAAM,EAAE;IACpF;0EACwE;CACzE,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAsB,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC"}

package/dist/telemetry/ch/schema.d.ts

@@ -0,0 +1,12 @@
+import { type ChClient } from "./http.js";
+/** Every statement, in order. A change to a shipped table needs an ALTER added here, never an edit to its CREATE. */
+export declare const CH_SCHEMA: readonly string[];
+/** Tables whose rows belong to one project, for clearing it. Views are left out. */
+export declare const PROJECT_TABLES: readonly string[];
+/** Every statement without its TTL clauses: for tests on a fixed clock, whose rows would expire on insert. */
+export declare const withoutTtl: (sql: string) => string;
+/** Create the database (through the default one) and every table and view. `ttl: false` keeps every row (tests only). */
+export declare function migrateClickHouse(url: string, client: ChClient, o?: {
+    ttl?: boolean;
+}): Promise<void>;
+//# sourceMappingURL=schema.d.ts.map

package/dist/telemetry/ch/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,QAAQ,EAAY,MAAM,WAAW,CAAC;AAapD,qHAAqH;AACrH,eAAO,MAAM,SAAS,EAAE,SAAS,MAAM,EAAmG,CAAC;AAE3I,oFAAoF;AACpF,eAAO,MAAM,cAAc,EAAE,SAAS,MAAM,EAAgD,CAAC;AAE7F,8GAA8G;AAC9G,eAAO,MAAM,UAAU,QAAS,MAAM,KAAG,MAAmD,CAAC;AAE7F,yHAAyH;AACzH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAE;IAAE,GAAG,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAK/G"}

package/dist/telemetry/ch/schema.js

@@ -0,0 +1,25 @@
+// ClickHouse schema setup: the database, then every table and view. Statements are idempotent, so each start runs them all.
+import { chClient } from "./http.js";
+import { FACT_ALTERS, FACT_DDL } from "./schema-facts.js";
+import { RAW_TABLES } from "./schema-raw.js";
+import { ROLLUP_DDL } from "./schema-rollups.js";
+const TABLES = [...RAW_TABLES, ...FACT_DDL, ...ROLLUP_DDL].map((s) => /^CREATE TABLE IF NOT EXISTS (\w+)/.exec(s)?.[1]).filter((t) => !!t);
+/** New parts write no column statistics (see http.ts); old parts lose theirs as they merge. */
+const NO_STATISTICS = TABLES.map((t) => `ALTER TABLE ${t} MODIFY SETTING auto_statistics_types = ''`);
+/** Tables only settling writes, always with a token: a retried insert lands once (see settle-rows.ts, Label). */
+const DEDUPLICATED = TABLES.filter((t) => /^(fact_|rollup_|hist_)|^settle_state$/.test(t)).map((t) => `ALTER TABLE ${t} MODIFY SETTING non_replicated_deduplication_window = 1000`);
+/** Every statement, in order. A change to a shipped table needs an ALTER added here, never an edit to its CREATE. */
+export const CH_SCHEMA = [...RAW_TABLES, ...FACT_DDL, ...FACT_ALTERS, ...ROLLUP_DDL, ...NO_STATISTICS, ...DEDUPLICATED];
+/** Tables whose rows belong to one project, for clearing it. Views are left out. */
+export const PROJECT_TABLES = TABLES.filter((t) => t !== "trace_updates");
+/** Every statement without its TTL clauses: for tests on a fixed clock, whose rows would expire on insert. */
+export const withoutTtl = (sql) => sql.replace(/\sTTL\s[\s\S]*?DELETE/g, "");
+/** Create the database (through the default one) and every table and view. `ttl: false` keeps every row (tests only). */
+export async function migrateClickHouse(url, client, o = {}) {
+    const u = new URL(url);
+    u.pathname = "/default";
+    await chClient({ url: u.toString() }).command(`CREATE DATABASE IF NOT EXISTS \`${client.database.replace(/`/g, "")}\``);
+    for (const sql of CH_SCHEMA)
+        await client.command(o.ttl === false ? withoutTtl(sql) : sql);
+}
+//# sourceMappingURL=schema.js.map

package/dist/telemetry/ch/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/telemetry/ch/schema.ts"],"names":[],"mappings":"AAAA,4HAA4H;AAC5H,OAAO,EAAiB,QAAQ,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAEjD,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,mCAAmC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAExJ,+FAA+F;AAC/F,MAAM,aAAa,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4CAA4C,CAAC,CAAC;AAEtG,iHAAiH;AACjH,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uCAAuC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,4DAA4D,CAAC,CAAC;AAEpL,qHAAqH;AACrH,MAAM,CAAC,MAAM,SAAS,GAAsB,CAAC,GAAG,UAAU,EAAE,GAAG,QAAQ,EAAE,GAAG,WAAW,EAAE,GAAG,UAAU,EAAE,GAAG,aAAa,EAAE,GAAG,YAAY,CAAC,CAAC;AAE3I,oFAAoF;AACpF,MAAM,CAAC,MAAM,cAAc,GAAsB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,eAAe,CAAC,CAAC;AAE7F,8GAA8G;AAC9G,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;AAE7F,yHAAyH;AACzH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW,EAAE,MAAgB,EAAE,CAAC,GAAsB,EAAE;IAC9F,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC,CAAC,QAAQ,GAAG,UAAU,CAAC;IACxB,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mCAAmC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IACxH,KAAK,MAAM,GAAG,IAAI,SAAS;QAAE,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;AAC7F,CAAC"}

package/dist/telemetry/ch/settle-rows.d.ts

@@ -0,0 +1,13 @@
+import { type FactTable, type SettleDelta } from "@wizeworks/lens-core";
+import type { ChClient } from "./http.js";
+type Row = Record<string, unknown>;
+/** Fact rows as objects, with their version and expiry. `expire` maps a trace id to its row expiry base. */
+export declare function factObjects(rows: Record<FactTable, unknown[][]>, ver: number, expireOf: (r: Row) => number): Record<FactTable, Row[]>;
+/** Rows the re-settled traces had before and no longer have, as tombstones. Only traces settled before can have any. */
+export declare function tombstones(ch: ChClient, project: string, traceIds: readonly string[], fresh: Record<FactTable, Row[]>, ver: number): Promise<Record<FactTable, Row[]>>;
+/** Settings that make one batch's insert into a table safe to send again: ClickHouse keeps the first copy of a token. */
+export type Label = (table: string) => Record<string, string>;
+/** Write a merged settle delta into every tier. `retentionMs` is per project. */
+export declare function writeDeltas(ch: ChClient, delta: SettleDelta, retentionMs: (project: string) => number, label: Label): Promise<void>;
+export {};
+//# sourceMappingURL=settle-rows.d.ts.map

package/dist/telemetry/ch/settle-rows.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-rows.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AACA,OAAO,EAA6B,KAAK,SAAS,EAAgC,KAAK,WAAW,EAAS,MAAM,sBAAsB,CAAC;AAExI,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAG1C,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAMnC,4GAA4G;AAC5G,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,EAAE,GAAG,KAAK,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAUrI;AAED,wHAAwH;AACxH,wBAAsB,UAAU,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,CAa5K;AAoBD,yHAAyH;AACzH,MAAM,MAAM,KAAK,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAE9D,iFAAiF;AACjF,wBAAsB,WAAW,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAczI"}

package/dist/telemetry/ch/settle-rows.js

@@ -0,0 +1,67 @@
+// What one settle batch writes to ClickHouse: fact rows, tombstones for rows a trace no longer has, and rollup deltas.
+import { FACT_COLUMNS, FACT_TABLES, IDS_MAX, NONE_HIGH, NONE_LOW, TIERS } from "@wizeworks/lens-core";
+import { FACT_KEYS } from "./schema-facts.js";
+import { rowOf } from "./util.js";
+const COUNTERS = ["c0", "c1", "c2", "c3", "c4", "c5"];
+const keyCols = (t) => FACT_KEYS[`${t}_t`];
+const keyOf = (t, r) => [r.trace_id, ...keyCols(t).map((c) => r[c])].map((v) => JSON.stringify(v ?? null)).join("|");
+/** Fact rows as objects, with their version and expiry. `expire` maps a trace id to its row expiry base. */
+export function factObjects(rows, ver, expireOf) {
+    const out = {};
+    for (const t of FACT_TABLES) {
+        const cols = ["project_id", ...FACT_COLUMNS[t]];
+        out[t] = rows[t].map((values) => {
+            const r = rowOf(cols, values);
+            return { ...r, ver, is_deleted: 0, expire_ms: expireOf(r) };
+        });
+    }
+    return out;
+}
+/** Rows the re-settled traces had before and no longer have, as tombstones. Only traces settled before can have any. */
+export async function tombstones(ch, project, traceIds, fresh, ver) {
+    const out = {};
+    for (const t of FACT_TABLES) {
+        out[t] = [];
+        if (!traceIds.length)
+            continue;
+        const cols = ["trace_id", ...keyCols(t)];
+        const sql = `SELECT ${cols.join(", ")} FROM ${t}_t FINAL WHERE project_id = {p1:String} AND has({p2:Array(String)}, trace_id)`;
+        const keep = new Set(fresh[t].map((r) => keyOf(t, r)));
+        for (const old of await ch.query(sql, [project, traceIds])) {
+            if (!keep.has(keyOf(t, old)))
+                out[t].push({ project_id: project, ...old, ver, is_deleted: 1, expire_ms: NONE_HIGH });
+        }
+    }
+    return out;
+}
+const ROLLUP_INPUT = [
+    "project_id String, env String, kind String, key String, bucket_ms Float64, n Float64, err Float64, sum_v Float64",
+    ...COUNTERS.map((c) => `${c} Float64`),
+    "first_ms Float64, last_ms Float64, top_rank Float64, top Nullable(String), ids String, expire_ms Float64",
+].join(", ");
+/** Deltas add into the merge tables; a batch's deltas for one bucket and key become one row. */
+function rollupInsert(table) {
+    const sums = ["n", "err", "sum_v", ...COUNTERS].map((c) => `sum(${c})`).join(", ");
+    const ids = `groupArraySortedArrayState(${IDS_MAX})(arrayMap(x -> (-last_ms, x), arrayFilter(x -> x != '', splitByChar(' ', ids))))`;
+    return `${table}_t SELECT project_id, env, kind, key, bucket_ms, ${sums}, min(first_ms), max(last_ms), max(top_rank), argMaxState(top, top_rank), ${ids}, max(expire_ms)
+    FROM input('${ROLLUP_INPUT}') GROUP BY project_id, env, kind, key, bucket_ms`;
+}
+const HIST_INPUT = "project_id String, env String, kind String, key String, bucket_ms Float64, bin Int32, cnt Float64, expire_ms Float64";
+const histInsert = (table) => `${table}_t SELECT project_id, env, kind, key, bucket_ms, bin, sum(cnt), max(expire_ms) FROM input('${HIST_INPUT}') GROUP BY project_id, env, kind, key, bucket_ms, bin`;
+/** Write a merged settle delta into every tier. `retentionMs` is per project. */
+export async function writeDeltas(ch, delta, retentionMs, label) {
+    const writes = [];
+    for (const tier of TIERS) {
+        const rollups = delta.rollup.filter((r) => r.tier === tier.name).map((r) => ({
+            project_id: r.project, env: r.env, kind: r.kind, key: r.key, bucket_ms: r.bucket, n: r.n, err: r.err, sum_v: r.sum,
+            ...Object.fromEntries(COUNTERS.map((c, i) => [c, r.c[i] ?? 0])), first_ms: r.firstMs ?? NONE_HIGH, last_ms: r.lastMs ?? NONE_LOW,
+            top_rank: r.topRank ?? NONE_LOW, top: r.topRank === null ? null : JSON.stringify(r.top ?? null), ids: r.ids, expire_ms: r.bucket + retentionMs(r.project),
+        }));
+        const hists = delta.hist.filter((h) => h.tier === tier.name).map((h) => ({
+            project_id: h.project, env: h.env, kind: h.kind, key: h.key, bucket_ms: h.bucket, bin: h.bin, cnt: h.cnt, expire_ms: h.bucket + retentionMs(h.project),
+        }));
+        writes.push(ch.insert(rollupInsert(tier.table), rollups, label(tier.table)), ch.insert(histInsert(tier.hist), hists, label(tier.hist)));
+    }
+    await Promise.all(writes);
+}
+//# sourceMappingURL=settle-rows.js.map

package/dist/telemetry/ch/settle-rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle-rows.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle-rows.ts"],"names":[],"mappings":"AAAA,uHAAuH;AACvH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAkB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAoB,KAAK,EAAE,MAAM,sBAAsB,CAAC;AACxI,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAIlC,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;AACtD,MAAM,OAAO,GAAG,CAAC,CAAY,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;AACvD,MAAM,KAAK,GAAG,CAAC,CAAY,EAAE,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAErI,4GAA4G;AAC5G,MAAM,UAAU,WAAW,CAAC,IAAoC,EAAE,GAAW,EAAE,QAA4B;IACzG,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,CAAC,YAAY,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;QAChD,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YAC9B,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC9B,OAAO,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,wHAAwH;AACxH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAY,EAAE,OAAe,EAAE,QAA2B,EAAE,KAA+B,EAAE,GAAW;IACvI,MAAM,GAAG,GAAG,EAA8B,CAAC;IAC3C,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC5B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACZ,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS;QAC/B,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACzC,MAAM,GAAG,GAAG,UAAU,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,+EAA+E,CAAC;QAC/H,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC,KAAK,CAAM,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;gBAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QACvH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,YAAY,GAAG;IACnB,kHAAkH;IAClH,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC;IACtC,0GAA0G;CAC3G,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAEb,gGAAgG;AAChG,SAAS,YAAY,CAAC,KAAa;IACjC,MAAM,IAAI,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,GAAG,GAAG,8BAA8B,OAAO,mFAAmF,CAAC;IACrI,OAAO,GAAG,KAAK,oDAAoD,IAAI,6EAA6E,GAAG;kBACvI,YAAY,mDAAmD,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,GAAG,sHAAsH,CAAC;AAC1I,MAAM,UAAU,GAAG,CAAC,KAAa,EAAE,EAAE,CACnC,GAAG,KAAK,8FAA8F,UAAU,wDAAwD,CAAC;AAK3K,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,WAAwC,EAAE,KAAY;IACxH,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3E,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG;YAClH,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,IAAI,QAAQ;YAChI,QAAQ,EAAE,CAAC,CAAC,OAAO,IAAI,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;SAC1J,CAAC,CAAC,CAAC;QACJ,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACvE,UAAU,EAAE,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;SACvJ,CAAC,CAAC,CAAC;QACJ,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1I,CAAC;IACD,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC"}

package/dist/telemetry/ch/settle.d.ts

@@ -0,0 +1,7 @@
+import type { SettleOptions } from "../store.js";
+import type { ChClient } from "./http.js";
+import type { SettleQueue } from "./queue.js";
+import { type RetentionCache } from "./util.js";
+/** Settle due traces until none are left or the budget is spent. Batches overlap (`DEPTH` at once): one reads and computes while others wait on their writes. */
+export declare function settleClickHouse(ch: ChClient, queue: SettleQueue, retention: RetentionCache, now: number, o?: SettleOptions): Promise<number>;
+//# sourceMappingURL=settle.d.ts.map

package/dist/telemetry/ch/settle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/settle.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,KAAK,EAAS,WAAW,EAAE,MAAM,YAAY,CAAC;AAGrD,OAAO,EAAsB,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AA6EpE,iKAAiK;AACjK,wBAAsB,gBAAgB,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,GAAE,aAAkB,GAAG,OAAO,CAAC,MAAM,CAAC,CAuBvJ"}

package/dist/telemetry/ch/settle.js

@@ -0,0 +1,96 @@
+// Settling on ClickHouse: the same facts and deltas as Postgres, written as inserts. One process settles, so no lock is needed.
+import { compact, factRows, FACT_TABLES, mergeDeltas, settleDelta, storedOf, traceFacts, withoutTop } from "@wizeworks/lens-core";
+import { loadTraces } from "./read.js";
+import { factObjects, tombstones, writeDeltas } from "./settle-rows.js";
+import { byProject, nextVer } from "./util.js";
+const BATCH = 2_000;
+const BUDGET_MS = 2_000;
+const DEPTH = 3;
+const hasData = (t) => !!t && t.spanCount + t.logs.length > 0;
+/** Stored contributions of some traces, by trace id. */
+async function storedContribs(ch, project, ids) {
+    const rows = await ch.query("SELECT trace_id, contrib FROM settle_state FINAL WHERE project_id = {p1:String} AND has({p2:Array(String)}, trace_id)", [project, ids]);
+    return new Map(rows.map((r) => [r.trace_id, r.contrib]));
+}
+/** Facts, deltas, and new settle states for one project's dirty traces. */
+async function projectWork(ch, project, items, now, retentionMs, ver) {
+    const ids = items.map((d) => d.traceId);
+    const [traces, stored] = await Promise.all([loadTraces(ch, [project], ids), storedContribs(ch, project, ids)]);
+    const raw = Object.fromEntries(FACT_TABLES.map((t) => [t, []]));
+    const work = { facts: {}, deltas: [], states: [], resettled: [] };
+    items.forEach((d, i) => {
+        const trace = traces[i];
+        const scope = { project, env: d.env };
+        const facts = hasData(trace) ? traceFacts(trace) : null;
+        const next = facts ? compact(facts, scope) : null;
+        const old = stored.get(d.traceId);
+        if (old !== undefined)
+            work.resettled.push(d.traceId);
+        work.deltas.push(settleDelta(storedOf(old || null), next, now, retentionMs));
+        if (facts)
+            for (const [t, rows] of Object.entries(factRows(facts, scope, trace)))
+                raw[t].push(...rows);
+        const start = trace?.startMs ?? now;
+        work.states.push({ project_id: project, trace_id: d.traceId, settled_ms: d.updatedMs, contrib: next ? JSON.stringify(withoutTop(next)) : "", ver, expire_ms: start + retentionMs });
+    });
+    const timeOf = (r) => Number(r.start_ms ?? r.time_ms ?? now);
+    work.facts = factObjects(raw, ver, (r) => timeOf(r) + retentionMs);
+    return work;
+}
+/** Write one project's work: facts (with tombstones) and rollups at once, then the settle states, so a failed write settles again. */
+async function writeWork(ch, project, w, ver, retentionMs, label) {
+    const dead = await tombstones(ch, project, w.resettled, w.facts, ver);
+    const facts = FACT_TABLES.map((t) => ch.insert(`${t}_t`, [...dead[t], ...w.facts[t]], label(t)));
+    await Promise.all([...facts, writeDeltas(ch, mergeDeltas(w.deltas), () => retentionMs, label)]);
+    await ch.insert("settle_state", w.states, label("settle_state"));
+}
+/** Settle one batch of taken traces. A failure releases the ones not written yet. */
+async function settleItems(ch, queue, retention, now, due, o) {
+    let left = [...due];
+    try {
+        const days = await retention.daysOf(due.map((d) => d.project));
+        for (const [project, items] of byProject(due)) {
+            const retentionMs = days.get(project) * 86_400_000;
+            const ver = nextVer(now);
+            const label = (table) => ({ insert_deduplication_token: `${ver}:${project}:${table}` });
+            await writeWork(ch, project, await projectWork(ch, project, items, now, retentionMs, ver), ver, retentionMs, label);
+            for (const d of items)
+                queue.settled(d, d.updatedMs);
+            left = left.filter((d) => d.project !== project);
+            o.onSettled?.(project);
+        }
+        return due.length;
+    }
+    finally {
+        queue.release(left);
+    }
+}
+/** Settle due traces until none are left or the budget is spent. Batches overlap (`DEPTH` at once): one reads and computes while others wait on their writes. */
+export async function settleClickHouse(ch, queue, retention, now, o = {}) {
+    const start = Date.now();
+    const batch = o.batch ?? BATCH;
+    const running = [];
+    const started = [];
+    let total = 0;
+    try {
+        for (;;) {
+            const due = queue.take(now, batch);
+            if (due.length) {
+                const run = settleItems(ch, queue, retention, now, due, o);
+                // Handled now, awaited in turn: a batch that fails while another is awaited must not end the process.
+                run.catch(() => undefined);
+                running.push(run);
+                started.push(run);
+            }
+            const last = !due.length || due.length < batch || Date.now() - start >= (o.budgetMs ?? BUDGET_MS);
+            while (running.length && (last || running.length >= DEPTH))
+                total += await running.shift();
+            if (last)
+                return total;
+        }
+    }
+    finally {
+        await Promise.allSettled(started);
+    }
+}
+//# sourceMappingURL=settle.js.map

package/dist/telemetry/ch/settle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"settle.js","sourceRoot":"","sources":["../../../src/telemetry/ch/settle.ts"],"names":[],"mappings":"AAAA,gIAAgI;AAChI,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAkB,WAAW,EAAoB,WAAW,EAAE,QAAQ,EAAc,UAAU,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAIhL,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,WAAW,EAAc,UAAU,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACpF,OAAO,EAAE,SAAS,EAAE,OAAO,EAAuB,MAAM,WAAW,CAAC;AAEpE,MAAM,KAAK,GAAG,KAAK,CAAC;AACpB,MAAM,SAAS,GAAG,KAAK,CAAC;AACxB,MAAM,KAAK,GAAG,CAAC,CAAC;AAIhB,MAAM,OAAO,GAAG,CAAC,CAAoB,EAAc,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AAE7F,wDAAwD;AACxD,KAAK,UAAU,cAAc,CAAC,EAAY,EAAE,OAAe,EAAE,GAAsB;IACjF,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,KAAK,CACzB,uHAAuH,EACvH,CAAC,OAAO,EAAE,GAAG,CAAC,CACf,CAAC;IACF,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC3D,CAAC;AASD,2EAA2E;AAC3E,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,OAAe,EAAE,KAAuB,EAAE,GAAW,EAAE,WAAmB,EAAE,GAAW;IAC9H,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,EAAE,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAC/G,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAiB,CAAC,CAAC,CAAmC,CAAC;IACjH,MAAM,IAAI,GAAgB,EAAE,KAAK,EAAE,EAA8B,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC3G,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrB,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAClD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAClC,IAAI,GAAG,KAAK,SAAS;YAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC7E,IAAI,KAAK;YAAE,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAA+B;gBAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;QACrI,MAAM,KAAK,GAAG,KAAK,EAAE,OAAO,IAAI,GAAG,CAAC;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,GAAG,WAAW,EAAE,CAAC,CAAC;IACtL,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,CAAC,CAAM,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,IAAI,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC;IACnE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,sIAAsI;AACtI,KAAK,UAAU,SAAS,CAAC,EAAY,EAAE,OAAe,EAAE,CAAc,EAAE,GAAW,EAAE,WAAmB,EAAE,KAAY;IACpH,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACtE,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,WAAW,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IAChG,MAAM,EAAE,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACnE,CAAC;AAED,qFAAqF;AACrF,KAAK,UAAU,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,SAAyB,EAAE,GAAW,EAAE,GAAqB,EAAE,CAAgB;IAC1I,IAAI,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC/D,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAE,GAAG,UAAU,CAAC;YACpD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YACzB,MAAM,KAAK,GAAU,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,0BAA0B,EAAE,GAAG,GAAG,IAAI,OAAO,IAAI,KAAK,EAAE,EAAE,CAAC,CAAC;YAC/F,MAAM,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC;YACpH,KAAK,MAAM,CAAC,IAAI,KAAK;gBAAE,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,CAAC;YACjD,CAAC,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC;QACzB,CAAC;QACD,OAAO,GAAG,CAAC,MAAM,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED,iKAAiK;AACjK,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,EAAY,EAAE,KAAkB,EAAE,SAAyB,EAAE,GAAW,EAAE,CAAC,GAAkB,EAAE;IACpI,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC;IAC/B,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC;QACH,SAAS,CAAC;YACR,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YACnC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;gBAC3D,sGAAsG;gBACtG,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC3B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAClB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,CAAC;YACD,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC;YAClG,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC;gBAAE,KAAK,IAAI,MAAM,OAAO,CAAC,KAAK,EAAG,CAAC;YAC5F,IAAI,IAAI;gBAAE,OAAO,KAAK,CAAC;QACzB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;AACH,CAAC"}

package/dist/telemetry/ch/store.d.ts

@@ -0,0 +1,16 @@
+import { type SqlRunner } from "@wizeworks/lens-core";
+import type { RetentionOf, TelemetryStore } from "../store.js";
+import { type ChClient } from "./http.js";
+export interface ChStoreOptions {
+    url: string;
+    retentionOf: RetentionOf;
+    defaultRetentionDays: number;
+    /** The clock for recovery on start. */
+    now: () => number;
+    /** False leaves the TTLs out (tests on a fixed clock). Default true. */
+    ttl?: boolean;
+}
+export declare const chRunner: (ch: ChClient) => SqlRunner;
+/** Create the schema, rebuild the settle queue, and return the store. */
+export declare function openClickHouse(o: ChStoreOptions): Promise<TelemetryStore>;
+//# sourceMappingURL=store.d.ts.map

package/dist/telemetry/ch/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/store.ts"],"names":[],"mappings":"AACA,OAAO,EAAoH,KAAK,SAAS,EAAc,MAAM,sBAAsB,CAAC;AACpL,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAE/D,OAAO,EAAE,KAAK,QAAQ,EAAY,MAAM,WAAW,CAAC;AAUpD,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,EAAE,WAAW,CAAC;IACzB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uCAAuC;IACvC,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,wEAAwE;IACxE,GAAG,CAAC,EAAE,OAAO,CAAC;CACf;AAED,eAAO,MAAM,QAAQ,OAAQ,QAAQ,KAAG,SAAuF,CAAC;AAWhI,yEAAyE;AACzE,wBAAsB,cAAc,CAAC,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAO/E"}

package/dist/telemetry/ch/store.js

@@ -0,0 +1,65 @@
+// Scale mode: telemetry in ClickHouse. Reads go through views with the Postgres names, so the shared SQL answers pages and searches.
+import { alignNow, CH_DIALECT, metricAlertSteps, metricPageSteps, runMetricAsync, searchSql, sqlExecutor, suggestSql } from "@wizeworks/lens-core";
+import { checkFactsVersionCh, clearProjectCh, syncProjects } from "./admin.js";
+import { chClient } from "./http.js";
+import { MetricWriter } from "./metrics.js";
+import { activeHours, counts, lastTraceMs, slowestEndpoints, traceEnvs } from "./queries.js";
+import { SettleQueue } from "./queue.js";
+import { findTraceIds, getTrace, joinedTrace, loadTraces } from "./read.js";
+import { migrateClickHouse } from "./schema.js";
+import { settleClickHouse } from "./settle.js";
+import { RetentionCache } from "./util.js";
+import { writeRaw } from "./write.js";
+export const chRunner = (ch) => ({ all: (sql, params) => ch.query(sql, params) });
+/** ClickHouse may still be starting (a deploy starts both at once): wait up to two minutes. */
+async function waitReady(ch) {
+    for (let i = 0; i < 60; i++) {
+        if (await ch.ping())
+            return;
+        await new Promise((r) => setTimeout(r, 2_000));
+    }
+    throw new Error("ClickHouse did not answer /ping for two minutes. Check --clickhouse and that the server runs.");
+}
+/** Create the schema, rebuild the settle queue, and return the store. */
+export async function openClickHouse(o) {
+    const ch = chClient({ url: o.url });
+    await waitReady(ch);
+    await migrateClickHouse(o.url, ch, { ttl: o.ttl });
+    const queue = new SettleQueue();
+    await queue.recover(ch, o.now());
+    return chTelemetry(ch, queue, new RetentionCache(o.retentionOf, o.defaultRetentionDays), o.retentionOf);
+}
+function chTelemetry(ch, queue, retention, retentionOf) {
+    const src = (projectId) => ({ d: CH_DIALECT, run: chRunner(ch), scope: { project: projectId } });
+    const metrics = new MetricWriter(ch);
+    const runMetric = (s) => ch.query(s.sql, s.args);
+    return {
+        kind: "clickhouse",
+        async write(projectId, w, now) {
+            const retentionMs = await retention.msOf(projectId);
+            await writeRaw(ch, queue, projectId, w, now, retentionMs);
+            await metrics.write(projectId, w.sender.environment ?? "", w.metrics ?? [], now, retentionMs);
+        },
+        traces: async (t) => loadTraces(ch, [t.projectId], await findTraceIds(ch, t, Date.now())),
+        trace: (projectId, traceId) => getTrace(ch, [projectId], traceId),
+        joinedTrace: (primaryId, scope, traceId) => joinedTrace(ch, primaryId, scope, traceId),
+        counts: (projectId) => counts(ch, projectId),
+        traceEnvs: (projectId) => traceEnvs(ch, projectId),
+        activeHours: (projectId, from, to) => activeHours(ch, projectId, from, to),
+        lastTraceMs: () => lastTraceMs(ch, Date.now()),
+        executor: (projectId, retentionMs, env) => sqlExecutor({ ...src(projectId), scope: { project: projectId, env }, retentionMs, loadTrace: (id) => getTrace(ch, [projectId], id).then((t) => t ?? null) }),
+        settling: async (projectId, env) => queue.count(projectId, env) || null,
+        search: (projectId, q) => searchSql(src(projectId), q),
+        suggest: (projectId, r) => suggestSql(src(projectId), r),
+        slowestEndpoints: (projectId, w, minCount, limit) => slowestEndpoints(ch, projectId, w, minCount, limit),
+        metricData: (m, range, params, now) => runMetricAsync(metricPageSteps("ch", { project: m.projectId, env: m.env }, range, alignNow(now), params), runMetric),
+        metricAlertData: (m, name, w, now) => runMetricAsync(metricAlertSteps("ch", { project: m.projectId, env: m.env, service: m.service }, name, w, now), runMetric),
+        settle: (now, o) => settleClickHouse(ch, queue, retention, now, o),
+        prune: (now) => syncProjects(ch, queue, retentionOf, now),
+        pruneRollups: async () => undefined,
+        checkFactsVersion: () => checkFactsVersionCh(ch, queue),
+        clearProject: (projectId) => clearProjectCh(ch, queue, projectId),
+        close: async () => undefined,
+    };
+}
+//# sourceMappingURL=store.js.map

package/dist/telemetry/ch/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../../src/telemetry/ch/store.ts"],"names":[],"mappings":"AAAA,qIAAqI;AACrI,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,gBAAgB,EAAE,eAAe,EAAmB,cAAc,EAAE,SAAS,EAAE,WAAW,EAAkB,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAEpL,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAiB,QAAQ,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,WAAW,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC7F,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAYtC,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAY,EAAa,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAA0B,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;AAEhI,+FAA+F;AAC/F,KAAK,UAAU,SAAS,CAAC,EAAY;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,MAAM,EAAE,CAAC,IAAI,EAAE;YAAE,OAAO;QAC5B,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,+FAA+F,CAAC,CAAC;AACnH,CAAC;AAED,yEAAyE;AACzE,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,CAAiB;IACpD,MAAM,EAAE,GAAG,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACpC,MAAM,SAAS,CAAC,EAAE,CAAC,CAAC;IACpB,MAAM,iBAAiB,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC;IAChC,MAAM,KAAK,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;IACjC,OAAO,WAAW,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,cAAc,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC,WAAW,CAAC,CAAC;AAC1G,CAAC;AAED,SAAS,WAAW,CAAC,EAAY,EAAE,KAAkB,EAAE,SAAyB,EAAE,WAAwB;IACxG,MAAM,GAAG,GAAG,CAAC,SAAiB,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IACzG,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC;IACrC,MAAM,SAAS,GAAG,CAAC,CAAmC,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAwB,CAAC;IAC1G,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,EAAE,GAAG;YAC3B,MAAM,WAAW,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACpD,MAAM,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;YAC1D,MAAM,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC;QAChG,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,MAAM,YAAY,CAAC,EAAE,EAAE,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QACzF,KAAK,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC;QACjE,WAAW,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC;QACtF,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,CAAC;QAC5C,SAAS,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC;QAClD,WAAW,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC;QAC1E,WAAW,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC;QAC9C,QAAQ,EAAE,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,CACxC,WAAW,CAAC,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QAC9J,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,IAAI;QACvE,MAAM,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACtD,OAAO,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACxD,gBAAgB,EAAE,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC;QACxG,UAAU,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,eAAe,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,SAAS,CAAC;QAC3J,eAAe,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,cAAc,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC;QAC/J,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;QAClE,KAAK,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,CAAC;QACzD,YAAY,EAAE,KAAK,IAAI,EAAE,CAAC,SAAS;QACnC,iBAAiB,EAAE,GAAG,EAAE,CAAC,mBAAmB,CAAC,EAAE,EAAE,KAAK,CAAC;QACvD,YAAY,EAAE,CAAC,SAAS,EAAE,EAAE,CAAC,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,SAAS,CAAC;QACjE,KAAK,EAAE,KAAK,IAAI,EAAE,CAAC,SAAS;KAC7B,CAAC;AACJ,CAAC"}

package/dist/telemetry/ch/util.d.ts

@@ -0,0 +1,20 @@
+import type { RetentionOf } from "../store.js";
+declare const DAY_MS = 86400000;
+/** A row version that only grows: ms times 1000, plus a counter within the same ms. */
+export declare function nextVer(now: number): number;
+/** Retention days per project, from Postgres, cached for a minute. */
+export declare class RetentionCache {
+    private readonly lookup;
+    private readonly defaultDays;
+    private readonly cache;
+    constructor(lookup: RetentionOf, defaultDays: number);
+    daysOf(projectIds: readonly string[]): Promise<Map<string, number>>;
+    msOf(projectId: string): Promise<number>;
+}
+/** A row object from a column list and values in the same order. */
+export declare function rowOf(cols: readonly string[], values: readonly unknown[]): Record<string, unknown>;
+export declare const byProject: <T extends {
+    project: string;
+}>(items: readonly T[]) => Map<string, T[]>;
+export { DAY_MS };
+//# sourceMappingURL=util.d.ts.map

package/dist/telemetry/ch/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../src/telemetry/ch/util.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,QAAA,MAAM,MAAM,WAAa,CAAC;AAK1B,uFAAuF;AACvF,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAG3C;AAED,sEAAsE;AACtE,qBAAa,cAAc;IAIvB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAJ9B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqD;IAE3E,YACmB,MAAM,EAAE,WAAW,EACnB,WAAW,EAAE,MAAM,EAClC;IAEE,MAAM,CAAC,UAAU,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAQxE;IAEK,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAE7C;CACF;AAED,oEAAoE;AACpE,wBAAgB,KAAK,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,MAAM,EAAE,SAAS,OAAO,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAIlG;AAED,eAAO,MAAM,SAAS,GAAI,CAAC,SAAS;IAAE,OAAO,EAAE,MAAM,CAAA;CAAE,SAAS,SAAS,CAAC,EAAE,KAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,CAI7F,CAAC;AAEF,OAAO,EAAE,MAAM,EAAE,CAAC"}