@voltro/plugin-clickhouse 0.84.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1215 -0
- package/THIRD-PARTY-NOTICES.md +118 -600
- package/dist/index.js +28 -288
- package/dist/sink.js +259 -0
- package/package.json +4 -4
package/dist/index.js
CHANGED
|
@@ -1,300 +1,40 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
CREATE TABLE IF NOT EXISTS ${e}.${s(t)} (
|
|
8
|
-
id String,
|
|
9
|
-
data String CODEC(ZSTD(1)),
|
|
10
|
-
version UInt64,
|
|
11
|
-
is_deleted UInt8 DEFAULT 0
|
|
12
|
-
) ENGINE = ReplacingMergeTree(version)
|
|
13
|
-
ORDER BY id
|
|
14
|
-
`.trim(), l = (e, t) => `
|
|
15
|
-
CREATE TABLE IF NOT EXISTS ${e}.${t} (
|
|
16
|
-
id UUID DEFAULT generateUUIDv7(),
|
|
17
|
-
name LowCardinality(String),
|
|
18
|
-
subject_id Nullable(String),
|
|
19
|
-
tenant_id Nullable(String),
|
|
20
|
-
properties String CODEC(ZSTD(1)),
|
|
21
|
-
occurred_at DateTime64(3),
|
|
22
|
-
ingested_at DateTime64(3) DEFAULT now64()
|
|
23
|
-
) ENGINE = MergeTree
|
|
24
|
-
ORDER BY (name, occurred_at, id)
|
|
25
|
-
`.trim(), u = (e) => (t) => t instanceof i ? new i({
|
|
26
|
-
provider: o,
|
|
27
|
-
operation: e,
|
|
28
|
-
cause: t.cause
|
|
29
|
-
}) : new i({
|
|
30
|
-
provider: o,
|
|
31
|
-
operation: e,
|
|
32
|
-
cause: t instanceof Error ? `${t.name}: ${t.message}` : String(t)
|
|
33
|
-
}), d = (e) => (r) => r.pipe(t.catchSomeDefect((e) => e instanceof i ? n.some(t.fail(e)) : n.none()), t.mapError(u(e))), f = /^[A-Za-z_][A-Za-z0-9_]*$/, p = (e, t) => {
|
|
34
|
-
if (!f.test(e)) throw new i({
|
|
35
|
-
provider: o,
|
|
36
|
-
operation: "config",
|
|
37
|
-
cause: `invalid ${t} ${JSON.stringify(e)} — must match ${f.source} (letters, digits, underscore; no dots/quotes/spaces)`
|
|
38
|
-
});
|
|
39
|
-
return e;
|
|
40
|
-
}, m = /['\\]/, h = (e) => {
|
|
41
|
-
for (let t = 0; t < e.length; t++) if (e.charCodeAt(t) < 32) return !0;
|
|
42
|
-
return !1;
|
|
43
|
-
}, g = (e) => (t) => {
|
|
44
|
-
if (m.test(t) || h(t)) throw new i({
|
|
45
|
-
provider: o,
|
|
46
|
-
operation: e,
|
|
47
|
-
cause: `invalid property key ${JSON.stringify(t)} — must not contain quotes, backslashes, or control characters`
|
|
48
|
-
});
|
|
49
|
-
return t;
|
|
50
|
-
}, _ = (e) => e.client === null ? t.fail(new i({
|
|
51
|
-
provider: o,
|
|
52
|
-
operation: "ensureClient",
|
|
53
|
-
cause: "client not initialized — spec.initialize must run before queries"
|
|
54
|
-
})) : t.succeed(e.client), v = (e) => t.gen(function* () {
|
|
55
|
-
let n = e.client;
|
|
56
|
-
e.client = null, n !== null && (yield* t.tryPromise(() => n.close()).pipe(t.catchAll((e) => t.logWarning("clickhouse analytics sink teardown failed", {
|
|
57
|
-
provider: o,
|
|
58
|
-
err: e
|
|
59
|
-
}))));
|
|
60
|
-
}), y = (e) => ({
|
|
61
|
-
name: e.name,
|
|
62
|
-
subject_id: e.subjectId ?? null,
|
|
63
|
-
tenant_id: e.tenantId ?? null,
|
|
64
|
-
properties: JSON.stringify(e.properties),
|
|
65
|
-
occurred_at: (e.occurredAt ?? /* @__PURE__ */ new Date()).toISOString()
|
|
66
|
-
}), b = (e, t) => {
|
|
67
|
-
let n = [], r, i = () => {
|
|
68
|
-
r !== void 0 && (clearTimeout(r), r = void 0);
|
|
69
|
-
}, o = async () => {
|
|
70
|
-
if (i(), n.length === 0) return;
|
|
71
|
-
let t = n;
|
|
72
|
-
n = [];
|
|
73
|
-
let r = e.client;
|
|
74
|
-
if (r === null) {
|
|
75
|
-
a.warn("clickhouse batch flush skipped — client not initialized; events dropped", { dropped: t.length });
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
try {
|
|
79
|
-
await r.insert({
|
|
80
|
-
table: `${e.database}.${e.table}`,
|
|
81
|
-
values: t,
|
|
82
|
-
format: "JSONEachRow"
|
|
83
|
-
});
|
|
84
|
-
} catch (e) {
|
|
85
|
-
a.warn("clickhouse batch flush failed — dropped events", { dropped: t.length }, e);
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
return {
|
|
89
|
-
add: (e) => {
|
|
90
|
-
if (n.push(y(e)), n.length >= t.maxSize) {
|
|
91
|
-
o();
|
|
92
|
-
return;
|
|
93
|
-
}
|
|
94
|
-
r === void 0 && (r = setTimeout(() => {
|
|
95
|
-
o();
|
|
96
|
-
}, t.flushIntervalMs), r.unref?.());
|
|
97
|
-
},
|
|
98
|
-
flush: o
|
|
99
|
-
};
|
|
100
|
-
}, x = (e) => {
|
|
101
|
-
switch (e) {
|
|
102
|
-
case "minute": return "toStartOfMinute";
|
|
103
|
-
case "hour": return "toStartOfHour";
|
|
104
|
-
case "day": return "toStartOfDay";
|
|
105
|
-
case "week": return "toStartOfWeek";
|
|
106
|
-
case "month": return "toStartOfMonth";
|
|
107
|
-
}
|
|
108
|
-
}, S = (e, t) => `JSONExtractString(properties, '${g(e)(t)}')`, C = (e, t) => `JSONExtractFloat(properties, '${g(e)(t)}')`, w = (e, t) => t === "count" ? "count()" : t === "unique" ? "uniqExact(subject_id)" : "sum" in t ? `sum(${C(e, t.sum)})` : `avg(${C(e, t.avg)})`, T = (e, t) => t === "count" ? "count()" : `sum(${C(e, t.sum)})`, E = (e, t, n, r, i) => {
|
|
109
|
-
let a = {
|
|
110
|
-
event: t,
|
|
111
|
-
from: n.from.toISOString(),
|
|
112
|
-
...n.to === void 0 ? {} : { to: n.to.toISOString() }
|
|
113
|
-
}, o = "name = {event:String} AND occurred_at >= {from:DateTime64(3)}" + (n.to === void 0 ? "" : " AND occurred_at < {to:DateTime64(3)}");
|
|
114
|
-
if (i != null && (a.tenant = i, o += " AND tenant_id = {tenant:String}"), r !== void 0) {
|
|
115
|
-
let t = 0;
|
|
116
|
-
for (let [n, i] of Object.entries(r)) {
|
|
117
|
-
let r = `f${t++}`;
|
|
118
|
-
a[r] = String(i), o += ` AND ${S(e, n)} = {${r}:String}`;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
return {
|
|
122
|
-
sql: o,
|
|
123
|
-
params: a
|
|
124
|
-
};
|
|
125
|
-
}, D = (e) => {
|
|
126
|
-
let n = (n, r) => t.gen(function* () {
|
|
127
|
-
p(n, "mirror table");
|
|
128
|
-
let i = yield* _(e);
|
|
129
|
-
yield* t.tryPromise({
|
|
130
|
-
try: () => i.insert({
|
|
131
|
-
table: `${e.database}.${s(n)}`,
|
|
132
|
-
values: [r],
|
|
133
|
-
format: "JSONEachRow"
|
|
134
|
-
}),
|
|
135
|
-
catch: (e) => e
|
|
136
|
-
});
|
|
137
|
-
});
|
|
138
|
-
return {
|
|
139
|
-
tables: e.mirrorTables,
|
|
140
|
-
primaryKey: e.mirrorPrimaryKey,
|
|
141
|
-
upsert: ({ table: r, row: i, version: a }) => t.gen(function* () {
|
|
142
|
-
let t = String(i[e.mirrorPrimaryKey]);
|
|
143
|
-
yield* n(r, {
|
|
144
|
-
id: t,
|
|
145
|
-
data: JSON.stringify(i),
|
|
146
|
-
version: a,
|
|
147
|
-
is_deleted: 0
|
|
148
|
-
});
|
|
149
|
-
}).pipe(d("mirror.upsert")),
|
|
150
|
-
remove: ({ table: e, primaryKeyValue: r, version: i }) => t.gen(function* () {
|
|
151
|
-
yield* n(e, {
|
|
152
|
-
id: String(r),
|
|
153
|
-
data: "{}",
|
|
154
|
-
version: i,
|
|
155
|
-
is_deleted: 1
|
|
156
|
-
});
|
|
157
|
-
}).pipe(d("mirror.remove")),
|
|
158
|
-
maxVersion: ({ table: n, primaryKeyValue: r }) => t.gen(function* () {
|
|
159
|
-
p(String(n), "mirror table");
|
|
160
|
-
let i = yield* _(e), a = `SELECT max(version) AS v, count() AS n FROM ${e.database}.${s(String(n))} WHERE id = {id: String} FORMAT JSON`, o = yield* t.tryPromise({
|
|
161
|
-
try: () => i.query({
|
|
162
|
-
query: a,
|
|
163
|
-
query_params: { id: String(r) }
|
|
164
|
-
}),
|
|
165
|
-
catch: (e) => e
|
|
166
|
-
}), { data: c } = yield* t.tryPromise({
|
|
167
|
-
try: () => o.json(),
|
|
168
|
-
catch: (e) => e
|
|
169
|
-
}), l = c[0];
|
|
170
|
-
return l === void 0 || Number(l.n) === 0 || l.v === null ? null : Number(l.v);
|
|
171
|
-
}).pipe(d("mirror.maxVersion"))
|
|
172
|
-
};
|
|
173
|
-
}, O = (e) => {
|
|
174
|
-
let n = e.mirrorTables.length > 0 ? D(e) : void 0;
|
|
175
|
-
return {
|
|
176
|
-
info: {
|
|
177
|
-
provider: o,
|
|
178
|
-
capabilities: {
|
|
179
|
-
track: !0,
|
|
180
|
-
aggregate: !0,
|
|
181
|
-
timeseries: !0,
|
|
182
|
-
topN: !0,
|
|
183
|
-
mirror: n !== void 0
|
|
184
|
-
},
|
|
185
|
-
detail: {
|
|
186
|
-
database: e.database,
|
|
187
|
-
table: e.table,
|
|
188
|
-
...e.mirrorTables.length > 0 ? { mirrorTables: e.mirrorTables } : {},
|
|
189
|
-
...e.batcher === void 0 ? {} : { batching: !0 }
|
|
190
|
-
}
|
|
191
|
-
},
|
|
192
|
-
track: (n) => e.batcher ? t.sync(() => e.batcher.add(n)) : t.gen(function* () {
|
|
193
|
-
let r = yield* _(e);
|
|
194
|
-
yield* t.tryPromise({
|
|
195
|
-
try: () => r.insert({
|
|
196
|
-
table: `${e.database}.${e.table}`,
|
|
197
|
-
values: [y(n)],
|
|
198
|
-
format: "JSONEachRow"
|
|
199
|
-
}),
|
|
200
|
-
catch: (e) => e
|
|
201
|
-
});
|
|
202
|
-
}).pipe(d("track")),
|
|
203
|
-
aggregate: (n) => t.gen(function* () {
|
|
204
|
-
let r = yield* _(e), i = E("aggregate", n.event, n.range, n.filter, n.tenantId), a = `SELECT ${w("aggregate", n.metric)} AS value FROM ${e.database}.${e.table} WHERE ${i.sql} FORMAT JSON`, o = yield* t.tryPromise({
|
|
205
|
-
try: () => r.query({
|
|
206
|
-
query: a,
|
|
207
|
-
query_params: i.params
|
|
208
|
-
}),
|
|
209
|
-
catch: (e) => e
|
|
210
|
-
}), { data: s } = yield* t.tryPromise({
|
|
211
|
-
try: () => o.json(),
|
|
212
|
-
catch: (e) => e
|
|
213
|
-
}), c = s[0]?.value ?? 0;
|
|
214
|
-
return typeof c == "number" ? c : Number(c ?? 0);
|
|
215
|
-
}).pipe(d("aggregate")),
|
|
216
|
-
timeseries: (n) => t.gen(function* () {
|
|
217
|
-
let r = yield* _(e), i = E("timeseries", n.event, n.range, n.filter, n.tenantId), a = `
|
|
218
|
-
SELECT ${`${x(n.bucket)}(occurred_at)`} AS bucket_start, ${w("timeseries", n.metric)} AS value
|
|
219
|
-
FROM ${e.database}.${e.table}
|
|
220
|
-
WHERE ${i.sql}
|
|
221
|
-
GROUP BY bucket_start
|
|
222
|
-
ORDER BY bucket_start ASC
|
|
223
|
-
FORMAT JSON
|
|
224
|
-
`, o = yield* t.tryPromise({
|
|
225
|
-
try: () => r.query({
|
|
226
|
-
query: a,
|
|
227
|
-
query_params: i.params
|
|
228
|
-
}),
|
|
229
|
-
catch: (e) => e
|
|
230
|
-
}), { data: s } = yield* t.tryPromise({
|
|
231
|
-
try: () => o.json(),
|
|
232
|
-
catch: (e) => e
|
|
233
|
-
});
|
|
234
|
-
return s.map((e) => ({
|
|
235
|
-
bucketStart: new Date(e.bucket_start),
|
|
236
|
-
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
237
|
-
}));
|
|
238
|
-
}).pipe(d("timeseries")),
|
|
239
|
-
topN: (n) => t.gen(function* () {
|
|
240
|
-
let r = yield* _(e), i = E("topN", n.event, n.range, n.filter, n.tenantId), a = Math.max(1, Math.floor(n.n)), o = S("topN", n.groupBy), s = `
|
|
241
|
-
SELECT ${o} AS key, ${T("topN", n.metric)} AS value
|
|
242
|
-
FROM ${e.database}.${e.table}
|
|
243
|
-
WHERE ${i.sql}
|
|
244
|
-
GROUP BY ${o}
|
|
245
|
-
ORDER BY value DESC
|
|
246
|
-
LIMIT ${a}
|
|
247
|
-
FORMAT JSON
|
|
248
|
-
`, c = yield* t.tryPromise({
|
|
249
|
-
try: () => r.query({
|
|
250
|
-
query: s,
|
|
251
|
-
query_params: i.params
|
|
252
|
-
}),
|
|
253
|
-
catch: (e) => e
|
|
254
|
-
}), { data: l } = yield* t.tryPromise({
|
|
255
|
-
try: () => c.json(),
|
|
256
|
-
catch: (e) => e
|
|
257
|
-
});
|
|
258
|
-
return l.filter((e) => e.key !== null).map((e) => ({
|
|
259
|
-
key: e.key,
|
|
260
|
-
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
261
|
-
}));
|
|
262
|
-
}).pipe(d("topN")),
|
|
263
|
-
...n === void 0 ? {} : { mirror: n }
|
|
264
|
-
};
|
|
265
|
-
}, k = 20, A = 5e3, j = (n) => {
|
|
266
|
-
let r = (n.mirrorTables ?? []).map((e) => p(e, "mirror table")), i = {
|
|
1
|
+
import { EVENTS_TABLE_DDL as e, MIRROR_TABLE_DDL as t, assertIdentifier as n, buildClickhouseSink as r, closeClickhouseState as i, makeEventBatcher as a } from "./sink.js";
|
|
2
|
+
import { createClient as o } from "@clickhouse/client";
|
|
3
|
+
import { Effect as s } from "effect";
|
|
4
|
+
//#region src/index.ts
|
|
5
|
+
var c = 20, l = 5e3, u = (u) => {
|
|
6
|
+
let d = (u.mirrorTables ?? []).map((e) => n(e, "mirror table")), f = {
|
|
267
7
|
client: null,
|
|
268
|
-
database:
|
|
269
|
-
table:
|
|
270
|
-
mirrorTables:
|
|
271
|
-
mirrorPrimaryKey:
|
|
8
|
+
database: n(u.database ?? "default", "database"),
|
|
9
|
+
table: n(u.table ?? "events", "table"),
|
|
10
|
+
mirrorTables: d,
|
|
11
|
+
mirrorPrimaryKey: u.mirrorPrimaryKey ?? "id"
|
|
272
12
|
};
|
|
273
|
-
if (
|
|
274
|
-
let e =
|
|
275
|
-
|
|
276
|
-
maxSize: e.maxSize ??
|
|
277
|
-
flushIntervalMs: e.flushIntervalMs ??
|
|
13
|
+
if (u.batch !== !1) {
|
|
14
|
+
let e = u.batch ?? {};
|
|
15
|
+
f.batcher = a(f, {
|
|
16
|
+
maxSize: e.maxSize ?? c,
|
|
17
|
+
flushIntervalMs: e.flushIntervalMs ?? l
|
|
278
18
|
});
|
|
279
19
|
}
|
|
280
20
|
return {
|
|
281
21
|
kind: "clickhouse",
|
|
282
|
-
build: (e) =>
|
|
283
|
-
initialize: () =>
|
|
284
|
-
let
|
|
285
|
-
url:
|
|
286
|
-
database:
|
|
287
|
-
...
|
|
288
|
-
...
|
|
22
|
+
build: (e) => r(f),
|
|
23
|
+
initialize: () => s.gen(function* () {
|
|
24
|
+
let n = yield* s.promise(() => Promise.resolve(o({
|
|
25
|
+
url: u.url,
|
|
26
|
+
database: f.database,
|
|
27
|
+
...u.username === void 0 ? {} : { username: u.username },
|
|
28
|
+
...u.password === void 0 ? {} : { password: u.password }
|
|
289
29
|
})));
|
|
290
|
-
yield*
|
|
291
|
-
for (let e of
|
|
292
|
-
|
|
30
|
+
yield* s.promise(() => n.ping()), yield* s.promise(() => n.command({ query: e(f.database, f.table) }));
|
|
31
|
+
for (let e of f.mirrorTables) yield* s.promise(() => n.command({ query: t(f.database, e) }));
|
|
32
|
+
f.client = n;
|
|
293
33
|
}),
|
|
294
|
-
dispose: () =>
|
|
295
|
-
|
|
34
|
+
dispose: () => s.gen(function* () {
|
|
35
|
+
f.batcher && (yield* s.promise(() => f.batcher.flush())), yield* i(f);
|
|
296
36
|
})
|
|
297
37
|
};
|
|
298
38
|
};
|
|
299
39
|
//#endregion
|
|
300
|
-
export {
|
|
40
|
+
export { e as EVENTS_TABLE_DDL, t as MIRROR_TABLE_DDL, u as clickhouseAnalytics };
|
package/dist/sink.js
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import { Effect as e } from "effect";
|
|
2
|
+
import { createLogger as t } from "@voltro/logger";
|
|
3
|
+
import { AnalyticsError as n } from "@voltro/runtime";
|
|
4
|
+
//#region src/sink.ts
|
|
5
|
+
var r = t({ scope: "@voltro/plugin-clickhouse" }), i = "clickhouse", a = (e) => `voltro_mirror_${e}`, o = (e, t) => `
|
|
6
|
+
CREATE TABLE IF NOT EXISTS ${e}.${a(t)} (
|
|
7
|
+
id String,
|
|
8
|
+
data String CODEC(ZSTD(1)),
|
|
9
|
+
version UInt64,
|
|
10
|
+
is_deleted UInt8 DEFAULT 0
|
|
11
|
+
) ENGINE = ReplacingMergeTree(version)
|
|
12
|
+
ORDER BY id
|
|
13
|
+
`.trim(), s = (e, t) => `
|
|
14
|
+
CREATE TABLE IF NOT EXISTS ${e}.${t} (
|
|
15
|
+
id UUID DEFAULT generateUUIDv7(),
|
|
16
|
+
name LowCardinality(String),
|
|
17
|
+
subject_id Nullable(String),
|
|
18
|
+
tenant_id Nullable(String),
|
|
19
|
+
properties String CODEC(ZSTD(1)),
|
|
20
|
+
occurred_at DateTime64(3),
|
|
21
|
+
ingested_at DateTime64(3) DEFAULT now64()
|
|
22
|
+
) ENGINE = MergeTree
|
|
23
|
+
ORDER BY (name, occurred_at, id)
|
|
24
|
+
`.trim(), c = (e) => (t) => t instanceof n ? new n({
|
|
25
|
+
provider: i,
|
|
26
|
+
operation: e,
|
|
27
|
+
cause: t.cause
|
|
28
|
+
}) : new n({
|
|
29
|
+
provider: i,
|
|
30
|
+
operation: e,
|
|
31
|
+
cause: t instanceof Error ? `${t.name}: ${t.message}` : String(t)
|
|
32
|
+
}), l = (t) => (r) => r.pipe(e.catchDefect((t) => t instanceof n ? e.fail(t) : e.die(t)), e.mapError(c(t))), u = /^[A-Za-z_][A-Za-z0-9_]*$/, d = (e, t) => {
|
|
33
|
+
if (!u.test(e)) throw new n({
|
|
34
|
+
provider: i,
|
|
35
|
+
operation: "config",
|
|
36
|
+
cause: `invalid ${t} ${JSON.stringify(e)} — must match ${u.source} (letters, digits, underscore; no dots/quotes/spaces)`
|
|
37
|
+
});
|
|
38
|
+
return e;
|
|
39
|
+
}, f = /['\\]/, p = (e) => {
|
|
40
|
+
for (let t = 0; t < e.length; t++) if (e.charCodeAt(t) < 32) return !0;
|
|
41
|
+
return !1;
|
|
42
|
+
}, m = (e) => (t) => {
|
|
43
|
+
if (f.test(t) || p(t)) throw new n({
|
|
44
|
+
provider: i,
|
|
45
|
+
operation: e,
|
|
46
|
+
cause: `invalid property key ${JSON.stringify(t)} — must not contain quotes, backslashes, or control characters`
|
|
47
|
+
});
|
|
48
|
+
return t;
|
|
49
|
+
}, h = (t) => t.client === null ? e.fail(new n({
|
|
50
|
+
provider: i,
|
|
51
|
+
operation: "ensureClient",
|
|
52
|
+
cause: "client not initialized — spec.initialize must run before queries"
|
|
53
|
+
})) : e.succeed(t.client), g = (t) => e.gen(function* () {
|
|
54
|
+
let n = t.client;
|
|
55
|
+
t.client = null, n !== null && (yield* e.tryPromise(() => n.close()).pipe(e.catch((t) => e.logWarning("clickhouse analytics sink teardown failed", {
|
|
56
|
+
provider: i,
|
|
57
|
+
err: t
|
|
58
|
+
}))));
|
|
59
|
+
}), _ = (e) => ({
|
|
60
|
+
name: e.name,
|
|
61
|
+
subject_id: e.subjectId ?? null,
|
|
62
|
+
tenant_id: e.tenantId ?? null,
|
|
63
|
+
properties: JSON.stringify(e.properties),
|
|
64
|
+
occurred_at: (e.occurredAt ?? /* @__PURE__ */ new Date()).toISOString()
|
|
65
|
+
}), v = (e, t) => {
|
|
66
|
+
let n = [], i, a = () => {
|
|
67
|
+
i !== void 0 && (clearTimeout(i), i = void 0);
|
|
68
|
+
}, o = async () => {
|
|
69
|
+
if (a(), n.length === 0) return;
|
|
70
|
+
let t = n;
|
|
71
|
+
n = [];
|
|
72
|
+
let i = e.client;
|
|
73
|
+
if (i === null) r.warn("clickhouse batch flush skipped — client not initialized; events dropped", { dropped: t.length });
|
|
74
|
+
else try {
|
|
75
|
+
await i.insert({
|
|
76
|
+
table: `${e.database}.${e.table}`,
|
|
77
|
+
values: t,
|
|
78
|
+
format: "JSONEachRow"
|
|
79
|
+
});
|
|
80
|
+
} catch (e) {
|
|
81
|
+
r.warn("clickhouse batch flush failed — dropped events", { dropped: t.length }, e);
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
return {
|
|
85
|
+
add: (e) => {
|
|
86
|
+
n.push(_(e)), n.length >= t.maxSize ? o() : i === void 0 && (i = setTimeout(() => {
|
|
87
|
+
o();
|
|
88
|
+
}, t.flushIntervalMs), i.unref?.());
|
|
89
|
+
},
|
|
90
|
+
flush: o
|
|
91
|
+
};
|
|
92
|
+
}, y = (e) => {
|
|
93
|
+
switch (e) {
|
|
94
|
+
case "minute": return "toStartOfMinute";
|
|
95
|
+
case "hour": return "toStartOfHour";
|
|
96
|
+
case "day": return "toStartOfDay";
|
|
97
|
+
case "week": return "toStartOfWeek";
|
|
98
|
+
case "month": return "toStartOfMonth";
|
|
99
|
+
}
|
|
100
|
+
}, b = (e, t) => `JSONExtractString(properties, '${m(e)(t)}')`, x = (e, t) => `JSONExtractFloat(properties, '${m(e)(t)}')`, S = (e, t) => t === "count" ? "count()" : t === "unique" ? "uniqExact(subject_id)" : "sum" in t ? `sum(${x(e, t.sum)})` : `avg(${x(e, t.avg)})`, C = (e, t) => t === "count" ? "count()" : `sum(${x(e, t.sum)})`, w = (e, t, n, r, i) => {
|
|
101
|
+
let a = {
|
|
102
|
+
event: t,
|
|
103
|
+
from: n.from.toISOString(),
|
|
104
|
+
...n.to === void 0 ? {} : { to: n.to.toISOString() }
|
|
105
|
+
}, o = "name = {event:String} AND occurred_at >= {from:DateTime64(3)}" + (n.to === void 0 ? "" : " AND occurred_at < {to:DateTime64(3)}");
|
|
106
|
+
if (i != null && (a.tenant = i, o += " AND tenant_id = {tenant:String}"), r !== void 0) {
|
|
107
|
+
let t = 0;
|
|
108
|
+
for (let [n, i] of Object.entries(r)) {
|
|
109
|
+
let r = `f${t++}`;
|
|
110
|
+
a[r] = String(i), o += ` AND ${b(e, n)} = {${r}:String}`;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return {
|
|
114
|
+
sql: o,
|
|
115
|
+
params: a
|
|
116
|
+
};
|
|
117
|
+
}, T = (t) => {
|
|
118
|
+
let n = (n, r) => e.gen(function* () {
|
|
119
|
+
d(n, "mirror table");
|
|
120
|
+
let i = yield* h(t);
|
|
121
|
+
yield* e.tryPromise({
|
|
122
|
+
try: () => i.insert({
|
|
123
|
+
table: `${t.database}.${a(n)}`,
|
|
124
|
+
values: [r],
|
|
125
|
+
format: "JSONEachRow"
|
|
126
|
+
}),
|
|
127
|
+
catch: (e) => e
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
return {
|
|
131
|
+
tables: t.mirrorTables,
|
|
132
|
+
primaryKey: t.mirrorPrimaryKey,
|
|
133
|
+
upsert: ({ table: r, row: i, version: a }) => e.gen(function* () {
|
|
134
|
+
let e = String(i[t.mirrorPrimaryKey]);
|
|
135
|
+
yield* n(r, {
|
|
136
|
+
id: e,
|
|
137
|
+
data: JSON.stringify(i),
|
|
138
|
+
version: a,
|
|
139
|
+
is_deleted: 0
|
|
140
|
+
});
|
|
141
|
+
}).pipe(l("mirror.upsert")),
|
|
142
|
+
remove: ({ table: t, primaryKeyValue: r, version: i }) => e.gen(function* () {
|
|
143
|
+
yield* n(t, {
|
|
144
|
+
id: String(r),
|
|
145
|
+
data: "{}",
|
|
146
|
+
version: i,
|
|
147
|
+
is_deleted: 1
|
|
148
|
+
});
|
|
149
|
+
}).pipe(l("mirror.remove")),
|
|
150
|
+
maxVersion: ({ table: n, primaryKeyValue: r }) => e.gen(function* () {
|
|
151
|
+
d(String(n), "mirror table");
|
|
152
|
+
let i = yield* h(t), o = `SELECT max(version) AS v, count() AS n FROM ${t.database}.${a(String(n))} WHERE id = {id: String} FORMAT JSON`, s = yield* e.tryPromise({
|
|
153
|
+
try: () => i.query({
|
|
154
|
+
query: o,
|
|
155
|
+
query_params: { id: String(r) }
|
|
156
|
+
}),
|
|
157
|
+
catch: (e) => e
|
|
158
|
+
}), { data: c } = yield* e.tryPromise({
|
|
159
|
+
try: () => s.json(),
|
|
160
|
+
catch: (e) => e
|
|
161
|
+
}), l = c[0];
|
|
162
|
+
return l === void 0 || Number(l.n) === 0 || l.v === null ? null : Number(l.v);
|
|
163
|
+
}).pipe(l("mirror.maxVersion"))
|
|
164
|
+
};
|
|
165
|
+
}, E = (t) => {
|
|
166
|
+
let n = t.mirrorTables.length > 0 ? T(t) : void 0;
|
|
167
|
+
return {
|
|
168
|
+
info: {
|
|
169
|
+
provider: i,
|
|
170
|
+
capabilities: {
|
|
171
|
+
track: !0,
|
|
172
|
+
aggregate: !0,
|
|
173
|
+
timeseries: !0,
|
|
174
|
+
topN: !0,
|
|
175
|
+
mirror: n !== void 0
|
|
176
|
+
},
|
|
177
|
+
detail: {
|
|
178
|
+
database: t.database,
|
|
179
|
+
table: t.table,
|
|
180
|
+
...t.mirrorTables.length > 0 ? { mirrorTables: t.mirrorTables } : {},
|
|
181
|
+
...t.batcher === void 0 ? {} : { batching: !0 }
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
track: (n) => t.batcher ? e.sync(() => t.batcher.add(n)) : e.gen(function* () {
|
|
185
|
+
let r = yield* h(t);
|
|
186
|
+
yield* e.tryPromise({
|
|
187
|
+
try: () => r.insert({
|
|
188
|
+
table: `${t.database}.${t.table}`,
|
|
189
|
+
values: [_(n)],
|
|
190
|
+
format: "JSONEachRow"
|
|
191
|
+
}),
|
|
192
|
+
catch: (e) => e
|
|
193
|
+
});
|
|
194
|
+
}).pipe(l("track")),
|
|
195
|
+
aggregate: (n) => e.gen(function* () {
|
|
196
|
+
let r = yield* h(t), i = w("aggregate", n.event, n.range, n.filter, n.tenantId), a = `SELECT ${S("aggregate", n.metric)} AS value FROM ${t.database}.${t.table} WHERE ${i.sql} FORMAT JSON`, o = yield* e.tryPromise({
|
|
197
|
+
try: () => r.query({
|
|
198
|
+
query: a,
|
|
199
|
+
query_params: i.params
|
|
200
|
+
}),
|
|
201
|
+
catch: (e) => e
|
|
202
|
+
}), { data: s } = yield* e.tryPromise({
|
|
203
|
+
try: () => o.json(),
|
|
204
|
+
catch: (e) => e
|
|
205
|
+
}), c = s[0]?.value ?? 0;
|
|
206
|
+
return typeof c == "number" ? c : Number(c ?? 0);
|
|
207
|
+
}).pipe(l("aggregate")),
|
|
208
|
+
timeseries: (n) => e.gen(function* () {
|
|
209
|
+
let r = yield* h(t), i = w("timeseries", n.event, n.range, n.filter, n.tenantId), a = `
|
|
210
|
+
SELECT ${`${y(n.bucket)}(occurred_at)`} AS bucket_start, ${S("timeseries", n.metric)} AS value
|
|
211
|
+
FROM ${t.database}.${t.table}
|
|
212
|
+
WHERE ${i.sql}
|
|
213
|
+
GROUP BY bucket_start
|
|
214
|
+
ORDER BY bucket_start ASC
|
|
215
|
+
FORMAT JSON
|
|
216
|
+
`, o = yield* e.tryPromise({
|
|
217
|
+
try: () => r.query({
|
|
218
|
+
query: a,
|
|
219
|
+
query_params: i.params
|
|
220
|
+
}),
|
|
221
|
+
catch: (e) => e
|
|
222
|
+
}), { data: s } = yield* e.tryPromise({
|
|
223
|
+
try: () => o.json(),
|
|
224
|
+
catch: (e) => e
|
|
225
|
+
});
|
|
226
|
+
return s.map((e) => ({
|
|
227
|
+
bucketStart: new Date(e.bucket_start),
|
|
228
|
+
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
229
|
+
}));
|
|
230
|
+
}).pipe(l("timeseries")),
|
|
231
|
+
topN: (n) => e.gen(function* () {
|
|
232
|
+
let r = yield* h(t), i = w("topN", n.event, n.range, n.filter, n.tenantId), a = Math.max(1, Math.floor(n.n)), o = b("topN", n.groupBy), s = `
|
|
233
|
+
SELECT ${o} AS key, ${C("topN", n.metric)} AS value
|
|
234
|
+
FROM ${t.database}.${t.table}
|
|
235
|
+
WHERE ${i.sql}
|
|
236
|
+
GROUP BY ${o}
|
|
237
|
+
ORDER BY value DESC
|
|
238
|
+
LIMIT ${a}
|
|
239
|
+
FORMAT JSON
|
|
240
|
+
`, c = yield* e.tryPromise({
|
|
241
|
+
try: () => r.query({
|
|
242
|
+
query: s,
|
|
243
|
+
query_params: i.params
|
|
244
|
+
}),
|
|
245
|
+
catch: (e) => e
|
|
246
|
+
}), { data: l } = yield* e.tryPromise({
|
|
247
|
+
try: () => c.json(),
|
|
248
|
+
catch: (e) => e
|
|
249
|
+
});
|
|
250
|
+
return l.filter((e) => e.key !== null).map((e) => ({
|
|
251
|
+
key: e.key,
|
|
252
|
+
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
253
|
+
}));
|
|
254
|
+
}).pipe(l("topN")),
|
|
255
|
+
...n === void 0 ? {} : { mirror: n }
|
|
256
|
+
};
|
|
257
|
+
};
|
|
258
|
+
//#endregion
|
|
259
|
+
export { s as EVENTS_TABLE_DDL, o as MIRROR_TABLE_DDL, d as assertIdentifier, E as buildClickhouseSink, g as closeClickhouseState, v as makeEventBatcher, a as mirrorTableName };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voltro/plugin-clickhouse",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.86.0",
|
|
4
4
|
"description": "ClickHouse AnalyticsSink. Native HTTP ingest into a ClickHouse cluster — self-hosted OR ClickHouse Cloud. Production-grade OLAP for high-cardinality events with millisecond aggregate queries over billions of rows.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"voltro",
|
|
@@ -34,11 +34,11 @@
|
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
36
|
"@clickhouse/client": "^1.23.1",
|
|
37
|
-
"@voltro/logger": "0.
|
|
38
|
-
"@voltro/runtime": "0.
|
|
37
|
+
"@voltro/logger": "0.86.0",
|
|
38
|
+
"@voltro/runtime": "0.86.0"
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
41
|
-
"effect": "
|
|
41
|
+
"effect": "4.0.1"
|
|
42
42
|
},
|
|
43
43
|
"publishConfig": {
|
|
44
44
|
"access": "public"
|