@voltro/plugin-clickhouse 0.85.0 → 0.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5987 -0
- package/README.md +27 -0
- package/THIRD-PARTY-NOTICES.md +864 -253
- package/dist/index.js +48 -289
- package/dist/sink.js +259 -0
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,300 +1,59 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
) ENGINE = ReplacingMergeTree(version)
|
|
13
|
-
ORDER BY id
|
|
14
|
-
`.trim(), c = (e, t) => `
|
|
15
|
-
CREATE TABLE IF NOT EXISTS ${e}.${t} (
|
|
16
|
-
id UUID DEFAULT generateUUIDv7(),
|
|
17
|
-
name LowCardinality(String),
|
|
18
|
-
subject_id Nullable(String),
|
|
19
|
-
tenant_id Nullable(String),
|
|
20
|
-
properties String CODEC(ZSTD(1)),
|
|
21
|
-
occurred_at DateTime64(3),
|
|
22
|
-
ingested_at DateTime64(3) DEFAULT now64()
|
|
23
|
-
) ENGINE = MergeTree
|
|
24
|
-
ORDER BY (name, occurred_at, id)
|
|
25
|
-
`.trim(), l = (e) => (t) => t instanceof r ? new r({
|
|
26
|
-
provider: a,
|
|
27
|
-
operation: e,
|
|
28
|
-
cause: t.cause
|
|
29
|
-
}) : new r({
|
|
30
|
-
provider: a,
|
|
31
|
-
operation: e,
|
|
32
|
-
cause: t instanceof Error ? `${t.name}: ${t.message}` : String(t)
|
|
33
|
-
}), u = (e) => (n) => n.pipe(t.catchDefect((e) => e instanceof r ? t.fail(e) : t.die(e)), t.mapError(l(e))), d = /^[A-Za-z_][A-Za-z0-9_]*$/, f = (e, t) => {
|
|
34
|
-
if (!d.test(e)) throw new r({
|
|
35
|
-
provider: a,
|
|
36
|
-
operation: "config",
|
|
37
|
-
cause: `invalid ${t} ${JSON.stringify(e)} — must match ${d.source} (letters, digits, underscore; no dots/quotes/spaces)`
|
|
38
|
-
});
|
|
39
|
-
return e;
|
|
40
|
-
}, p = /['\\]/, m = (e) => {
|
|
41
|
-
for (let t = 0; t < e.length; t++) if (e.charCodeAt(t) < 32) return !0;
|
|
42
|
-
return !1;
|
|
43
|
-
}, h = (e) => (t) => {
|
|
44
|
-
if (p.test(t) || m(t)) throw new r({
|
|
45
|
-
provider: a,
|
|
46
|
-
operation: e,
|
|
47
|
-
cause: `invalid property key ${JSON.stringify(t)} — must not contain quotes, backslashes, or control characters`
|
|
48
|
-
});
|
|
49
|
-
return t;
|
|
50
|
-
}, g = (e) => e.client === null ? t.fail(new r({
|
|
51
|
-
provider: a,
|
|
52
|
-
operation: "ensureClient",
|
|
53
|
-
cause: "client not initialized — spec.initialize must run before queries"
|
|
54
|
-
})) : t.succeed(e.client), _ = (e) => t.gen(function* () {
|
|
55
|
-
let n = e.client;
|
|
56
|
-
e.client = null, n !== null && (yield* t.tryPromise(() => n.close()).pipe(t.catch((e) => t.logWarning("clickhouse analytics sink teardown failed", {
|
|
57
|
-
provider: a,
|
|
58
|
-
err: e
|
|
59
|
-
}))));
|
|
60
|
-
}), v = (e) => ({
|
|
61
|
-
name: e.name,
|
|
62
|
-
subject_id: e.subjectId ?? null,
|
|
63
|
-
tenant_id: e.tenantId ?? null,
|
|
64
|
-
properties: JSON.stringify(e.properties),
|
|
65
|
-
occurred_at: (e.occurredAt ?? /* @__PURE__ */ new Date()).toISOString()
|
|
66
|
-
}), y = (e, t) => {
|
|
67
|
-
let n = [], r, a = () => {
|
|
68
|
-
r !== void 0 && (clearTimeout(r), r = void 0);
|
|
69
|
-
}, o = async () => {
|
|
70
|
-
if (a(), n.length === 0) return;
|
|
71
|
-
let t = n;
|
|
72
|
-
n = [];
|
|
73
|
-
let r = e.client;
|
|
74
|
-
if (r === null) {
|
|
75
|
-
i.warn("clickhouse batch flush skipped — client not initialized; events dropped", { dropped: t.length });
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
try {
|
|
79
|
-
await r.insert({
|
|
80
|
-
table: `${e.database}.${e.table}`,
|
|
81
|
-
values: t,
|
|
82
|
-
format: "JSONEachRow"
|
|
83
|
-
});
|
|
84
|
-
} catch (e) {
|
|
85
|
-
i.warn("clickhouse batch flush failed — dropped events", { dropped: t.length }, e);
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
return {
|
|
89
|
-
add: (e) => {
|
|
90
|
-
if (n.push(v(e)), n.length >= t.maxSize) {
|
|
91
|
-
o();
|
|
92
|
-
return;
|
|
93
|
-
}
|
|
94
|
-
r === void 0 && (r = setTimeout(() => {
|
|
95
|
-
o();
|
|
96
|
-
}, t.flushIntervalMs), r.unref?.());
|
|
97
|
-
},
|
|
98
|
-
flush: o
|
|
99
|
-
};
|
|
100
|
-
}, b = (e) => {
|
|
101
|
-
switch (e) {
|
|
102
|
-
case "minute": return "toStartOfMinute";
|
|
103
|
-
case "hour": return "toStartOfHour";
|
|
104
|
-
case "day": return "toStartOfDay";
|
|
105
|
-
case "week": return "toStartOfWeek";
|
|
106
|
-
case "month": return "toStartOfMonth";
|
|
107
|
-
}
|
|
108
|
-
}, x = (e, t) => `JSONExtractString(properties, '${h(e)(t)}')`, S = (e, t) => `JSONExtractFloat(properties, '${h(e)(t)}')`, C = (e, t) => t === "count" ? "count()" : t === "unique" ? "uniqExact(subject_id)" : "sum" in t ? `sum(${S(e, t.sum)})` : `avg(${S(e, t.avg)})`, w = (e, t) => t === "count" ? "count()" : `sum(${S(e, t.sum)})`, T = (e, t, n, r, i) => {
|
|
109
|
-
let a = {
|
|
110
|
-
event: t,
|
|
111
|
-
from: n.from.toISOString(),
|
|
112
|
-
...n.to === void 0 ? {} : { to: n.to.toISOString() }
|
|
113
|
-
}, o = "name = {event:String} AND occurred_at >= {from:DateTime64(3)}" + (n.to === void 0 ? "" : " AND occurred_at < {to:DateTime64(3)}");
|
|
114
|
-
if (i != null && (a.tenant = i, o += " AND tenant_id = {tenant:String}"), r !== void 0) {
|
|
115
|
-
let t = 0;
|
|
116
|
-
for (let [n, i] of Object.entries(r)) {
|
|
117
|
-
let r = `f${t++}`;
|
|
118
|
-
a[r] = String(i), o += ` AND ${x(e, n)} = {${r}:String}`;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
return {
|
|
122
|
-
sql: o,
|
|
123
|
-
params: a
|
|
1
|
+
import { EVENTS_TABLE_DDL as e, MIRROR_TABLE_DDL as t, assertIdentifier as n, buildClickhouseSink as r, closeClickhouseState as i, makeEventBatcher as a } from "./sink.js";
|
|
2
|
+
import { createClient as o } from "@clickhouse/client";
|
|
3
|
+
import { Effect as s } from "effect";
|
|
4
|
+
//#region src/index.ts
|
|
5
|
+
var c = 20, l = 5e3, u = (u) => {
|
|
6
|
+
let d = (u.mirrorTables ?? []).map((e) => n(e, "mirror table")), f = {
|
|
7
|
+
client: null,
|
|
8
|
+
database: n(u.database ?? "default", "database"),
|
|
9
|
+
table: n(u.table ?? "events", "table"),
|
|
10
|
+
mirrorTables: d,
|
|
11
|
+
mirrorPrimaryKey: u.mirrorPrimaryKey ?? "id"
|
|
124
12
|
};
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
f(
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
try: () => i.insert({
|
|
131
|
-
table: `${e.database}.${o(n)}`,
|
|
132
|
-
values: [r],
|
|
133
|
-
format: "JSONEachRow"
|
|
134
|
-
}),
|
|
135
|
-
catch: (e) => e
|
|
13
|
+
if (u.batch !== !1) {
|
|
14
|
+
let e = u.batch ?? {};
|
|
15
|
+
f.batcher = a(f, {
|
|
16
|
+
maxSize: e.maxSize ?? c,
|
|
17
|
+
flushIntervalMs: e.flushIntervalMs ?? l
|
|
136
18
|
});
|
|
137
|
-
}
|
|
138
|
-
return {
|
|
139
|
-
tables: e.mirrorTables,
|
|
140
|
-
primaryKey: e.mirrorPrimaryKey,
|
|
141
|
-
upsert: ({ table: r, row: i, version: a }) => t.gen(function* () {
|
|
142
|
-
let t = String(i[e.mirrorPrimaryKey]);
|
|
143
|
-
yield* n(r, {
|
|
144
|
-
id: t,
|
|
145
|
-
data: JSON.stringify(i),
|
|
146
|
-
version: a,
|
|
147
|
-
is_deleted: 0
|
|
148
|
-
});
|
|
149
|
-
}).pipe(u("mirror.upsert")),
|
|
150
|
-
remove: ({ table: e, primaryKeyValue: r, version: i }) => t.gen(function* () {
|
|
151
|
-
yield* n(e, {
|
|
152
|
-
id: String(r),
|
|
153
|
-
data: "{}",
|
|
154
|
-
version: i,
|
|
155
|
-
is_deleted: 1
|
|
156
|
-
});
|
|
157
|
-
}).pipe(u("mirror.remove")),
|
|
158
|
-
maxVersion: ({ table: n, primaryKeyValue: r }) => t.gen(function* () {
|
|
159
|
-
f(String(n), "mirror table");
|
|
160
|
-
let i = yield* g(e), a = `SELECT max(version) AS v, count() AS n FROM ${e.database}.${o(String(n))} WHERE id = {id: String} FORMAT JSON`, s = yield* t.tryPromise({
|
|
161
|
-
try: () => i.query({
|
|
162
|
-
query: a,
|
|
163
|
-
query_params: { id: String(r) }
|
|
164
|
-
}),
|
|
165
|
-
catch: (e) => e
|
|
166
|
-
}), { data: c } = yield* t.tryPromise({
|
|
167
|
-
try: () => s.json(),
|
|
168
|
-
catch: (e) => e
|
|
169
|
-
}), l = c[0];
|
|
170
|
-
return l === void 0 || Number(l.n) === 0 || l.v === null ? null : Number(l.v);
|
|
171
|
-
}).pipe(u("mirror.maxVersion"))
|
|
172
|
-
};
|
|
173
|
-
}, D = (e) => {
|
|
174
|
-
let n = e.mirrorTables.length > 0 ? E(e) : void 0;
|
|
19
|
+
}
|
|
175
20
|
return {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
}
|
|
191
|
-
},
|
|
192
|
-
track: (n) => e.batcher ? t.sync(() => e.batcher.add(n)) : t.gen(function* () {
|
|
193
|
-
let r = yield* g(e);
|
|
194
|
-
yield* t.tryPromise({
|
|
195
|
-
try: () => r.insert({
|
|
196
|
-
table: `${e.database}.${e.table}`,
|
|
197
|
-
values: [v(n)],
|
|
198
|
-
format: "JSONEachRow"
|
|
199
|
-
}),
|
|
200
|
-
catch: (e) => e
|
|
201
|
-
});
|
|
202
|
-
}).pipe(u("track")),
|
|
203
|
-
aggregate: (n) => t.gen(function* () {
|
|
204
|
-
let r = yield* g(e), i = T("aggregate", n.event, n.range, n.filter, n.tenantId), a = `SELECT ${C("aggregate", n.metric)} AS value FROM ${e.database}.${e.table} WHERE ${i.sql} FORMAT JSON`, o = yield* t.tryPromise({
|
|
205
|
-
try: () => r.query({
|
|
206
|
-
query: a,
|
|
207
|
-
query_params: i.params
|
|
21
|
+
kind: "clickhouse",
|
|
22
|
+
sql: "none",
|
|
23
|
+
build: () => r(f),
|
|
24
|
+
initialize: () => s.gen(function* () {
|
|
25
|
+
let n = yield* s.sync(() => o({
|
|
26
|
+
url: u.url,
|
|
27
|
+
database: f.database,
|
|
28
|
+
...u.username === void 0 ? {} : { username: u.username },
|
|
29
|
+
...u.password === void 0 ? {} : { password: u.password }
|
|
30
|
+
}));
|
|
31
|
+
if (f.client = n, !(yield* s.tryPromise({
|
|
32
|
+
try: (e) => n.ping({
|
|
33
|
+
select: !1,
|
|
34
|
+
abort_signal: e
|
|
208
35
|
}),
|
|
209
|
-
catch: (
|
|
210
|
-
})
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}).pipe(u("aggregate")),
|
|
216
|
-
timeseries: (n) => t.gen(function* () {
|
|
217
|
-
let r = yield* g(e), i = T("timeseries", n.event, n.range, n.filter, n.tenantId), a = `
|
|
218
|
-
SELECT ${`${b(n.bucket)}(occurred_at)`} AS bucket_start, ${C("timeseries", n.metric)} AS value
|
|
219
|
-
FROM ${e.database}.${e.table}
|
|
220
|
-
WHERE ${i.sql}
|
|
221
|
-
GROUP BY bucket_start
|
|
222
|
-
ORDER BY bucket_start ASC
|
|
223
|
-
FORMAT JSON
|
|
224
|
-
`, o = yield* t.tryPromise({
|
|
225
|
-
try: () => r.query({
|
|
226
|
-
query: a,
|
|
227
|
-
query_params: i.params
|
|
36
|
+
catch: () => /* @__PURE__ */ Error("ClickHouse readiness failed")
|
|
37
|
+
})).success) return yield* s.fail(/* @__PURE__ */ Error("ClickHouse readiness failed"));
|
|
38
|
+
yield* s.tryPromise({
|
|
39
|
+
try: (t) => n.command({
|
|
40
|
+
query: e(f.database, f.table),
|
|
41
|
+
abort_signal: t
|
|
228
42
|
}),
|
|
229
|
-
catch: (
|
|
230
|
-
}), { data: s } = yield* t.tryPromise({
|
|
231
|
-
try: () => o.json(),
|
|
232
|
-
catch: (e) => e
|
|
43
|
+
catch: () => /* @__PURE__ */ Error("ClickHouse schema initialization failed")
|
|
233
44
|
});
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
}).pipe(u("timeseries")),
|
|
239
|
-
topN: (n) => t.gen(function* () {
|
|
240
|
-
let r = yield* g(e), i = T("topN", n.event, n.range, n.filter, n.tenantId), a = Math.max(1, Math.floor(n.n)), o = x("topN", n.groupBy), s = `
|
|
241
|
-
SELECT ${o} AS key, ${w("topN", n.metric)} AS value
|
|
242
|
-
FROM ${e.database}.${e.table}
|
|
243
|
-
WHERE ${i.sql}
|
|
244
|
-
GROUP BY ${o}
|
|
245
|
-
ORDER BY value DESC
|
|
246
|
-
LIMIT ${a}
|
|
247
|
-
FORMAT JSON
|
|
248
|
-
`, c = yield* t.tryPromise({
|
|
249
|
-
try: () => r.query({
|
|
250
|
-
query: s,
|
|
251
|
-
query_params: i.params
|
|
45
|
+
for (let e of f.mirrorTables) yield* s.tryPromise({
|
|
46
|
+
try: (r) => n.command({
|
|
47
|
+
query: t(f.database, e),
|
|
48
|
+
abort_signal: r
|
|
252
49
|
}),
|
|
253
|
-
catch: (
|
|
254
|
-
}), { data: l } = yield* t.tryPromise({
|
|
255
|
-
try: () => c.json(),
|
|
256
|
-
catch: (e) => e
|
|
50
|
+
catch: () => /* @__PURE__ */ Error("ClickHouse mirror initialization failed")
|
|
257
51
|
});
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
}).pipe(u("topN")),
|
|
263
|
-
...n === void 0 ? {} : { mirror: n }
|
|
264
|
-
};
|
|
265
|
-
}, O = 20, k = 5e3, A = (n) => {
|
|
266
|
-
let r = (n.mirrorTables ?? []).map((e) => f(e, "mirror table")), i = {
|
|
267
|
-
client: null,
|
|
268
|
-
database: f(n.database ?? "default", "database"),
|
|
269
|
-
table: f(n.table ?? "events", "table"),
|
|
270
|
-
mirrorTables: r,
|
|
271
|
-
mirrorPrimaryKey: n.mirrorPrimaryKey ?? "id"
|
|
272
|
-
};
|
|
273
|
-
if (n.batch !== !1) {
|
|
274
|
-
let e = n.batch ?? {};
|
|
275
|
-
i.batcher = y(i, {
|
|
276
|
-
maxSize: e.maxSize ?? O,
|
|
277
|
-
flushIntervalMs: e.flushIntervalMs ?? k
|
|
278
|
-
});
|
|
279
|
-
}
|
|
280
|
-
return {
|
|
281
|
-
kind: "clickhouse",
|
|
282
|
-
build: (e) => D(i),
|
|
283
|
-
initialize: () => t.gen(function* () {
|
|
284
|
-
let r = yield* t.promise(() => Promise.resolve(e({
|
|
285
|
-
url: n.url,
|
|
286
|
-
database: i.database,
|
|
287
|
-
...n.username === void 0 ? {} : { username: n.username },
|
|
288
|
-
...n.password === void 0 ? {} : { password: n.password }
|
|
289
|
-
})));
|
|
290
|
-
yield* t.promise(() => r.ping()), yield* t.promise(() => r.command({ query: c(i.database, i.table) }));
|
|
291
|
-
for (let e of i.mirrorTables) yield* t.promise(() => r.command({ query: s(i.database, e) }));
|
|
292
|
-
i.client = r;
|
|
293
|
-
}),
|
|
294
|
-
dispose: () => t.gen(function* () {
|
|
295
|
-
i.batcher && (yield* t.promise(() => i.batcher.flush())), yield* _(i);
|
|
296
|
-
})
|
|
52
|
+
}).pipe(s.onError(() => i(f))),
|
|
53
|
+
dispose: () => s.gen(function* () {
|
|
54
|
+
f.batcher && (yield* s.promise(() => f.batcher.flush()));
|
|
55
|
+
}).pipe(s.ensuring(i(f)))
|
|
297
56
|
};
|
|
298
57
|
};
|
|
299
58
|
//#endregion
|
|
300
|
-
export {
|
|
59
|
+
export { e as EVENTS_TABLE_DDL, t as MIRROR_TABLE_DDL, u as clickhouseAnalytics };
|
package/dist/sink.js
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import { Effect as e } from "effect";
|
|
2
|
+
import { createLogger as t } from "@voltro/logger";
|
|
3
|
+
import { AnalyticsError as n } from "@voltro/runtime";
|
|
4
|
+
//#region src/sink.ts
|
|
5
|
+
var r = t({ scope: "@voltro/plugin-clickhouse" }), i = "clickhouse", a = (e) => `voltro_mirror_${e}`, o = (e, t) => `
|
|
6
|
+
CREATE TABLE IF NOT EXISTS ${e}.${a(t)} (
|
|
7
|
+
id String,
|
|
8
|
+
data String CODEC(ZSTD(1)),
|
|
9
|
+
version UInt64,
|
|
10
|
+
is_deleted UInt8 DEFAULT 0
|
|
11
|
+
) ENGINE = ReplacingMergeTree(version)
|
|
12
|
+
ORDER BY id
|
|
13
|
+
`.trim(), s = (e, t) => `
|
|
14
|
+
CREATE TABLE IF NOT EXISTS ${e}.${t} (
|
|
15
|
+
id UUID DEFAULT generateUUIDv7(),
|
|
16
|
+
name LowCardinality(String),
|
|
17
|
+
subject_id Nullable(String),
|
|
18
|
+
tenant_id Nullable(String),
|
|
19
|
+
properties String CODEC(ZSTD(1)),
|
|
20
|
+
occurred_at DateTime64(3),
|
|
21
|
+
ingested_at DateTime64(3) DEFAULT now64()
|
|
22
|
+
) ENGINE = MergeTree
|
|
23
|
+
ORDER BY (name, occurred_at, id)
|
|
24
|
+
`.trim(), c = (e) => (t) => t instanceof n ? new n({
|
|
25
|
+
provider: i,
|
|
26
|
+
operation: e,
|
|
27
|
+
cause: t.cause
|
|
28
|
+
}) : new n({
|
|
29
|
+
provider: i,
|
|
30
|
+
operation: e,
|
|
31
|
+
cause: t instanceof Error ? `${t.name}: ${t.message}` : String(t)
|
|
32
|
+
}), l = (t) => (r) => r.pipe(e.catchDefect((t) => t instanceof n ? e.fail(t) : e.die(t)), e.mapError(c(t))), u = /^[A-Za-z_][A-Za-z0-9_]*$/, d = (e, t) => {
|
|
33
|
+
if (!u.test(e)) throw new n({
|
|
34
|
+
provider: i,
|
|
35
|
+
operation: "config",
|
|
36
|
+
cause: `invalid ${t} ${JSON.stringify(e)} — must match ${u.source} (letters, digits, underscore; no dots/quotes/spaces)`
|
|
37
|
+
});
|
|
38
|
+
return e;
|
|
39
|
+
}, f = /['\\]/, p = (e) => {
|
|
40
|
+
for (let t = 0; t < e.length; t++) if (e.charCodeAt(t) < 32) return !0;
|
|
41
|
+
return !1;
|
|
42
|
+
}, m = (e) => (t) => {
|
|
43
|
+
if (f.test(t) || p(t)) throw new n({
|
|
44
|
+
provider: i,
|
|
45
|
+
operation: e,
|
|
46
|
+
cause: `invalid property key ${JSON.stringify(t)} — must not contain quotes, backslashes, or control characters`
|
|
47
|
+
});
|
|
48
|
+
return t;
|
|
49
|
+
}, h = (t) => t.client === null ? e.fail(new n({
|
|
50
|
+
provider: i,
|
|
51
|
+
operation: "ensureClient",
|
|
52
|
+
cause: "client not initialized — spec.initialize must run before queries"
|
|
53
|
+
})) : e.succeed(t.client), g = (t) => e.gen(function* () {
|
|
54
|
+
let n = t.client;
|
|
55
|
+
t.client = null, n !== null && (yield* e.tryPromise(() => n.close()).pipe(e.catch((t) => e.logWarning("clickhouse analytics sink teardown failed", {
|
|
56
|
+
provider: i,
|
|
57
|
+
err: t
|
|
58
|
+
}))));
|
|
59
|
+
}), _ = (e) => ({
|
|
60
|
+
name: e.name,
|
|
61
|
+
subject_id: e.subjectId ?? null,
|
|
62
|
+
tenant_id: e.tenantId ?? null,
|
|
63
|
+
properties: JSON.stringify(e.properties),
|
|
64
|
+
occurred_at: (e.occurredAt ?? /* @__PURE__ */ new Date()).toISOString()
|
|
65
|
+
}), v = (e, t) => {
|
|
66
|
+
let n = [], i, a = () => {
|
|
67
|
+
i !== void 0 && (clearTimeout(i), i = void 0);
|
|
68
|
+
}, o = async () => {
|
|
69
|
+
if (a(), n.length === 0) return;
|
|
70
|
+
let t = n;
|
|
71
|
+
n = [];
|
|
72
|
+
let i = e.client;
|
|
73
|
+
if (i === null) r.warn("clickhouse batch flush skipped — client not initialized; events dropped", { dropped: t.length });
|
|
74
|
+
else try {
|
|
75
|
+
await i.insert({
|
|
76
|
+
table: `${e.database}.${e.table}`,
|
|
77
|
+
values: t,
|
|
78
|
+
format: "JSONEachRow"
|
|
79
|
+
});
|
|
80
|
+
} catch (e) {
|
|
81
|
+
r.warn("clickhouse batch flush failed — dropped events", { dropped: t.length }, e);
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
return {
|
|
85
|
+
add: (e) => {
|
|
86
|
+
n.push(_(e)), n.length >= t.maxSize ? o() : i === void 0 && (i = setTimeout(() => {
|
|
87
|
+
o();
|
|
88
|
+
}, t.flushIntervalMs), i.unref?.());
|
|
89
|
+
},
|
|
90
|
+
flush: o
|
|
91
|
+
};
|
|
92
|
+
}, y = (e) => {
|
|
93
|
+
switch (e) {
|
|
94
|
+
case "minute": return "toStartOfMinute";
|
|
95
|
+
case "hour": return "toStartOfHour";
|
|
96
|
+
case "day": return "toStartOfDay";
|
|
97
|
+
case "week": return "toStartOfWeek";
|
|
98
|
+
case "month": return "toStartOfMonth";
|
|
99
|
+
}
|
|
100
|
+
}, b = (e, t) => `JSONExtractString(properties, '${m(e)(t)}')`, x = (e, t) => `JSONExtractFloat(properties, '${m(e)(t)}')`, S = (e, t) => t === "count" ? "count()" : t === "unique" ? "uniqExact(subject_id)" : "sum" in t ? `sum(${x(e, t.sum)})` : `avg(${x(e, t.avg)})`, C = (e, t) => t === "count" ? "count()" : `sum(${x(e, t.sum)})`, w = (e, t, n, r, i) => {
|
|
101
|
+
let a = {
|
|
102
|
+
event: t,
|
|
103
|
+
from: n.from.toISOString(),
|
|
104
|
+
...n.to === void 0 ? {} : { to: n.to.toISOString() }
|
|
105
|
+
}, o = "name = {event:String} AND occurred_at >= {from:DateTime64(3)}" + (n.to === void 0 ? "" : " AND occurred_at < {to:DateTime64(3)}");
|
|
106
|
+
if (i != null && (a.tenant = i, o += " AND tenant_id = {tenant:String}"), r !== void 0) {
|
|
107
|
+
let t = 0;
|
|
108
|
+
for (let [n, i] of Object.entries(r)) {
|
|
109
|
+
let r = `f${t++}`;
|
|
110
|
+
a[r] = String(i), o += ` AND ${b(e, n)} = {${r}:String}`;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return {
|
|
114
|
+
sql: o,
|
|
115
|
+
params: a
|
|
116
|
+
};
|
|
117
|
+
}, T = (t) => {
|
|
118
|
+
let n = (n, r) => e.gen(function* () {
|
|
119
|
+
d(n, "mirror table");
|
|
120
|
+
let i = yield* h(t);
|
|
121
|
+
yield* e.tryPromise({
|
|
122
|
+
try: () => i.insert({
|
|
123
|
+
table: `${t.database}.${a(n)}`,
|
|
124
|
+
values: [r],
|
|
125
|
+
format: "JSONEachRow"
|
|
126
|
+
}),
|
|
127
|
+
catch: (e) => e
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
return {
|
|
131
|
+
tables: t.mirrorTables,
|
|
132
|
+
primaryKey: t.mirrorPrimaryKey,
|
|
133
|
+
upsert: ({ table: r, row: i, version: a }) => e.gen(function* () {
|
|
134
|
+
let e = String(i[t.mirrorPrimaryKey]);
|
|
135
|
+
yield* n(r, {
|
|
136
|
+
id: e,
|
|
137
|
+
data: JSON.stringify(i),
|
|
138
|
+
version: a,
|
|
139
|
+
is_deleted: 0
|
|
140
|
+
});
|
|
141
|
+
}).pipe(l("mirror.upsert")),
|
|
142
|
+
remove: ({ table: t, primaryKeyValue: r, version: i }) => e.gen(function* () {
|
|
143
|
+
yield* n(t, {
|
|
144
|
+
id: String(r),
|
|
145
|
+
data: "{}",
|
|
146
|
+
version: i,
|
|
147
|
+
is_deleted: 1
|
|
148
|
+
});
|
|
149
|
+
}).pipe(l("mirror.remove")),
|
|
150
|
+
maxVersion: ({ table: n, primaryKeyValue: r }) => e.gen(function* () {
|
|
151
|
+
d(String(n), "mirror table");
|
|
152
|
+
let i = yield* h(t), o = `SELECT max(version) AS v, count() AS n FROM ${t.database}.${a(String(n))} WHERE id = {id: String} FORMAT JSON`, s = yield* e.tryPromise({
|
|
153
|
+
try: () => i.query({
|
|
154
|
+
query: o,
|
|
155
|
+
query_params: { id: String(r) }
|
|
156
|
+
}),
|
|
157
|
+
catch: (e) => e
|
|
158
|
+
}), { data: c } = yield* e.tryPromise({
|
|
159
|
+
try: () => s.json(),
|
|
160
|
+
catch: (e) => e
|
|
161
|
+
}), l = c[0];
|
|
162
|
+
return l === void 0 || Number(l.n) === 0 || l.v === null ? null : Number(l.v);
|
|
163
|
+
}).pipe(l("mirror.maxVersion"))
|
|
164
|
+
};
|
|
165
|
+
}, E = (t) => {
|
|
166
|
+
let n = t.mirrorTables.length > 0 ? T(t) : void 0;
|
|
167
|
+
return {
|
|
168
|
+
info: {
|
|
169
|
+
provider: i,
|
|
170
|
+
capabilities: {
|
|
171
|
+
track: !0,
|
|
172
|
+
aggregate: !0,
|
|
173
|
+
timeseries: !0,
|
|
174
|
+
topN: !0,
|
|
175
|
+
mirror: n !== void 0
|
|
176
|
+
},
|
|
177
|
+
detail: {
|
|
178
|
+
database: t.database,
|
|
179
|
+
table: t.table,
|
|
180
|
+
...t.mirrorTables.length > 0 ? { mirrorTables: t.mirrorTables } : {},
|
|
181
|
+
...t.batcher === void 0 ? {} : { batching: !0 }
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
track: (n) => t.batcher ? e.sync(() => t.batcher.add(n)) : e.gen(function* () {
|
|
185
|
+
let r = yield* h(t);
|
|
186
|
+
yield* e.tryPromise({
|
|
187
|
+
try: () => r.insert({
|
|
188
|
+
table: `${t.database}.${t.table}`,
|
|
189
|
+
values: [_(n)],
|
|
190
|
+
format: "JSONEachRow"
|
|
191
|
+
}),
|
|
192
|
+
catch: (e) => e
|
|
193
|
+
});
|
|
194
|
+
}).pipe(l("track")),
|
|
195
|
+
aggregate: (n) => e.gen(function* () {
|
|
196
|
+
let r = yield* h(t), i = w("aggregate", n.event, n.range, n.filter, n.tenantId), a = `SELECT ${S("aggregate", n.metric)} AS value FROM ${t.database}.${t.table} WHERE ${i.sql} FORMAT JSON`, o = yield* e.tryPromise({
|
|
197
|
+
try: () => r.query({
|
|
198
|
+
query: a,
|
|
199
|
+
query_params: i.params
|
|
200
|
+
}),
|
|
201
|
+
catch: (e) => e
|
|
202
|
+
}), { data: s } = yield* e.tryPromise({
|
|
203
|
+
try: () => o.json(),
|
|
204
|
+
catch: (e) => e
|
|
205
|
+
}), c = s[0]?.value ?? 0;
|
|
206
|
+
return typeof c == "number" ? c : Number(c ?? 0);
|
|
207
|
+
}).pipe(l("aggregate")),
|
|
208
|
+
timeseries: (n) => e.gen(function* () {
|
|
209
|
+
let r = yield* h(t), i = w("timeseries", n.event, n.range, n.filter, n.tenantId), a = `
|
|
210
|
+
SELECT ${`${y(n.bucket)}(occurred_at)`} AS bucket_start, ${S("timeseries", n.metric)} AS value
|
|
211
|
+
FROM ${t.database}.${t.table}
|
|
212
|
+
WHERE ${i.sql}
|
|
213
|
+
GROUP BY bucket_start
|
|
214
|
+
ORDER BY bucket_start ASC
|
|
215
|
+
FORMAT JSON
|
|
216
|
+
`, o = yield* e.tryPromise({
|
|
217
|
+
try: () => r.query({
|
|
218
|
+
query: a,
|
|
219
|
+
query_params: i.params
|
|
220
|
+
}),
|
|
221
|
+
catch: (e) => e
|
|
222
|
+
}), { data: s } = yield* e.tryPromise({
|
|
223
|
+
try: () => o.json(),
|
|
224
|
+
catch: (e) => e
|
|
225
|
+
});
|
|
226
|
+
return s.map((e) => ({
|
|
227
|
+
bucketStart: new Date(e.bucket_start),
|
|
228
|
+
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
229
|
+
}));
|
|
230
|
+
}).pipe(l("timeseries")),
|
|
231
|
+
topN: (n) => e.gen(function* () {
|
|
232
|
+
let r = yield* h(t), i = w("topN", n.event, n.range, n.filter, n.tenantId), a = Math.max(1, Math.floor(n.n)), o = b("topN", n.groupBy), s = `
|
|
233
|
+
SELECT ${o} AS key, ${C("topN", n.metric)} AS value
|
|
234
|
+
FROM ${t.database}.${t.table}
|
|
235
|
+
WHERE ${i.sql}
|
|
236
|
+
GROUP BY ${o}
|
|
237
|
+
ORDER BY value DESC
|
|
238
|
+
LIMIT ${a}
|
|
239
|
+
FORMAT JSON
|
|
240
|
+
`, c = yield* e.tryPromise({
|
|
241
|
+
try: () => r.query({
|
|
242
|
+
query: s,
|
|
243
|
+
query_params: i.params
|
|
244
|
+
}),
|
|
245
|
+
catch: (e) => e
|
|
246
|
+
}), { data: l } = yield* e.tryPromise({
|
|
247
|
+
try: () => c.json(),
|
|
248
|
+
catch: (e) => e
|
|
249
|
+
});
|
|
250
|
+
return l.filter((e) => e.key !== null).map((e) => ({
|
|
251
|
+
key: e.key,
|
|
252
|
+
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
253
|
+
}));
|
|
254
|
+
}).pipe(l("topN")),
|
|
255
|
+
...n === void 0 ? {} : { mirror: n }
|
|
256
|
+
};
|
|
257
|
+
};
|
|
258
|
+
//#endregion
|
|
259
|
+
export { s as EVENTS_TABLE_DDL, o as MIRROR_TABLE_DDL, d as assertIdentifier, E as buildClickhouseSink, g as closeClickhouseState, v as makeEventBatcher, a as mirrorTableName };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voltro/plugin-clickhouse",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.87.0",
|
|
4
4
|
"description": "ClickHouse AnalyticsSink. Native HTTP ingest into a ClickHouse cluster — self-hosted OR ClickHouse Cloud. Production-grade OLAP for high-cardinality events with millisecond aggregate queries over billions of rows.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"voltro",
|
|
@@ -34,8 +34,8 @@
|
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
36
|
"@clickhouse/client": "^1.23.1",
|
|
37
|
-
"@voltro/logger": "0.
|
|
38
|
-
"@voltro/runtime": "0.
|
|
37
|
+
"@voltro/logger": "0.87.0",
|
|
38
|
+
"@voltro/runtime": "0.87.0"
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
41
41
|
"effect": "4.0.1"
|