@voltro/plugin-clickhouse 0.85.0 → 0.86.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +666 -0
- package/THIRD-PARTY-NOTICES.md +5 -33
- package/dist/index.js +28 -288
- package/dist/sink.js +259 -0
- package/package.json +3 -3
package/THIRD-PARTY-NOTICES.md
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# Third-Party Notices — @voltro/plugin-clickhouse
|
|
2
2
|
|
|
3
|
-
This package
|
|
3
|
+
This package uses the open-source software listed below. Each is the
|
|
4
4
|
property of its respective copyright holders and is used under the terms of
|
|
5
5
|
its license. This file is provided for attribution; it grants no rights in
|
|
6
6
|
@voltro/plugin-clickhouse itself, which is proprietary (see LICENSE).
|
|
7
7
|
|
|
8
|
-
Generated from the resolved runtime dependency closure (
|
|
8
|
+
Generated from the resolved runtime dependency closure (42 packages).
|
|
9
9
|
|
|
10
10
|
---
|
|
11
11
|
|
|
@@ -3846,34 +3846,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
|
3846
3846
|
SOFTWARE.
|
|
3847
3847
|
```
|
|
3848
3848
|
|
|
3849
|
-
## layerr@3.0.0
|
|
3850
|
-
|
|
3851
|
-
License: MIT
|
|
3852
|
-
|
|
3853
|
-
```
|
|
3854
|
-
MIT License
|
|
3855
|
-
|
|
3856
|
-
Copyright (c) 2020 Perry Mitchell
|
|
3857
|
-
|
|
3858
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
3859
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
3860
|
-
in the Software without restriction, including without limitation the rights
|
|
3861
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
3862
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
3863
|
-
furnished to do so, subject to the following conditions:
|
|
3864
|
-
|
|
3865
|
-
The above copyright notice and this permission notice shall be included in all
|
|
3866
|
-
copies or substantial portions of the Software.
|
|
3867
|
-
|
|
3868
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
3869
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
3870
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
3871
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
3872
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
3873
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
3874
|
-
SOFTWARE.
|
|
3875
|
-
```
|
|
3876
|
-
|
|
3877
3849
|
## lib0@0.2.117
|
|
3878
3850
|
|
|
3879
3851
|
License: MIT
|
|
@@ -4223,14 +4195,14 @@ Apache License
|
|
|
4223
4195
|
limitations under the License.
|
|
4224
4196
|
```
|
|
4225
4197
|
|
|
4226
|
-
##
|
|
4198
|
+
## ulid@3.0.2
|
|
4227
4199
|
|
|
4228
4200
|
License: MIT
|
|
4229
4201
|
|
|
4230
4202
|
```
|
|
4231
|
-
MIT License
|
|
4203
|
+
The MIT License (MIT)
|
|
4232
4204
|
|
|
4233
|
-
Copyright (c)
|
|
4205
|
+
Copyright (c) 2017 Alizain Feerasta
|
|
4234
4206
|
|
|
4235
4207
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4236
4208
|
of this software and associated documentation files (the "Software"), to deal
|
package/dist/index.js
CHANGED
|
@@ -1,300 +1,40 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
CREATE TABLE IF NOT EXISTS ${e}.${o(t)} (
|
|
8
|
-
id String,
|
|
9
|
-
data String CODEC(ZSTD(1)),
|
|
10
|
-
version UInt64,
|
|
11
|
-
is_deleted UInt8 DEFAULT 0
|
|
12
|
-
) ENGINE = ReplacingMergeTree(version)
|
|
13
|
-
ORDER BY id
|
|
14
|
-
`.trim(), c = (e, t) => `
|
|
15
|
-
CREATE TABLE IF NOT EXISTS ${e}.${t} (
|
|
16
|
-
id UUID DEFAULT generateUUIDv7(),
|
|
17
|
-
name LowCardinality(String),
|
|
18
|
-
subject_id Nullable(String),
|
|
19
|
-
tenant_id Nullable(String),
|
|
20
|
-
properties String CODEC(ZSTD(1)),
|
|
21
|
-
occurred_at DateTime64(3),
|
|
22
|
-
ingested_at DateTime64(3) DEFAULT now64()
|
|
23
|
-
) ENGINE = MergeTree
|
|
24
|
-
ORDER BY (name, occurred_at, id)
|
|
25
|
-
`.trim(), l = (e) => (t) => t instanceof r ? new r({
|
|
26
|
-
provider: a,
|
|
27
|
-
operation: e,
|
|
28
|
-
cause: t.cause
|
|
29
|
-
}) : new r({
|
|
30
|
-
provider: a,
|
|
31
|
-
operation: e,
|
|
32
|
-
cause: t instanceof Error ? `${t.name}: ${t.message}` : String(t)
|
|
33
|
-
}), u = (e) => (n) => n.pipe(t.catchDefect((e) => e instanceof r ? t.fail(e) : t.die(e)), t.mapError(l(e))), d = /^[A-Za-z_][A-Za-z0-9_]*$/, f = (e, t) => {
|
|
34
|
-
if (!d.test(e)) throw new r({
|
|
35
|
-
provider: a,
|
|
36
|
-
operation: "config",
|
|
37
|
-
cause: `invalid ${t} ${JSON.stringify(e)} — must match ${d.source} (letters, digits, underscore; no dots/quotes/spaces)`
|
|
38
|
-
});
|
|
39
|
-
return e;
|
|
40
|
-
}, p = /['\\]/, m = (e) => {
|
|
41
|
-
for (let t = 0; t < e.length; t++) if (e.charCodeAt(t) < 32) return !0;
|
|
42
|
-
return !1;
|
|
43
|
-
}, h = (e) => (t) => {
|
|
44
|
-
if (p.test(t) || m(t)) throw new r({
|
|
45
|
-
provider: a,
|
|
46
|
-
operation: e,
|
|
47
|
-
cause: `invalid property key ${JSON.stringify(t)} — must not contain quotes, backslashes, or control characters`
|
|
48
|
-
});
|
|
49
|
-
return t;
|
|
50
|
-
}, g = (e) => e.client === null ? t.fail(new r({
|
|
51
|
-
provider: a,
|
|
52
|
-
operation: "ensureClient",
|
|
53
|
-
cause: "client not initialized — spec.initialize must run before queries"
|
|
54
|
-
})) : t.succeed(e.client), _ = (e) => t.gen(function* () {
|
|
55
|
-
let n = e.client;
|
|
56
|
-
e.client = null, n !== null && (yield* t.tryPromise(() => n.close()).pipe(t.catch((e) => t.logWarning("clickhouse analytics sink teardown failed", {
|
|
57
|
-
provider: a,
|
|
58
|
-
err: e
|
|
59
|
-
}))));
|
|
60
|
-
}), v = (e) => ({
|
|
61
|
-
name: e.name,
|
|
62
|
-
subject_id: e.subjectId ?? null,
|
|
63
|
-
tenant_id: e.tenantId ?? null,
|
|
64
|
-
properties: JSON.stringify(e.properties),
|
|
65
|
-
occurred_at: (e.occurredAt ?? /* @__PURE__ */ new Date()).toISOString()
|
|
66
|
-
}), y = (e, t) => {
|
|
67
|
-
let n = [], r, a = () => {
|
|
68
|
-
r !== void 0 && (clearTimeout(r), r = void 0);
|
|
69
|
-
}, o = async () => {
|
|
70
|
-
if (a(), n.length === 0) return;
|
|
71
|
-
let t = n;
|
|
72
|
-
n = [];
|
|
73
|
-
let r = e.client;
|
|
74
|
-
if (r === null) {
|
|
75
|
-
i.warn("clickhouse batch flush skipped — client not initialized; events dropped", { dropped: t.length });
|
|
76
|
-
return;
|
|
77
|
-
}
|
|
78
|
-
try {
|
|
79
|
-
await r.insert({
|
|
80
|
-
table: `${e.database}.${e.table}`,
|
|
81
|
-
values: t,
|
|
82
|
-
format: "JSONEachRow"
|
|
83
|
-
});
|
|
84
|
-
} catch (e) {
|
|
85
|
-
i.warn("clickhouse batch flush failed — dropped events", { dropped: t.length }, e);
|
|
86
|
-
}
|
|
87
|
-
};
|
|
88
|
-
return {
|
|
89
|
-
add: (e) => {
|
|
90
|
-
if (n.push(v(e)), n.length >= t.maxSize) {
|
|
91
|
-
o();
|
|
92
|
-
return;
|
|
93
|
-
}
|
|
94
|
-
r === void 0 && (r = setTimeout(() => {
|
|
95
|
-
o();
|
|
96
|
-
}, t.flushIntervalMs), r.unref?.());
|
|
97
|
-
},
|
|
98
|
-
flush: o
|
|
99
|
-
};
|
|
100
|
-
}, b = (e) => {
|
|
101
|
-
switch (e) {
|
|
102
|
-
case "minute": return "toStartOfMinute";
|
|
103
|
-
case "hour": return "toStartOfHour";
|
|
104
|
-
case "day": return "toStartOfDay";
|
|
105
|
-
case "week": return "toStartOfWeek";
|
|
106
|
-
case "month": return "toStartOfMonth";
|
|
107
|
-
}
|
|
108
|
-
}, x = (e, t) => `JSONExtractString(properties, '${h(e)(t)}')`, S = (e, t) => `JSONExtractFloat(properties, '${h(e)(t)}')`, C = (e, t) => t === "count" ? "count()" : t === "unique" ? "uniqExact(subject_id)" : "sum" in t ? `sum(${S(e, t.sum)})` : `avg(${S(e, t.avg)})`, w = (e, t) => t === "count" ? "count()" : `sum(${S(e, t.sum)})`, T = (e, t, n, r, i) => {
|
|
109
|
-
let a = {
|
|
110
|
-
event: t,
|
|
111
|
-
from: n.from.toISOString(),
|
|
112
|
-
...n.to === void 0 ? {} : { to: n.to.toISOString() }
|
|
113
|
-
}, o = "name = {event:String} AND occurred_at >= {from:DateTime64(3)}" + (n.to === void 0 ? "" : " AND occurred_at < {to:DateTime64(3)}");
|
|
114
|
-
if (i != null && (a.tenant = i, o += " AND tenant_id = {tenant:String}"), r !== void 0) {
|
|
115
|
-
let t = 0;
|
|
116
|
-
for (let [n, i] of Object.entries(r)) {
|
|
117
|
-
let r = `f${t++}`;
|
|
118
|
-
a[r] = String(i), o += ` AND ${x(e, n)} = {${r}:String}`;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
return {
|
|
122
|
-
sql: o,
|
|
123
|
-
params: a
|
|
124
|
-
};
|
|
125
|
-
}, E = (e) => {
|
|
126
|
-
let n = (n, r) => t.gen(function* () {
|
|
127
|
-
f(n, "mirror table");
|
|
128
|
-
let i = yield* g(e);
|
|
129
|
-
yield* t.tryPromise({
|
|
130
|
-
try: () => i.insert({
|
|
131
|
-
table: `${e.database}.${o(n)}`,
|
|
132
|
-
values: [r],
|
|
133
|
-
format: "JSONEachRow"
|
|
134
|
-
}),
|
|
135
|
-
catch: (e) => e
|
|
136
|
-
});
|
|
137
|
-
});
|
|
138
|
-
return {
|
|
139
|
-
tables: e.mirrorTables,
|
|
140
|
-
primaryKey: e.mirrorPrimaryKey,
|
|
141
|
-
upsert: ({ table: r, row: i, version: a }) => t.gen(function* () {
|
|
142
|
-
let t = String(i[e.mirrorPrimaryKey]);
|
|
143
|
-
yield* n(r, {
|
|
144
|
-
id: t,
|
|
145
|
-
data: JSON.stringify(i),
|
|
146
|
-
version: a,
|
|
147
|
-
is_deleted: 0
|
|
148
|
-
});
|
|
149
|
-
}).pipe(u("mirror.upsert")),
|
|
150
|
-
remove: ({ table: e, primaryKeyValue: r, version: i }) => t.gen(function* () {
|
|
151
|
-
yield* n(e, {
|
|
152
|
-
id: String(r),
|
|
153
|
-
data: "{}",
|
|
154
|
-
version: i,
|
|
155
|
-
is_deleted: 1
|
|
156
|
-
});
|
|
157
|
-
}).pipe(u("mirror.remove")),
|
|
158
|
-
maxVersion: ({ table: n, primaryKeyValue: r }) => t.gen(function* () {
|
|
159
|
-
f(String(n), "mirror table");
|
|
160
|
-
let i = yield* g(e), a = `SELECT max(version) AS v, count() AS n FROM ${e.database}.${o(String(n))} WHERE id = {id: String} FORMAT JSON`, s = yield* t.tryPromise({
|
|
161
|
-
try: () => i.query({
|
|
162
|
-
query: a,
|
|
163
|
-
query_params: { id: String(r) }
|
|
164
|
-
}),
|
|
165
|
-
catch: (e) => e
|
|
166
|
-
}), { data: c } = yield* t.tryPromise({
|
|
167
|
-
try: () => s.json(),
|
|
168
|
-
catch: (e) => e
|
|
169
|
-
}), l = c[0];
|
|
170
|
-
return l === void 0 || Number(l.n) === 0 || l.v === null ? null : Number(l.v);
|
|
171
|
-
}).pipe(u("mirror.maxVersion"))
|
|
172
|
-
};
|
|
173
|
-
}, D = (e) => {
|
|
174
|
-
let n = e.mirrorTables.length > 0 ? E(e) : void 0;
|
|
175
|
-
return {
|
|
176
|
-
info: {
|
|
177
|
-
provider: a,
|
|
178
|
-
capabilities: {
|
|
179
|
-
track: !0,
|
|
180
|
-
aggregate: !0,
|
|
181
|
-
timeseries: !0,
|
|
182
|
-
topN: !0,
|
|
183
|
-
mirror: n !== void 0
|
|
184
|
-
},
|
|
185
|
-
detail: {
|
|
186
|
-
database: e.database,
|
|
187
|
-
table: e.table,
|
|
188
|
-
...e.mirrorTables.length > 0 ? { mirrorTables: e.mirrorTables } : {},
|
|
189
|
-
...e.batcher === void 0 ? {} : { batching: !0 }
|
|
190
|
-
}
|
|
191
|
-
},
|
|
192
|
-
track: (n) => e.batcher ? t.sync(() => e.batcher.add(n)) : t.gen(function* () {
|
|
193
|
-
let r = yield* g(e);
|
|
194
|
-
yield* t.tryPromise({
|
|
195
|
-
try: () => r.insert({
|
|
196
|
-
table: `${e.database}.${e.table}`,
|
|
197
|
-
values: [v(n)],
|
|
198
|
-
format: "JSONEachRow"
|
|
199
|
-
}),
|
|
200
|
-
catch: (e) => e
|
|
201
|
-
});
|
|
202
|
-
}).pipe(u("track")),
|
|
203
|
-
aggregate: (n) => t.gen(function* () {
|
|
204
|
-
let r = yield* g(e), i = T("aggregate", n.event, n.range, n.filter, n.tenantId), a = `SELECT ${C("aggregate", n.metric)} AS value FROM ${e.database}.${e.table} WHERE ${i.sql} FORMAT JSON`, o = yield* t.tryPromise({
|
|
205
|
-
try: () => r.query({
|
|
206
|
-
query: a,
|
|
207
|
-
query_params: i.params
|
|
208
|
-
}),
|
|
209
|
-
catch: (e) => e
|
|
210
|
-
}), { data: s } = yield* t.tryPromise({
|
|
211
|
-
try: () => o.json(),
|
|
212
|
-
catch: (e) => e
|
|
213
|
-
}), c = s[0]?.value ?? 0;
|
|
214
|
-
return typeof c == "number" ? c : Number(c ?? 0);
|
|
215
|
-
}).pipe(u("aggregate")),
|
|
216
|
-
timeseries: (n) => t.gen(function* () {
|
|
217
|
-
let r = yield* g(e), i = T("timeseries", n.event, n.range, n.filter, n.tenantId), a = `
|
|
218
|
-
SELECT ${`${b(n.bucket)}(occurred_at)`} AS bucket_start, ${C("timeseries", n.metric)} AS value
|
|
219
|
-
FROM ${e.database}.${e.table}
|
|
220
|
-
WHERE ${i.sql}
|
|
221
|
-
GROUP BY bucket_start
|
|
222
|
-
ORDER BY bucket_start ASC
|
|
223
|
-
FORMAT JSON
|
|
224
|
-
`, o = yield* t.tryPromise({
|
|
225
|
-
try: () => r.query({
|
|
226
|
-
query: a,
|
|
227
|
-
query_params: i.params
|
|
228
|
-
}),
|
|
229
|
-
catch: (e) => e
|
|
230
|
-
}), { data: s } = yield* t.tryPromise({
|
|
231
|
-
try: () => o.json(),
|
|
232
|
-
catch: (e) => e
|
|
233
|
-
});
|
|
234
|
-
return s.map((e) => ({
|
|
235
|
-
bucketStart: new Date(e.bucket_start),
|
|
236
|
-
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
237
|
-
}));
|
|
238
|
-
}).pipe(u("timeseries")),
|
|
239
|
-
topN: (n) => t.gen(function* () {
|
|
240
|
-
let r = yield* g(e), i = T("topN", n.event, n.range, n.filter, n.tenantId), a = Math.max(1, Math.floor(n.n)), o = x("topN", n.groupBy), s = `
|
|
241
|
-
SELECT ${o} AS key, ${w("topN", n.metric)} AS value
|
|
242
|
-
FROM ${e.database}.${e.table}
|
|
243
|
-
WHERE ${i.sql}
|
|
244
|
-
GROUP BY ${o}
|
|
245
|
-
ORDER BY value DESC
|
|
246
|
-
LIMIT ${a}
|
|
247
|
-
FORMAT JSON
|
|
248
|
-
`, c = yield* t.tryPromise({
|
|
249
|
-
try: () => r.query({
|
|
250
|
-
query: s,
|
|
251
|
-
query_params: i.params
|
|
252
|
-
}),
|
|
253
|
-
catch: (e) => e
|
|
254
|
-
}), { data: l } = yield* t.tryPromise({
|
|
255
|
-
try: () => c.json(),
|
|
256
|
-
catch: (e) => e
|
|
257
|
-
});
|
|
258
|
-
return l.filter((e) => e.key !== null).map((e) => ({
|
|
259
|
-
key: e.key,
|
|
260
|
-
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
261
|
-
}));
|
|
262
|
-
}).pipe(u("topN")),
|
|
263
|
-
...n === void 0 ? {} : { mirror: n }
|
|
264
|
-
};
|
|
265
|
-
}, O = 20, k = 5e3, A = (n) => {
|
|
266
|
-
let r = (n.mirrorTables ?? []).map((e) => f(e, "mirror table")), i = {
|
|
1
|
+
import { EVENTS_TABLE_DDL as e, MIRROR_TABLE_DDL as t, assertIdentifier as n, buildClickhouseSink as r, closeClickhouseState as i, makeEventBatcher as a } from "./sink.js";
|
|
2
|
+
import { createClient as o } from "@clickhouse/client";
|
|
3
|
+
import { Effect as s } from "effect";
|
|
4
|
+
//#region src/index.ts
|
|
5
|
+
var c = 20, l = 5e3, u = (u) => {
|
|
6
|
+
let d = (u.mirrorTables ?? []).map((e) => n(e, "mirror table")), f = {
|
|
267
7
|
client: null,
|
|
268
|
-
database:
|
|
269
|
-
table:
|
|
270
|
-
mirrorTables:
|
|
271
|
-
mirrorPrimaryKey:
|
|
8
|
+
database: n(u.database ?? "default", "database"),
|
|
9
|
+
table: n(u.table ?? "events", "table"),
|
|
10
|
+
mirrorTables: d,
|
|
11
|
+
mirrorPrimaryKey: u.mirrorPrimaryKey ?? "id"
|
|
272
12
|
};
|
|
273
|
-
if (
|
|
274
|
-
let e =
|
|
275
|
-
|
|
276
|
-
maxSize: e.maxSize ??
|
|
277
|
-
flushIntervalMs: e.flushIntervalMs ??
|
|
13
|
+
if (u.batch !== !1) {
|
|
14
|
+
let e = u.batch ?? {};
|
|
15
|
+
f.batcher = a(f, {
|
|
16
|
+
maxSize: e.maxSize ?? c,
|
|
17
|
+
flushIntervalMs: e.flushIntervalMs ?? l
|
|
278
18
|
});
|
|
279
19
|
}
|
|
280
20
|
return {
|
|
281
21
|
kind: "clickhouse",
|
|
282
|
-
build: (e) =>
|
|
283
|
-
initialize: () =>
|
|
284
|
-
let
|
|
285
|
-
url:
|
|
286
|
-
database:
|
|
287
|
-
...
|
|
288
|
-
...
|
|
22
|
+
build: (e) => r(f),
|
|
23
|
+
initialize: () => s.gen(function* () {
|
|
24
|
+
let n = yield* s.promise(() => Promise.resolve(o({
|
|
25
|
+
url: u.url,
|
|
26
|
+
database: f.database,
|
|
27
|
+
...u.username === void 0 ? {} : { username: u.username },
|
|
28
|
+
...u.password === void 0 ? {} : { password: u.password }
|
|
289
29
|
})));
|
|
290
|
-
yield*
|
|
291
|
-
for (let e of
|
|
292
|
-
|
|
30
|
+
yield* s.promise(() => n.ping()), yield* s.promise(() => n.command({ query: e(f.database, f.table) }));
|
|
31
|
+
for (let e of f.mirrorTables) yield* s.promise(() => n.command({ query: t(f.database, e) }));
|
|
32
|
+
f.client = n;
|
|
293
33
|
}),
|
|
294
|
-
dispose: () =>
|
|
295
|
-
|
|
34
|
+
dispose: () => s.gen(function* () {
|
|
35
|
+
f.batcher && (yield* s.promise(() => f.batcher.flush())), yield* i(f);
|
|
296
36
|
})
|
|
297
37
|
};
|
|
298
38
|
};
|
|
299
39
|
//#endregion
|
|
300
|
-
export {
|
|
40
|
+
export { e as EVENTS_TABLE_DDL, t as MIRROR_TABLE_DDL, u as clickhouseAnalytics };
|
package/dist/sink.js
ADDED
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
import { Effect as e } from "effect";
|
|
2
|
+
import { createLogger as t } from "@voltro/logger";
|
|
3
|
+
import { AnalyticsError as n } from "@voltro/runtime";
|
|
4
|
+
//#region src/sink.ts
|
|
5
|
+
var r = t({ scope: "@voltro/plugin-clickhouse" }), i = "clickhouse", a = (e) => `voltro_mirror_${e}`, o = (e, t) => `
|
|
6
|
+
CREATE TABLE IF NOT EXISTS ${e}.${a(t)} (
|
|
7
|
+
id String,
|
|
8
|
+
data String CODEC(ZSTD(1)),
|
|
9
|
+
version UInt64,
|
|
10
|
+
is_deleted UInt8 DEFAULT 0
|
|
11
|
+
) ENGINE = ReplacingMergeTree(version)
|
|
12
|
+
ORDER BY id
|
|
13
|
+
`.trim(), s = (e, t) => `
|
|
14
|
+
CREATE TABLE IF NOT EXISTS ${e}.${t} (
|
|
15
|
+
id UUID DEFAULT generateUUIDv7(),
|
|
16
|
+
name LowCardinality(String),
|
|
17
|
+
subject_id Nullable(String),
|
|
18
|
+
tenant_id Nullable(String),
|
|
19
|
+
properties String CODEC(ZSTD(1)),
|
|
20
|
+
occurred_at DateTime64(3),
|
|
21
|
+
ingested_at DateTime64(3) DEFAULT now64()
|
|
22
|
+
) ENGINE = MergeTree
|
|
23
|
+
ORDER BY (name, occurred_at, id)
|
|
24
|
+
`.trim(), c = (e) => (t) => t instanceof n ? new n({
|
|
25
|
+
provider: i,
|
|
26
|
+
operation: e,
|
|
27
|
+
cause: t.cause
|
|
28
|
+
}) : new n({
|
|
29
|
+
provider: i,
|
|
30
|
+
operation: e,
|
|
31
|
+
cause: t instanceof Error ? `${t.name}: ${t.message}` : String(t)
|
|
32
|
+
}), l = (t) => (r) => r.pipe(e.catchDefect((t) => t instanceof n ? e.fail(t) : e.die(t)), e.mapError(c(t))), u = /^[A-Za-z_][A-Za-z0-9_]*$/, d = (e, t) => {
|
|
33
|
+
if (!u.test(e)) throw new n({
|
|
34
|
+
provider: i,
|
|
35
|
+
operation: "config",
|
|
36
|
+
cause: `invalid ${t} ${JSON.stringify(e)} — must match ${u.source} (letters, digits, underscore; no dots/quotes/spaces)`
|
|
37
|
+
});
|
|
38
|
+
return e;
|
|
39
|
+
}, f = /['\\]/, p = (e) => {
|
|
40
|
+
for (let t = 0; t < e.length; t++) if (e.charCodeAt(t) < 32) return !0;
|
|
41
|
+
return !1;
|
|
42
|
+
}, m = (e) => (t) => {
|
|
43
|
+
if (f.test(t) || p(t)) throw new n({
|
|
44
|
+
provider: i,
|
|
45
|
+
operation: e,
|
|
46
|
+
cause: `invalid property key ${JSON.stringify(t)} — must not contain quotes, backslashes, or control characters`
|
|
47
|
+
});
|
|
48
|
+
return t;
|
|
49
|
+
}, h = (t) => t.client === null ? e.fail(new n({
|
|
50
|
+
provider: i,
|
|
51
|
+
operation: "ensureClient",
|
|
52
|
+
cause: "client not initialized — spec.initialize must run before queries"
|
|
53
|
+
})) : e.succeed(t.client), g = (t) => e.gen(function* () {
|
|
54
|
+
let n = t.client;
|
|
55
|
+
t.client = null, n !== null && (yield* e.tryPromise(() => n.close()).pipe(e.catch((t) => e.logWarning("clickhouse analytics sink teardown failed", {
|
|
56
|
+
provider: i,
|
|
57
|
+
err: t
|
|
58
|
+
}))));
|
|
59
|
+
}), _ = (e) => ({
|
|
60
|
+
name: e.name,
|
|
61
|
+
subject_id: e.subjectId ?? null,
|
|
62
|
+
tenant_id: e.tenantId ?? null,
|
|
63
|
+
properties: JSON.stringify(e.properties),
|
|
64
|
+
occurred_at: (e.occurredAt ?? /* @__PURE__ */ new Date()).toISOString()
|
|
65
|
+
}), v = (e, t) => {
|
|
66
|
+
let n = [], i, a = () => {
|
|
67
|
+
i !== void 0 && (clearTimeout(i), i = void 0);
|
|
68
|
+
}, o = async () => {
|
|
69
|
+
if (a(), n.length === 0) return;
|
|
70
|
+
let t = n;
|
|
71
|
+
n = [];
|
|
72
|
+
let i = e.client;
|
|
73
|
+
if (i === null) r.warn("clickhouse batch flush skipped — client not initialized; events dropped", { dropped: t.length });
|
|
74
|
+
else try {
|
|
75
|
+
await i.insert({
|
|
76
|
+
table: `${e.database}.${e.table}`,
|
|
77
|
+
values: t,
|
|
78
|
+
format: "JSONEachRow"
|
|
79
|
+
});
|
|
80
|
+
} catch (e) {
|
|
81
|
+
r.warn("clickhouse batch flush failed — dropped events", { dropped: t.length }, e);
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
return {
|
|
85
|
+
add: (e) => {
|
|
86
|
+
n.push(_(e)), n.length >= t.maxSize ? o() : i === void 0 && (i = setTimeout(() => {
|
|
87
|
+
o();
|
|
88
|
+
}, t.flushIntervalMs), i.unref?.());
|
|
89
|
+
},
|
|
90
|
+
flush: o
|
|
91
|
+
};
|
|
92
|
+
}, y = (e) => {
|
|
93
|
+
switch (e) {
|
|
94
|
+
case "minute": return "toStartOfMinute";
|
|
95
|
+
case "hour": return "toStartOfHour";
|
|
96
|
+
case "day": return "toStartOfDay";
|
|
97
|
+
case "week": return "toStartOfWeek";
|
|
98
|
+
case "month": return "toStartOfMonth";
|
|
99
|
+
}
|
|
100
|
+
}, b = (e, t) => `JSONExtractString(properties, '${m(e)(t)}')`, x = (e, t) => `JSONExtractFloat(properties, '${m(e)(t)}')`, S = (e, t) => t === "count" ? "count()" : t === "unique" ? "uniqExact(subject_id)" : "sum" in t ? `sum(${x(e, t.sum)})` : `avg(${x(e, t.avg)})`, C = (e, t) => t === "count" ? "count()" : `sum(${x(e, t.sum)})`, w = (e, t, n, r, i) => {
|
|
101
|
+
let a = {
|
|
102
|
+
event: t,
|
|
103
|
+
from: n.from.toISOString(),
|
|
104
|
+
...n.to === void 0 ? {} : { to: n.to.toISOString() }
|
|
105
|
+
}, o = "name = {event:String} AND occurred_at >= {from:DateTime64(3)}" + (n.to === void 0 ? "" : " AND occurred_at < {to:DateTime64(3)}");
|
|
106
|
+
if (i != null && (a.tenant = i, o += " AND tenant_id = {tenant:String}"), r !== void 0) {
|
|
107
|
+
let t = 0;
|
|
108
|
+
for (let [n, i] of Object.entries(r)) {
|
|
109
|
+
let r = `f${t++}`;
|
|
110
|
+
a[r] = String(i), o += ` AND ${b(e, n)} = {${r}:String}`;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return {
|
|
114
|
+
sql: o,
|
|
115
|
+
params: a
|
|
116
|
+
};
|
|
117
|
+
}, T = (t) => {
|
|
118
|
+
let n = (n, r) => e.gen(function* () {
|
|
119
|
+
d(n, "mirror table");
|
|
120
|
+
let i = yield* h(t);
|
|
121
|
+
yield* e.tryPromise({
|
|
122
|
+
try: () => i.insert({
|
|
123
|
+
table: `${t.database}.${a(n)}`,
|
|
124
|
+
values: [r],
|
|
125
|
+
format: "JSONEachRow"
|
|
126
|
+
}),
|
|
127
|
+
catch: (e) => e
|
|
128
|
+
});
|
|
129
|
+
});
|
|
130
|
+
return {
|
|
131
|
+
tables: t.mirrorTables,
|
|
132
|
+
primaryKey: t.mirrorPrimaryKey,
|
|
133
|
+
upsert: ({ table: r, row: i, version: a }) => e.gen(function* () {
|
|
134
|
+
let e = String(i[t.mirrorPrimaryKey]);
|
|
135
|
+
yield* n(r, {
|
|
136
|
+
id: e,
|
|
137
|
+
data: JSON.stringify(i),
|
|
138
|
+
version: a,
|
|
139
|
+
is_deleted: 0
|
|
140
|
+
});
|
|
141
|
+
}).pipe(l("mirror.upsert")),
|
|
142
|
+
remove: ({ table: t, primaryKeyValue: r, version: i }) => e.gen(function* () {
|
|
143
|
+
yield* n(t, {
|
|
144
|
+
id: String(r),
|
|
145
|
+
data: "{}",
|
|
146
|
+
version: i,
|
|
147
|
+
is_deleted: 1
|
|
148
|
+
});
|
|
149
|
+
}).pipe(l("mirror.remove")),
|
|
150
|
+
maxVersion: ({ table: n, primaryKeyValue: r }) => e.gen(function* () {
|
|
151
|
+
d(String(n), "mirror table");
|
|
152
|
+
let i = yield* h(t), o = `SELECT max(version) AS v, count() AS n FROM ${t.database}.${a(String(n))} WHERE id = {id: String} FORMAT JSON`, s = yield* e.tryPromise({
|
|
153
|
+
try: () => i.query({
|
|
154
|
+
query: o,
|
|
155
|
+
query_params: { id: String(r) }
|
|
156
|
+
}),
|
|
157
|
+
catch: (e) => e
|
|
158
|
+
}), { data: c } = yield* e.tryPromise({
|
|
159
|
+
try: () => s.json(),
|
|
160
|
+
catch: (e) => e
|
|
161
|
+
}), l = c[0];
|
|
162
|
+
return l === void 0 || Number(l.n) === 0 || l.v === null ? null : Number(l.v);
|
|
163
|
+
}).pipe(l("mirror.maxVersion"))
|
|
164
|
+
};
|
|
165
|
+
}, E = (t) => {
|
|
166
|
+
let n = t.mirrorTables.length > 0 ? T(t) : void 0;
|
|
167
|
+
return {
|
|
168
|
+
info: {
|
|
169
|
+
provider: i,
|
|
170
|
+
capabilities: {
|
|
171
|
+
track: !0,
|
|
172
|
+
aggregate: !0,
|
|
173
|
+
timeseries: !0,
|
|
174
|
+
topN: !0,
|
|
175
|
+
mirror: n !== void 0
|
|
176
|
+
},
|
|
177
|
+
detail: {
|
|
178
|
+
database: t.database,
|
|
179
|
+
table: t.table,
|
|
180
|
+
...t.mirrorTables.length > 0 ? { mirrorTables: t.mirrorTables } : {},
|
|
181
|
+
...t.batcher === void 0 ? {} : { batching: !0 }
|
|
182
|
+
}
|
|
183
|
+
},
|
|
184
|
+
track: (n) => t.batcher ? e.sync(() => t.batcher.add(n)) : e.gen(function* () {
|
|
185
|
+
let r = yield* h(t);
|
|
186
|
+
yield* e.tryPromise({
|
|
187
|
+
try: () => r.insert({
|
|
188
|
+
table: `${t.database}.${t.table}`,
|
|
189
|
+
values: [_(n)],
|
|
190
|
+
format: "JSONEachRow"
|
|
191
|
+
}),
|
|
192
|
+
catch: (e) => e
|
|
193
|
+
});
|
|
194
|
+
}).pipe(l("track")),
|
|
195
|
+
aggregate: (n) => e.gen(function* () {
|
|
196
|
+
let r = yield* h(t), i = w("aggregate", n.event, n.range, n.filter, n.tenantId), a = `SELECT ${S("aggregate", n.metric)} AS value FROM ${t.database}.${t.table} WHERE ${i.sql} FORMAT JSON`, o = yield* e.tryPromise({
|
|
197
|
+
try: () => r.query({
|
|
198
|
+
query: a,
|
|
199
|
+
query_params: i.params
|
|
200
|
+
}),
|
|
201
|
+
catch: (e) => e
|
|
202
|
+
}), { data: s } = yield* e.tryPromise({
|
|
203
|
+
try: () => o.json(),
|
|
204
|
+
catch: (e) => e
|
|
205
|
+
}), c = s[0]?.value ?? 0;
|
|
206
|
+
return typeof c == "number" ? c : Number(c ?? 0);
|
|
207
|
+
}).pipe(l("aggregate")),
|
|
208
|
+
timeseries: (n) => e.gen(function* () {
|
|
209
|
+
let r = yield* h(t), i = w("timeseries", n.event, n.range, n.filter, n.tenantId), a = `
|
|
210
|
+
SELECT ${`${y(n.bucket)}(occurred_at)`} AS bucket_start, ${S("timeseries", n.metric)} AS value
|
|
211
|
+
FROM ${t.database}.${t.table}
|
|
212
|
+
WHERE ${i.sql}
|
|
213
|
+
GROUP BY bucket_start
|
|
214
|
+
ORDER BY bucket_start ASC
|
|
215
|
+
FORMAT JSON
|
|
216
|
+
`, o = yield* e.tryPromise({
|
|
217
|
+
try: () => r.query({
|
|
218
|
+
query: a,
|
|
219
|
+
query_params: i.params
|
|
220
|
+
}),
|
|
221
|
+
catch: (e) => e
|
|
222
|
+
}), { data: s } = yield* e.tryPromise({
|
|
223
|
+
try: () => o.json(),
|
|
224
|
+
catch: (e) => e
|
|
225
|
+
});
|
|
226
|
+
return s.map((e) => ({
|
|
227
|
+
bucketStart: new Date(e.bucket_start),
|
|
228
|
+
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
229
|
+
}));
|
|
230
|
+
}).pipe(l("timeseries")),
|
|
231
|
+
topN: (n) => e.gen(function* () {
|
|
232
|
+
let r = yield* h(t), i = w("topN", n.event, n.range, n.filter, n.tenantId), a = Math.max(1, Math.floor(n.n)), o = b("topN", n.groupBy), s = `
|
|
233
|
+
SELECT ${o} AS key, ${C("topN", n.metric)} AS value
|
|
234
|
+
FROM ${t.database}.${t.table}
|
|
235
|
+
WHERE ${i.sql}
|
|
236
|
+
GROUP BY ${o}
|
|
237
|
+
ORDER BY value DESC
|
|
238
|
+
LIMIT ${a}
|
|
239
|
+
FORMAT JSON
|
|
240
|
+
`, c = yield* e.tryPromise({
|
|
241
|
+
try: () => r.query({
|
|
242
|
+
query: s,
|
|
243
|
+
query_params: i.params
|
|
244
|
+
}),
|
|
245
|
+
catch: (e) => e
|
|
246
|
+
}), { data: l } = yield* e.tryPromise({
|
|
247
|
+
try: () => c.json(),
|
|
248
|
+
catch: (e) => e
|
|
249
|
+
});
|
|
250
|
+
return l.filter((e) => e.key !== null).map((e) => ({
|
|
251
|
+
key: e.key,
|
|
252
|
+
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
253
|
+
}));
|
|
254
|
+
}).pipe(l("topN")),
|
|
255
|
+
...n === void 0 ? {} : { mirror: n }
|
|
256
|
+
};
|
|
257
|
+
};
|
|
258
|
+
//#endregion
|
|
259
|
+
export { s as EVENTS_TABLE_DDL, o as MIRROR_TABLE_DDL, d as assertIdentifier, E as buildClickhouseSink, g as closeClickhouseState, v as makeEventBatcher, a as mirrorTableName };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voltro/plugin-clickhouse",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.86.0",
|
|
4
4
|
"description": "ClickHouse AnalyticsSink. Native HTTP ingest into a ClickHouse cluster — self-hosted OR ClickHouse Cloud. Production-grade OLAP for high-cardinality events with millisecond aggregate queries over billions of rows.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"voltro",
|
|
@@ -34,8 +34,8 @@
|
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
36
|
"@clickhouse/client": "^1.23.1",
|
|
37
|
-
"@voltro/logger": "0.
|
|
38
|
-
"@voltro/runtime": "0.
|
|
37
|
+
"@voltro/logger": "0.86.0",
|
|
38
|
+
"@voltro/runtime": "0.86.0"
|
|
39
39
|
},
|
|
40
40
|
"peerDependencies": {
|
|
41
41
|
"effect": "4.0.1"
|