@voltro/plugin-duckdb 0.83.1 → 0.85.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1191 -0
- package/THIRD-PARTY-NOTICES.md +71 -317
- package/dist/index.js +94 -94
- package/package.json +3 -3
package/dist/index.js
CHANGED
|
@@ -1,36 +1,36 @@
|
|
|
1
|
-
import { Effect as e
|
|
2
|
-
import { AnalyticsError as
|
|
1
|
+
import { Effect as e } from "effect";
|
|
2
|
+
import { AnalyticsError as t } from "@voltro/runtime";
|
|
3
3
|
//#region src/sink.ts
|
|
4
|
-
var
|
|
4
|
+
var n = "duckdb", r = "voltro_events", i = null, a = () => i === null ? e.tryPromise({
|
|
5
5
|
try: async () => {
|
|
6
6
|
let e = await import("@duckdb/node-api");
|
|
7
|
-
return
|
|
7
|
+
return i = e, e;
|
|
8
8
|
},
|
|
9
|
-
catch: (e) => new
|
|
10
|
-
provider:
|
|
9
|
+
catch: (e) => new t({
|
|
10
|
+
provider: n,
|
|
11
11
|
operation: "loadNativeModule",
|
|
12
12
|
cause: "@duckdb/node-api could not be loaded — the native binary for this platform may be missing (it is an optionalDependency). " + (e instanceof Error ? `${e.name}: ${e.message}` : String(e))
|
|
13
13
|
})
|
|
14
|
-
}) : e.succeed(
|
|
15
|
-
if (!
|
|
16
|
-
provider:
|
|
14
|
+
}) : e.succeed(i), o = (e) => new i.DuckDBTimestampValue(BigInt(e.getTime()) * 1000n), s = (e) => `voltro_mirror_${e}`, c = /^[A-Za-z_][A-Za-z0-9_]*$/, l = (e, r, i) => {
|
|
15
|
+
if (!c.test(r)) throw new t({
|
|
16
|
+
provider: n,
|
|
17
17
|
operation: e,
|
|
18
|
-
cause: `invalid ${i} ${JSON.stringify(
|
|
18
|
+
cause: `invalid ${i} ${JSON.stringify(r)} — must match ${c.source} (letters, digits, underscore; no dots/quotes/spaces)`
|
|
19
19
|
});
|
|
20
|
-
return
|
|
21
|
-
},
|
|
20
|
+
return r;
|
|
21
|
+
}, u = /['\\]/, d = (e) => {
|
|
22
22
|
for (let t = 0; t < e.length; t++) if (e.charCodeAt(t) < 32) return !0;
|
|
23
23
|
return !1;
|
|
24
|
-
},
|
|
25
|
-
if (
|
|
26
|
-
provider:
|
|
24
|
+
}, f = (e, r) => {
|
|
25
|
+
if (u.test(r) || d(r)) throw new t({
|
|
26
|
+
provider: n,
|
|
27
27
|
operation: e,
|
|
28
|
-
cause: `invalid property key ${JSON.stringify(
|
|
28
|
+
cause: `invalid property key ${JSON.stringify(r)} — must not contain quotes, backslashes, or control characters`
|
|
29
29
|
});
|
|
30
|
-
return
|
|
31
|
-
},
|
|
30
|
+
return r;
|
|
31
|
+
}, p = `
|
|
32
32
|
CREATE SEQUENCE IF NOT EXISTS voltro_events_id_seq;
|
|
33
|
-
CREATE TABLE IF NOT EXISTS ${
|
|
33
|
+
CREATE TABLE IF NOT EXISTS ${r} (
|
|
34
34
|
id BIGINT PRIMARY KEY DEFAULT nextval('voltro_events_id_seq'),
|
|
35
35
|
name VARCHAR NOT NULL,
|
|
36
36
|
subject_id VARCHAR,
|
|
@@ -38,79 +38,79 @@ var r = "duckdb", i = "voltro_events", a = null, o = () => a === null ? e.tryPro
|
|
|
38
38
|
properties JSON NOT NULL DEFAULT '{}',
|
|
39
39
|
occurred_at TIMESTAMP NOT NULL DEFAULT now()
|
|
40
40
|
);
|
|
41
|
-
`.trim(),
|
|
42
|
-
provider:
|
|
43
|
-
operation:
|
|
41
|
+
`.trim(), m = (r) => (i) => i.pipe(e.catchDefect((n) => n instanceof t ? e.fail(n) : e.die(n)), e.mapError((e) => e instanceof t ? e : new t({
|
|
42
|
+
provider: n,
|
|
43
|
+
operation: r,
|
|
44
44
|
cause: e instanceof Error ? `${e.name}: ${e.message}` : String(e)
|
|
45
|
-
}))),
|
|
46
|
-
provider:
|
|
45
|
+
}))), h = (r) => r.conn === null ? e.fail(new t({
|
|
46
|
+
provider: n,
|
|
47
47
|
operation: "ensureConn",
|
|
48
48
|
cause: "connection not initialized — spec.initialize must run before queries"
|
|
49
|
-
})) : e.succeed(
|
|
50
|
-
let { VARCHAR:
|
|
51
|
-
if (d.push("name = ?"), l.push(t), u.push(
|
|
49
|
+
})) : e.succeed(r.conn), g = (e, t) => `date_trunc('${t}', ${e})`, _ = (e, t) => `json_extract_string(properties, '$.${f(e, t)}')`, v = (e, t) => `TRY_CAST(json_extract_string(properties, '$.${f(e, t)}') AS DOUBLE)`, y = (e, t) => t === "count" ? "COUNT(*)" : t === "unique" ? "COUNT(DISTINCT subject_id)" : "sum" in t ? `COALESCE(SUM(${v(e, t.sum)}), 0)` : `AVG(${v(e, t.avg)})`, b = (e, t) => t === "count" ? "COUNT(*)" : `COALESCE(SUM(${v(e, t.sum)}), 0)`, x = (e, t, n, r, a) => {
|
|
50
|
+
let { VARCHAR: s, TIMESTAMP: c } = i, l = [], u = [], d = [];
|
|
51
|
+
if (d.push("name = ?"), l.push(t), u.push(s), d.push("occurred_at >= ?"), l.push(o(n.from)), u.push(c), n.to !== void 0 && (d.push("occurred_at < ?"), l.push(o(n.to)), u.push(c)), a != null && (d.push("tenant_id = ?"), l.push(a), u.push(s)), r !== void 0) for (let [t, n] of Object.entries(r)) d.push(`${_(e, t)} = ?`), l.push(String(n)), u.push(s);
|
|
52
52
|
return {
|
|
53
53
|
sql: d.join(" AND "),
|
|
54
54
|
params: l,
|
|
55
55
|
types: u
|
|
56
56
|
};
|
|
57
|
-
},
|
|
57
|
+
}, S = (e, t, n, r, a) => [
|
|
58
58
|
`INSERT INTO ${e} (id, data, version, is_deleted) VALUES (?, ?::JSON, ?::BIGINT, ?) ON CONFLICT (id) DO UPDATE SET data = EXCLUDED.data, version = EXCLUDED.version, is_deleted = EXCLUDED.is_deleted WHERE EXCLUDED.version > ${e}.version`,
|
|
59
59
|
[
|
|
60
60
|
t,
|
|
61
61
|
n,
|
|
62
62
|
String(r),
|
|
63
|
-
|
|
63
|
+
a
|
|
64
64
|
],
|
|
65
65
|
[
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
i.VARCHAR,
|
|
67
|
+
i.VARCHAR,
|
|
68
|
+
i.VARCHAR,
|
|
69
|
+
i.BOOLEAN
|
|
70
70
|
]
|
|
71
|
-
],
|
|
72
|
-
let
|
|
73
|
-
if (
|
|
74
|
-
|
|
75
|
-
let
|
|
71
|
+
], C = (t, n, r) => {
|
|
72
|
+
let a = /* @__PURE__ */ new Set(), o = (n, r) => e.gen(function* () {
|
|
73
|
+
if (a.has(r)) return;
|
|
74
|
+
l(n, r, "mirror table name");
|
|
75
|
+
let i = yield* h(t), o = s(r);
|
|
76
76
|
yield* e.tryPromise({
|
|
77
|
-
try: () =>
|
|
77
|
+
try: () => i.run(`CREATE TABLE IF NOT EXISTS ${o} (id VARCHAR PRIMARY KEY, data JSON NOT NULL, version BIGINT NOT NULL DEFAULT 0, is_deleted BOOLEAN NOT NULL DEFAULT false)`),
|
|
78
78
|
catch: (e) => e
|
|
79
|
-
}),
|
|
79
|
+
}), a.add(r);
|
|
80
80
|
});
|
|
81
81
|
return {
|
|
82
82
|
tables: n,
|
|
83
83
|
primaryKey: r,
|
|
84
84
|
upsert: ({ table: n, row: i, version: a }) => e.gen(function* () {
|
|
85
85
|
yield* o("mirror.upsert", n);
|
|
86
|
-
let
|
|
86
|
+
let c = yield* h(t), l = s(n), u = String(i[r]);
|
|
87
87
|
yield* e.tryPromise({
|
|
88
|
-
try: () =>
|
|
88
|
+
try: () => c.run(...S(l, u, JSON.stringify(i), a, !1)),
|
|
89
89
|
catch: (e) => e
|
|
90
90
|
});
|
|
91
|
-
}).pipe(
|
|
91
|
+
}).pipe(m("mirror.upsert")),
|
|
92
92
|
remove: ({ table: n, primaryKeyValue: r, version: i }) => e.gen(function* () {
|
|
93
93
|
yield* o("mirror.remove", n);
|
|
94
|
-
let a = yield*
|
|
94
|
+
let a = yield* h(t), c = s(n), l = String(r);
|
|
95
95
|
yield* e.tryPromise({
|
|
96
|
-
try: () => a.run(...
|
|
96
|
+
try: () => a.run(...S(c, l, "{}", i, !0)),
|
|
97
97
|
catch: (e) => e
|
|
98
98
|
});
|
|
99
|
-
}).pipe(
|
|
99
|
+
}).pipe(m("mirror.remove")),
|
|
100
100
|
maxVersion: ({ table: n, primaryKeyValue: r }) => e.gen(function* () {
|
|
101
101
|
yield* o("mirror.maxVersion", n);
|
|
102
|
-
let
|
|
103
|
-
try: () =>
|
|
102
|
+
let a = yield* h(t), c = s(n), l = (yield* e.tryPromise({
|
|
103
|
+
try: () => a.runAndReadAll(`SELECT version FROM ${c} WHERE id = ?`, [String(r)], [i.VARCHAR]),
|
|
104
104
|
catch: (e) => e
|
|
105
105
|
})).getRowObjectsJS()[0]?.version;
|
|
106
106
|
return l == null ? null : Number(l);
|
|
107
|
-
}).pipe(
|
|
107
|
+
}).pipe(m("mirror.maxVersion"))
|
|
108
108
|
};
|
|
109
|
-
},
|
|
110
|
-
let
|
|
109
|
+
}, w = (t, a) => {
|
|
110
|
+
let s = a.mirrorTables ?? [], c = s.length > 0 ? C(t, s, a.mirrorPrimaryKey ?? "id") : void 0;
|
|
111
111
|
return {
|
|
112
112
|
info: {
|
|
113
|
-
provider:
|
|
113
|
+
provider: n,
|
|
114
114
|
capabilities: {
|
|
115
115
|
track: !0,
|
|
116
116
|
aggregate: !0,
|
|
@@ -119,107 +119,107 @@ var r = "duckdb", i = "voltro_events", a = null, o = () => a === null ? e.tryPro
|
|
|
119
119
|
mirror: c !== void 0
|
|
120
120
|
},
|
|
121
121
|
detail: {
|
|
122
|
-
path:
|
|
123
|
-
...
|
|
122
|
+
path: a.path ?? ":memory:",
|
|
123
|
+
...s.length > 0 ? { mirrorTables: s } : {}
|
|
124
124
|
}
|
|
125
125
|
},
|
|
126
126
|
track: (n) => e.gen(function* () {
|
|
127
|
-
let
|
|
127
|
+
let a = yield* h(t), { VARCHAR: s, TIMESTAMP: c } = i, l = n.occurredAt ?? /* @__PURE__ */ new Date(), u = JSON.stringify(n.properties), d = n.subjectId ?? null, f = n.tenantId ?? null;
|
|
128
128
|
yield* e.tryPromise({
|
|
129
|
-
try: () =>
|
|
129
|
+
try: () => a.run(`INSERT INTO ${r} (name, subject_id, tenant_id, properties, occurred_at) VALUES (?, ?, ?, ?::JSON, ?)`, [
|
|
130
130
|
n.name,
|
|
131
131
|
d,
|
|
132
132
|
f,
|
|
133
133
|
u,
|
|
134
|
-
|
|
134
|
+
o(l)
|
|
135
135
|
], [
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
136
|
+
s,
|
|
137
|
+
s,
|
|
138
|
+
s,
|
|
139
|
+
s,
|
|
140
140
|
c
|
|
141
141
|
]),
|
|
142
142
|
catch: (e) => e
|
|
143
143
|
});
|
|
144
|
-
}).pipe(
|
|
144
|
+
}).pipe(m("track")),
|
|
145
145
|
aggregate: (n) => e.gen(function* () {
|
|
146
|
-
let
|
|
147
|
-
try: () =>
|
|
146
|
+
let i = yield* h(t), a = x("aggregate", n.event, n.range, n.filter, n.tenantId), o = `SELECT ${y("aggregate", n.metric)} AS value FROM ${r} WHERE ${a.sql}`, s = (yield* e.tryPromise({
|
|
147
|
+
try: () => i.runAndReadAll(o, a.params, a.types),
|
|
148
148
|
catch: (e) => e
|
|
149
149
|
})).getRowObjectsJS()[0]?.value ?? 0;
|
|
150
150
|
return typeof s == "number" ? s : Number(s ?? 0);
|
|
151
|
-
}).pipe(
|
|
151
|
+
}).pipe(m("aggregate")),
|
|
152
152
|
timeseries: (n) => e.gen(function* () {
|
|
153
|
-
let
|
|
154
|
-
SELECT ${o} AS bucket_start, ${
|
|
155
|
-
FROM ${
|
|
153
|
+
let i = yield* h(t), a = x("timeseries", n.event, n.range, n.filter, n.tenantId), o = g("occurred_at", n.bucket), s = `
|
|
154
|
+
SELECT ${o} AS bucket_start, ${y("timeseries", n.metric)} AS value
|
|
155
|
+
FROM ${r}
|
|
156
156
|
WHERE ${a.sql}
|
|
157
157
|
GROUP BY ${o}
|
|
158
158
|
ORDER BY ${o} ASC
|
|
159
159
|
`;
|
|
160
160
|
return (yield* e.tryPromise({
|
|
161
|
-
try: () =>
|
|
161
|
+
try: () => i.runAndReadAll(s, a.params, a.types),
|
|
162
162
|
catch: (e) => e
|
|
163
163
|
})).getRowObjectsJS().map((e) => ({
|
|
164
164
|
bucketStart: e.bucket_start instanceof Date ? e.bucket_start : new Date(e.bucket_start),
|
|
165
165
|
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
166
166
|
}));
|
|
167
|
-
}).pipe(
|
|
167
|
+
}).pipe(m("timeseries")),
|
|
168
168
|
topN: (n) => e.gen(function* () {
|
|
169
|
-
let
|
|
170
|
-
SELECT ${s} AS key, ${
|
|
171
|
-
FROM ${
|
|
169
|
+
let i = yield* h(t), a = x("topN", n.event, n.range, n.filter, n.tenantId), o = Math.max(1, Math.floor(n.n)), s = _("topN", n.groupBy), c = `
|
|
170
|
+
SELECT ${s} AS key, ${b("topN", n.metric)} AS value
|
|
171
|
+
FROM ${r}
|
|
172
172
|
WHERE ${a.sql}
|
|
173
173
|
GROUP BY ${s}
|
|
174
174
|
ORDER BY value DESC
|
|
175
175
|
LIMIT ${o}
|
|
176
176
|
`;
|
|
177
177
|
return (yield* e.tryPromise({
|
|
178
|
-
try: () =>
|
|
178
|
+
try: () => i.runAndReadAll(c, a.params, a.types),
|
|
179
179
|
catch: (e) => e
|
|
180
180
|
})).getRowObjectsJS().filter((e) => e.key !== null).map((e) => ({
|
|
181
181
|
key: e.key,
|
|
182
182
|
value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
|
|
183
183
|
}));
|
|
184
|
-
}).pipe(
|
|
184
|
+
}).pipe(m("topN")),
|
|
185
185
|
...c === void 0 ? {} : { mirror: c }
|
|
186
186
|
};
|
|
187
|
-
},
|
|
188
|
-
if (
|
|
189
|
-
let
|
|
187
|
+
}, T = (r, i) => e.gen(function* () {
|
|
188
|
+
if (r.conn !== null) return;
|
|
189
|
+
let o = yield* a(), s = yield* e.tryPromise({
|
|
190
190
|
try: async () => {
|
|
191
|
-
let e = await
|
|
192
|
-
return await t.run(
|
|
191
|
+
let e = await o.DuckDBInstance.create(i.path ?? ":memory:"), t = await e.connect();
|
|
192
|
+
return await t.run(p), {
|
|
193
193
|
instance: e,
|
|
194
194
|
connection: t
|
|
195
195
|
};
|
|
196
196
|
},
|
|
197
|
-
catch: (e) => new
|
|
198
|
-
provider:
|
|
197
|
+
catch: (e) => new t({
|
|
198
|
+
provider: n,
|
|
199
199
|
operation: "initialize",
|
|
200
200
|
cause: e instanceof Error ? `${e.name}: ${e.message}` : String(e)
|
|
201
201
|
})
|
|
202
202
|
});
|
|
203
|
-
|
|
204
|
-
}),
|
|
205
|
-
let { conn:
|
|
206
|
-
t.conn = null, t.instance = null, (
|
|
207
|
-
|
|
208
|
-
}).pipe(e.
|
|
209
|
-
provider:
|
|
203
|
+
r.instance = s.instance, r.conn = s.connection;
|
|
204
|
+
}), E = (t) => e.gen(function* () {
|
|
205
|
+
let { conn: r, instance: i } = t;
|
|
206
|
+
t.conn = null, t.instance = null, (r !== null || i !== null) && (yield* e.try(() => {
|
|
207
|
+
r?.closeSync(), i?.closeSync();
|
|
208
|
+
}).pipe(e.catch((t) => e.logWarning("duckdb analytics sink teardown failed", {
|
|
209
|
+
provider: n,
|
|
210
210
|
err: t
|
|
211
211
|
}))));
|
|
212
|
-
}),
|
|
212
|
+
}), D = (e = {}) => {
|
|
213
213
|
let t = {
|
|
214
214
|
conn: null,
|
|
215
215
|
instance: null
|
|
216
216
|
};
|
|
217
217
|
return {
|
|
218
218
|
kind: "duckdb",
|
|
219
|
-
build: (n) =>
|
|
220
|
-
initialize: () =>
|
|
221
|
-
dispose: () =>
|
|
219
|
+
build: (n) => w(t, e),
|
|
220
|
+
initialize: () => T(t, e),
|
|
221
|
+
dispose: () => E(t)
|
|
222
222
|
};
|
|
223
223
|
};
|
|
224
224
|
//#endregion
|
|
225
|
-
export {
|
|
225
|
+
export { D as duckdbAnalytics };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@voltro/plugin-duckdb",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.85.0",
|
|
4
4
|
"description": "Embedded DuckDB AnalyticsSink. Runs DuckDB in-process for real OLAP performance without an external service. Events land in a DuckDB column-store table; aggregate / timeseries / topN queries execute vectorized.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"voltro",
|
|
@@ -33,13 +33,13 @@
|
|
|
33
33
|
"node": ">=24.0.0"
|
|
34
34
|
},
|
|
35
35
|
"dependencies": {
|
|
36
|
-
"@voltro/runtime": "0.
|
|
36
|
+
"@voltro/runtime": "0.85.0"
|
|
37
37
|
},
|
|
38
38
|
"optionalDependencies": {
|
|
39
39
|
"@duckdb/node-api": "1.5.5-r.5"
|
|
40
40
|
},
|
|
41
41
|
"peerDependencies": {
|
|
42
|
-
"effect": "
|
|
42
|
+
"effect": "4.0.1"
|
|
43
43
|
},
|
|
44
44
|
"publishConfig": {
|
|
45
45
|
"access": "public"
|