@voltro/plugin-clickhouse 0.1.0

package/dist/index.d.ts

@@ -0,0 +1,82 @@
+import { AggregateQuery } from '@voltro/runtime';
+import { AnalyticsBucket } from '@voltro/runtime';
+import { AnalyticsEvent } from '@voltro/runtime';
+import { AnalyticsSinkSpec } from '@voltro/runtime';
+import { AnalyticsTopEntry } from '@voltro/runtime';
+import { TimeseriesQuery } from '@voltro/runtime';
+import { TopNQuery } from '@voltro/runtime';
+
+export { AggregateQuery }
+
+export { AnalyticsBucket }
+
+export { AnalyticsEvent }
+
+export { AnalyticsTopEntry }
+
+/**
+ * Construct the spec. `initialize` opens the client + creates the
+ * events table on first boot.
+ */
+export declare const clickhouseAnalytics: (options: ClickhouseAnalyticsOptions) => AnalyticsSinkSpec;
+
+export declare interface ClickhouseAnalyticsOptions {
+    /** HTTP URL of the ClickHouse server (incl. protocol + port). */
+    readonly url: string;
+    readonly username?: string;
+    readonly password?: string;
+    /** Database name. The plugin creates the `events` table inside it
+     *  on first boot. Default: 'default'. */
+    readonly database?: string;
+    /** Override the events-table name. Default: 'events'. */
+    readonly table?: string;
+    /**
+     * Reactive tables to CDC-mirror into ClickHouse. Each table gets a
+     * `voltro_mirror_<table>` ReplacingMergeTree (`{ id, data, version,
+     * is_deleted }`); the framework streams `DataStore` changes into it so
+     * analytical queries can JOIN events against live user data:
+     *
+     *   SELECT JSONExtractString(m.data, 'tier'), count()
+     *   FROM voltro_events e
+     *   JOIN voltro_mirror_users FINAL m ON m.id = e.subject_id
+     *   WHERE m.is_deleted = 0
+     *   GROUP BY 1
+     *
+     * Empty / omitted → events-only.
+     */
+    readonly mirrorTables?: ReadonlyArray<string>;
+    /** Primary-key column on the mirrored source rows. Default `'id'`. */
+    readonly mirrorPrimaryKey?: string;
+    /** Opt into client-side batching of `track()` inserts. Omit for immediate
+     *  per-event inserts (a successful `track` = the row was accepted). See the
+     *  batching note in the README for the delivery-semantics trade-off. */
+    readonly batch?: ClickhouseBatchOptions;
+}
+
+/** Opt-in client-side batching. Buffers `track()` rows and flushes them in
+ *  ONE multi-row `client.insert`, cutting HTTP round-trips under load. */
+export declare interface ClickhouseBatchOptions {
+    /** Flush once the buffer reaches this many events. Default 1000. */
+    readonly maxSize?: number;
+    /** Flush a non-empty buffer this many ms after the first event lands
+     *  (so low-volume trickles don't sit until shutdown). Default 5000. */
+    readonly flushIntervalMs?: number;
+}
+
+export declare const EVENTS_TABLE_DDL: (database: string, table: string) => string;
+
+/**
+ * DDL for a mirror table. `ReplacingMergeTree(version)` keyed by `id`
+ * gives idempotent upserts — re-inserting the same row with a newer
+ * `version` collapses to the latest on merge. Deletes insert a tombstone
+ * (`is_deleted = 1`); queries filter `is_deleted = 0` (or use the
+ * `FINAL` modifier). `version` is a monotonic micro-timestamp so a later
+ * write always wins.
+ */
+export declare const MIRROR_TABLE_DDL: (database: string, sourceTable: string) => string;
+
+export { TimeseriesQuery }
+
+export { TopNQuery }
+
+export { }

package/dist/index.js

@@ -0,0 +1,282 @@
+import { createClient as e } from "@clickhouse/client";
+import { Effect as t, Option as n } from "effect";
+import { createLogger as r } from "@voltro/logger";
+import { AnalyticsError as i } from "@voltro/runtime";
+//#region src/sink.ts
+var a = r({ scope: "@voltro/plugin-clickhouse" }), o = "clickhouse", s = (e) => `voltro_mirror_${e}`, c = (e, t) => `
+  CREATE TABLE IF NOT EXISTS ${e}.${s(t)} (
+    id          String,
+    data        String CODEC(ZSTD(1)),
+    version     UInt64,
+    is_deleted  UInt8 DEFAULT 0
+  ) ENGINE = ReplacingMergeTree(version)
+  ORDER BY id
+`.trim(), l = (e, t) => `
+  CREATE TABLE IF NOT EXISTS ${e}.${t} (
+    id          UUID DEFAULT generateUUIDv7(),
+    name        LowCardinality(String),
+    subject_id  Nullable(String),
+    tenant_id   Nullable(String),
+    properties  String CODEC(ZSTD(1)),
+    occurred_at DateTime64(3),
+    ingested_at DateTime64(3) DEFAULT now64()
+  ) ENGINE = MergeTree
+  ORDER BY (name, occurred_at, id)
+`.trim(), u = (e) => (t) => t instanceof i ? new i({
+	provider: o,
+	operation: e,
+	cause: t.cause
+}) : new i({
+	provider: o,
+	operation: e,
+	cause: t instanceof Error ? `${t.name}: ${t.message}` : String(t)
+}), d = (e) => (r) => r.pipe(t.catchSomeDefect((e) => e instanceof i ? n.some(t.fail(e)) : n.none()), t.mapError(u(e))), f = /^[A-Za-z_][A-Za-z0-9_]*$/, p = (e, t) => {
+	if (!f.test(e)) throw new i({
+		provider: o,
+		operation: "config",
+		cause: `invalid ${t} ${JSON.stringify(e)} — must match ${f.source} (letters, digits, underscore; no dots/quotes/spaces)`
+	});
+	return e;
+}, m = /['\\]/, h = (e) => {
+	for (let t = 0; t < e.length; t++) if (e.charCodeAt(t) < 32) return !0;
+	return !1;
+}, g = (e) => (t) => {
+	if (m.test(t) || h(t)) throw new i({
+		provider: o,
+		operation: e,
+		cause: `invalid property key ${JSON.stringify(t)} — must not contain quotes, backslashes, or control characters`
+	});
+	return t;
+}, _ = (e) => e.client === null ? t.fail(new i({
+	provider: o,
+	operation: "ensureClient",
+	cause: "client not initialized — spec.initialize must run before queries"
+})) : t.succeed(e.client), v = (e) => t.gen(function* () {
+	let n = e.client;
+	e.client = null, n !== null && (yield* t.tryPromise(() => n.close()).pipe(t.catchAll((e) => t.logWarning("clickhouse analytics sink teardown failed", {
+		provider: o,
+		err: e
+	}))));
+}), y = (e) => ({
+	name: e.name,
+	subject_id: e.subjectId ?? null,
+	tenant_id: e.tenantId ?? null,
+	properties: JSON.stringify(e.properties),
+	occurred_at: (e.occurredAt ?? /* @__PURE__ */ new Date()).toISOString()
+}), b = (e, t) => {
+	let n = [], r, i = () => {
+		r !== void 0 && (clearTimeout(r), r = void 0);
+	}, o = async () => {
+		if (i(), n.length === 0) return;
+		let t = n;
+		n = [];
+		let r = e.client;
+		if (r === null) {
+			a.warn("clickhouse batch flush skipped — client not initialized; events dropped", { dropped: t.length });
+			return;
+		}
+		try {
+			await r.insert({
+				table: `${e.database}.${e.table}`,
+				values: t,
+				format: "JSONEachRow"
+			});
+		} catch (e) {
+			a.warn("clickhouse batch flush failed — dropped events", { dropped: t.length }, e);
+		}
+	};
+	return {
+		add: (e) => {
+			if (n.push(y(e)), n.length >= t.maxSize) {
+				o();
+				return;
+			}
+			r === void 0 && (r = setTimeout(() => {
+				o();
+			}, t.flushIntervalMs), r.unref?.());
+		},
+		flush: o
+	};
+}, x = (e) => {
+	switch (e) {
+		case "minute": return "toStartOfMinute";
+		case "hour": return "toStartOfHour";
+		case "day": return "toStartOfDay";
+		case "week": return "toStartOfWeek";
+		case "month": return "toStartOfMonth";
+	}
+}, S = (e, t) => `JSONExtractString(properties, '${g(e)(t)}')`, C = (e, t) => `JSONExtractFloat(properties, '${g(e)(t)}')`, w = (e, t) => t === "count" ? "count()" : t === "unique" ? "uniqExact(subject_id)" : "sum" in t ? `sum(${C(e, t.sum)})` : `avg(${C(e, t.avg)})`, T = (e, t) => t === "count" ? "count()" : `sum(${C(e, t.sum)})`, E = (e, t, n, r, i) => {
+	let a = {
+		event: t,
+		from: n.from.toISOString(),
+		to: (n.to ?? /* @__PURE__ */ new Date()).toISOString()
+	}, o = "name = {event:String} AND occurred_at >= {from:DateTime64(3)} AND occurred_at < {to:DateTime64(3)}";
+	if (i != null && (a.tenant = i, o += " AND tenant_id = {tenant:String}"), r !== void 0) {
+		let t = 0;
+		for (let [n, i] of Object.entries(r)) {
+			let r = `f${t++}`;
+			a[r] = String(i), o += ` AND ${S(e, n)} = {${r}:String}`;
+		}
+	}
+	return {
+		sql: o,
+		params: a
+	};
+}, D = (e) => {
+	let n = (n, r) => t.gen(function* () {
+		p(n, "mirror table");
+		let i = yield* _(e);
+		yield* t.tryPromise({
+			try: () => i.insert({
+				table: `${e.database}.${s(n)}`,
+				values: [r],
+				format: "JSONEachRow"
+			}),
+			catch: (e) => e
+		});
+	});
+	return {
+		tables: e.mirrorTables,
+		primaryKey: e.mirrorPrimaryKey,
+		upsert: (r, i) => t.gen(function* () {
+			let t = String(i[e.mirrorPrimaryKey]);
+			yield* n(r, {
+				id: t,
+				data: JSON.stringify(i),
+				version: Date.now() * 1e3,
+				is_deleted: 0
+			});
+		}).pipe(d("mirror.upsert")),
+		remove: (e, r) => t.gen(function* () {
+			yield* n(e, {
+				id: String(r),
+				data: "{}",
+				version: Date.now() * 1e3,
+				is_deleted: 1
+			});
+		}).pipe(d("mirror.remove"))
+	};
+}, O = (e) => {
+	let n = e.mirrorTables.length > 0 ? D(e) : void 0;
+	return {
+		info: {
+			provider: o,
+			capabilities: {
+				track: !0,
+				aggregate: !0,
+				timeseries: !0,
+				topN: !0,
+				mirror: n !== void 0
+			},
+			detail: {
+				database: e.database,
+				table: e.table,
+				...e.mirrorTables.length > 0 ? { mirrorTables: e.mirrorTables } : {},
+				...e.batcher === void 0 ? {} : { batching: !0 }
+			}
+		},
+		track: (n) => e.batcher ? t.sync(() => e.batcher.add(n)) : t.gen(function* () {
+			let r = yield* _(e);
+			yield* t.tryPromise({
+				try: () => r.insert({
+					table: `${e.database}.${e.table}`,
+					values: [y(n)],
+					format: "JSONEachRow"
+				}),
+				catch: (e) => e
+			});
+		}).pipe(d("track")),
+		aggregate: (n) => t.gen(function* () {
+			let r = yield* _(e), i = E("aggregate", n.event, n.range, n.filter, n.tenantId), a = `SELECT ${w("aggregate", n.metric)} AS value FROM ${e.database}.${e.table} WHERE ${i.sql} FORMAT JSON`, o = yield* t.tryPromise({
+				try: () => r.query({
+					query: a,
+					query_params: i.params
+				}),
+				catch: (e) => e
+			}), { data: s } = yield* t.tryPromise({
+				try: () => o.json(),
+				catch: (e) => e
+			}), c = s[0]?.value ?? 0;
+			return typeof c == "number" ? c : Number(c ?? 0);
+		}).pipe(d("aggregate")),
+		timeseries: (n) => t.gen(function* () {
+			let r = yield* _(e), i = E("timeseries", n.event, n.range, n.filter, n.tenantId), a = `
+        SELECT ${`${x(n.bucket)}(occurred_at)`} AS bucket_start, ${w("timeseries", n.metric)} AS value
+        FROM ${e.database}.${e.table}
+        WHERE ${i.sql}
+        GROUP BY bucket_start
+        ORDER BY bucket_start ASC
+        FORMAT JSON
+      `, o = yield* t.tryPromise({
+				try: () => r.query({
+					query: a,
+					query_params: i.params
+				}),
+				catch: (e) => e
+			}), { data: s } = yield* t.tryPromise({
+				try: () => o.json(),
+				catch: (e) => e
+			});
+			return s.map((e) => ({
+				bucketStart: new Date(e.bucket_start),
+				value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
+			}));
+		}).pipe(d("timeseries")),
+		topN: (n) => t.gen(function* () {
+			let r = yield* _(e), i = E("topN", n.event, n.range, n.filter, n.tenantId), a = Math.max(1, Math.floor(n.n)), o = S("topN", n.groupBy), s = `
+        SELECT ${o} AS key, ${T("topN", n.metric)} AS value
+        FROM ${e.database}.${e.table}
+        WHERE ${i.sql}
+        GROUP BY ${o}
+        ORDER BY value DESC
+        LIMIT ${a}
+        FORMAT JSON
+      `, c = yield* t.tryPromise({
+				try: () => r.query({
+					query: s,
+					query_params: i.params
+				}),
+				catch: (e) => e
+			}), { data: l } = yield* t.tryPromise({
+				try: () => c.json(),
+				catch: (e) => e
+			});
+			return l.filter((e) => e.key !== null).map((e) => ({
+				key: e.key,
+				value: typeof e.value == "number" ? e.value : Number(e.value ?? 0)
+			}));
+		}).pipe(d("topN")),
+		...n === void 0 ? {} : { mirror: n }
+	};
+}, k = 1e3, A = 5e3, j = (n) => {
+	let r = (n.mirrorTables ?? []).map((e) => p(e, "mirror table")), i = {
+		client: null,
+		database: p(n.database ?? "default", "database"),
+		table: p(n.table ?? "events", "table"),
+		mirrorTables: r,
+		mirrorPrimaryKey: n.mirrorPrimaryKey ?? "id"
+	};
+	return n.batch && (i.batcher = b(i, {
+		maxSize: n.batch.maxSize ?? k,
+		flushIntervalMs: n.batch.flushIntervalMs ?? A
+	})), {
+		kind: "clickhouse",
+		build: (e) => O(i),
+		initialize: () => t.gen(function* () {
+			let r = yield* t.promise(() => Promise.resolve(e({
+				url: n.url,
+				database: i.database,
+				...n.username === void 0 ? {} : { username: n.username },
+				...n.password === void 0 ? {} : { password: n.password }
+			})));
+			yield* t.promise(() => r.ping()), yield* t.promise(() => r.command({ query: l(i.database, i.table) }));
+			for (let e of i.mirrorTables) yield* t.promise(() => r.command({ query: c(i.database, e) }));
+			i.client = r;
+		}),
+		dispose: () => t.gen(function* () {
+			i.batcher && (yield* t.promise(() => i.batcher.flush())), yield* v(i);
+		})
+	};
+};
+//#endregion
+export { l as EVENTS_TABLE_DDL, c as MIRROR_TABLE_DDL, j as clickhouseAnalytics };

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@voltro/plugin-clickhouse",
+  "version": "0.1.0",
+  "description": "ClickHouse AnalyticsSink. Native HTTP ingest into a ClickHouse cluster — self-hosted OR ClickHouse Cloud. Production-grade OLAP for high-cardinality events with millisecond aggregate queries over billions of rows.",
+  "keywords": [
+    "voltro",
+    "typescript",
+    "framework"
+  ],
+  "license": "SEE LICENSE IN LICENSE",
+  "homepage": "https://voltro.dev",
+  "bugs": {
+    "email": "support@voltro.dev"
+  },
+  "author": {
+    "name": "Voltro UG",
+    "url": "https://voltro.dev"
+  },
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "sideEffects": false,
+  "engines": {
+    "node": ">=24.0.0"
+  },
+  "dependencies": {
+    "@clickhouse/client": "^1.23.1",
+    "@voltro/logger": "0.1.0",
+    "@voltro/runtime": "0.1.0"
+  },
+  "peerDependencies": {
+    "effect": "^3.21.4"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}