@voltro/plugin-clickhouse 0.33.0 → 0.35.0

package/dist/index.d.ts

@@ -47,16 +47,28 @@ export declare interface ClickhouseAnalyticsOptions {
     readonly mirrorTables?: ReadonlyArray<string>;
     /** Primary-key column on the mirrored source rows. Default `'id'`. */
     readonly mirrorPrimaryKey?: string;
-    /** Opt into client-side batching of `track()` inserts. Omit for immediate
-     *  per-event inserts (a successful `track` = the row was accepted). See the
-     *  batching note in the README for the delivery-semantics trade-off. */
-    readonly batch?: ClickhouseBatchOptions;
+    /**
+     * Client-side batching of `track()` inserts — ON by default (20 events /
+     * 5 s, plugin-posthog's numbers). What changes observably with batching: a
+     * successful `track()` means "buffered", the actual insert happens off the
+     * call path (flushed by size, on the timer, and on graceful shutdown via
+     * `dispose`; a flush failure is logged + that batch dropped; a hard crash
+     * loses whatever is still buffered).
+     *
+     * `batch: false` opts OUT: every `track()` is one immediate HTTP insert and
+     * a success means ClickHouse accepted the row — at one round-trip and one
+     * MergeTree part per event, which is the part-explosion PERF-11 measured.
+     * `batch: {...}` keeps batching and tunes size/interval.
+     */
+    readonly batch?: ClickhouseBatchOptions | false;
 }
 
-/** Opt-in client-side batching. Buffers `track()` rows and flushes them in
- *  ONE multi-row `client.insert`, cutting HTTP round-trips under load. */
+/** Client-side batching tunables (batching is ON by default). Buffers
+ *  `track()` rows and flushes them in ONE multi-row `client.insert`, cutting
+ *  HTTP round-trips and MergeTree part churn under load. */
 export declare interface ClickhouseBatchOptions {
-    /** Flush once the buffer reaches this many events. Default 1000. */
+    /** Flush once the buffer reaches this many events. Default 20 (mirrors
+     *  plugin-posthog's conservative batcher). */
     readonly maxSize?: number;
     /** Flush a non-empty buffer this many ms after the first event lands
      *  (so low-volume trickles don't sit until shutdown). Default 5000. */
@@ -70,8 +82,13 @@ export declare const EVENTS_TABLE_DDL: (database: string, table: string) => stri
  * gives idempotent upserts — re-inserting the same row with a newer
  * `version` collapses to the latest on merge. Deletes insert a tombstone
  * (`is_deleted = 1`); queries filter `is_deleted = 0` (or use the
- * `FINAL` modifier). `version` is a monotonic micro-timestamp so a later
- * write always wins.
+ * `FINAL` modifier).
+ *
+ * `version` is the framework's per-change ordering token
+ * (`MirrorVersion`), assigned in COMMIT order and carried on the write.
+ * It is never read from a clock inside this sink — that is what makes
+ * "a later write always wins" true of the later COMMIT rather than of
+ * whichever HTTP request happened to arrive last.
  */
 export declare const MIRROR_TABLE_DDL: (database: string, sourceTable: string) => string;
 

package/dist/index.js

@@ -138,23 +138,37 @@ var a = r({ scope: "@voltro/plugin-clickhouse" }), o = "clickhouse", s = (e) =>
 	return {
 		tables: e.mirrorTables,
 		primaryKey: e.mirrorPrimaryKey,
-		upsert: (r, i) => t.gen(function* () {
+		upsert: ({ table: r, row: i, version: a }) => t.gen(function* () {
 			let t = String(i[e.mirrorPrimaryKey]);
 			yield* n(r, {
 				id: t,
 				data: JSON.stringify(i),
-				version: Date.now() * 1e3,
+				version: a,
 				is_deleted: 0
 			});
 		}).pipe(d("mirror.upsert")),
-		remove: (e, r) => t.gen(function* () {
+		remove: ({ table: e, primaryKeyValue: r, version: i }) => t.gen(function* () {
 			yield* n(e, {
 				id: String(r),
 				data: "{}",
-				version: Date.now() * 1e3,
+				version: i,
 				is_deleted: 1
 			});
-		}).pipe(d("mirror.remove"))
+		}).pipe(d("mirror.remove")),
+		maxVersion: ({ table: n, primaryKeyValue: r }) => t.gen(function* () {
+			p(String(n), "mirror table");
+			let i = yield* _(e), a = `SELECT max(version) AS v, count() AS n FROM ${e.database}.${s(String(n))} WHERE id = {id: String} FORMAT JSON`, o = yield* t.tryPromise({
+				try: () => i.query({
+					query: a,
+					query_params: { id: String(r) }
+				}),
+				catch: (e) => e
+			}), { data: c } = yield* t.tryPromise({
+				try: () => o.json(),
+				catch: (e) => e
+			}), l = c[0];
+			return l === void 0 || Number(l.n) === 0 || l.v === null ? null : Number(l.v);
+		}).pipe(d("mirror.maxVersion"))
 	};
 }, O = (e) => {
 	let n = e.mirrorTables.length > 0 ? D(e) : void 0;
@@ -248,7 +262,7 @@ var a = r({ scope: "@voltro/plugin-clickhouse" }), o = "clickhouse", s = (e) =>
 		}).pipe(d("topN")),
 		...n === void 0 ? {} : { mirror: n }
 	};
-}, k = 1e3, A = 5e3, j = (n) => {
+}, k = 20, A = 5e3, j = (n) => {
 	let r = (n.mirrorTables ?? []).map((e) => p(e, "mirror table")), i = {
 		client: null,
 		database: p(n.database ?? "default", "database"),
@@ -256,10 +270,14 @@ var a = r({ scope: "@voltro/plugin-clickhouse" }), o = "clickhouse", s = (e) =>
 		mirrorTables: r,
 		mirrorPrimaryKey: n.mirrorPrimaryKey ?? "id"
 	};
-	return n.batch && (i.batcher = b(i, {
-		maxSize: n.batch.maxSize ?? k,
-		flushIntervalMs: n.batch.flushIntervalMs ?? A
-	})), {
+	if (n.batch !== !1) {
+		let e = n.batch ?? {};
+		i.batcher = b(i, {
+			maxSize: e.maxSize ?? k,
+			flushIntervalMs: e.flushIntervalMs ?? A
+		});
+	}
+	return {
 		kind: "clickhouse",
 		build: (e) => O(i),
 		initialize: () => t.gen(function* () {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@voltro/plugin-clickhouse",
-  "version": "0.33.0",
+  "version": "0.35.0",
   "description": "ClickHouse AnalyticsSink. Native HTTP ingest into a ClickHouse cluster — self-hosted OR ClickHouse Cloud. Production-grade OLAP for high-cardinality events with millisecond aggregate queries over billions of rows.",
   "keywords": [
     "voltro",
@@ -34,8 +34,8 @@
   },
   "dependencies": {
     "@clickhouse/client": "^1.23.1",
-    "@voltro/logger": "0.33.0",
-    "@voltro/runtime": "0.33.0"
+    "@voltro/logger": "0.35.0",
+    "@voltro/runtime": "0.35.0"
   },
   "peerDependencies": {
     "effect": "^3.22.0"