@walkeros/server-destination-clickhouse 0.0.0-bootstrap

package/dist/examples/index.d.mts

@@ -0,0 +1,91 @@
+import { DestinationServer } from '@walkeros/server-core';
+import { ClickHouseClientConfigOptions, ClickHouseClient } from '@clickhouse/client';
+import { Flow } from '@walkeros/core';
+
+/**
+ * The only two client methods this package is allowed to reach for. Narrowing
+ * the seam to them makes the INSERT-only rule a compile-time property rather
+ * than a convention: `query`, `command`, `exec` and `ping` are not reachable
+ * from a client typed this way, so no code path can read `system.*` or issue
+ * DDL. The SDK's own `createClient` stays assignable by structural typing.
+ */
+type ClickHouseClientSurface = Pick<ClickHouseClient, 'insert' | 'close'>;
+/** Creates the client this destination writes through. Matches `createClient`. */
+type ClickHouseClientFactory = (config: ClickHouseClientConfigOptions) => ClickHouseClientSurface;
+/**
+ * Env carries the client factory so tests and simulations inject a recording
+ * stand-in instead of opening a socket. Production leaves it undefined and the
+ * destination uses the SDK's own `createClient`.
+ */
+interface Env extends DestinationServer.Env {
+    ClickHouseClient?: ClickHouseClientFactory;
+}
+
+/**
+ * Example environment for the ClickHouse destination.
+ *
+ * The client factory is the recording stand-in from the package-local
+ * __mocks__ folder, so examples, tests and simulations share one source of
+ * truth and never reach a ClickHouse server.
+ */
+declare const push: Env;
+declare const simulation: string[];
+
+declare const env_push: typeof push;
+declare const env_simulation: typeof simulation;
+declare namespace env {
+  export { env_push as push, env_simulation as simulation };
+}
+
+/**
+ * Destination bootstrap.
+ *
+ * `in` is the config a user copy-pastes into a flow: the endpoint, the
+ * database and table to write to, and the credentials slot backed by a managed
+ * secret. Init creates exactly one ClickHouse client from it and holds that
+ * client on the resolved config, so every insert reuses the same connection
+ * pool.
+ *
+ * Two options the user did not write reach the client. `request_timeout` is
+ * derived from the collector's per-delivery timeout (10000 ms by default) minus
+ * what the retry delays can take, split across the attempts: at one retry that
+ * is 4812 ms, so the second attempt finishes before the collector stops waiting
+ * and dead-letters a batch the client would still land. Request compression is
+ * off in this client by default and insert bodies are large, repetitive JSON,
+ * so it is turned on. Both yield to `settings.clickhouse`.
+ */
+declare const init: Flow.StepExample;
+/**
+ * What happens to the raw `settings.clickhouse` passthrough.
+ *
+ * The merge runs in four layers, lowest first: the passthrough, then the
+ * derived defaults where the passthrough left their key unset, then `url` and
+ * `database`, then `credentials`. An insert names only the table, so a
+ * passthrough `database` that disagreed with the configured one would move
+ * every row to a different database in silence; the configured one wins. The
+ * credentials slot wins over a username or password left in the passthrough,
+ * and an option the destination does not name travels through untouched.
+ */
+declare const initPassthrough: Flow.StepExample;
+/**
+ * One event, one row, one insert. Every payload travels as a JSON-encoded
+ * string and the timestamp as a `YYYY-MM-DD hh:mm:ss.SSS` UTC literal, the
+ * format every ClickHouse date parser reads identically.
+ */
+declare const pageView: Flow.StepExample;
+/**
+ * An order carries an array of items. The whole `data` object, array
+ * included, is JSON-encoded into the single `data` column, so a flow can send
+ * any shape without the table knowing about it.
+ */
+declare const purchase: Flow.StepExample;
+
+declare const step_init: typeof init;
+declare const step_initPassthrough: typeof initPassthrough;
+declare const step_pageView: typeof pageView;
+declare const step_purchase: typeof purchase;
+declare namespace step {
+  export { step_init as init, step_initPassthrough as initPassthrough, step_pageView as pageView, step_purchase as purchase };
+}
+
+export { env, step };

package/dist/examples/index.d.ts

@@ -0,0 +1,91 @@
+import { DestinationServer } from '@walkeros/server-core';
+import { ClickHouseClientConfigOptions, ClickHouseClient } from '@clickhouse/client';
+import { Flow } from '@walkeros/core';
+
+/**
+ * The only two client methods this package is allowed to reach for. Narrowing
+ * the seam to them makes the INSERT-only rule a compile-time property rather
+ * than a convention: `query`, `command`, `exec` and `ping` are not reachable
+ * from a client typed this way, so no code path can read `system.*` or issue
+ * DDL. The SDK's own `createClient` stays assignable by structural typing.
+ */
+type ClickHouseClientSurface = Pick<ClickHouseClient, 'insert' | 'close'>;
+/** Creates the client this destination writes through. Matches `createClient`. */
+type ClickHouseClientFactory = (config: ClickHouseClientConfigOptions) => ClickHouseClientSurface;
+/**
+ * Env carries the client factory so tests and simulations inject a recording
+ * stand-in instead of opening a socket. Production leaves it undefined and the
+ * destination uses the SDK's own `createClient`.
+ */
+interface Env extends DestinationServer.Env {
+    ClickHouseClient?: ClickHouseClientFactory;
+}
+
+/**
+ * Example environment for the ClickHouse destination.
+ *
+ * The client factory is the recording stand-in from the package-local
+ * __mocks__ folder, so examples, tests and simulations share one source of
+ * truth and never reach a ClickHouse server.
+ */
+declare const push: Env;
+declare const simulation: string[];
+
+declare const env_push: typeof push;
+declare const env_simulation: typeof simulation;
+declare namespace env {
+  export { env_push as push, env_simulation as simulation };
+}
+
+/**
+ * Destination bootstrap.
+ *
+ * `in` is the config a user copy-pastes into a flow: the endpoint, the
+ * database and table to write to, and the credentials slot backed by a managed
+ * secret. Init creates exactly one ClickHouse client from it and holds that
+ * client on the resolved config, so every insert reuses the same connection
+ * pool.
+ *
+ * Two options the user did not write reach the client. `request_timeout` is
+ * derived from the collector's per-delivery timeout (10000 ms by default) minus
+ * what the retry delays can take, split across the attempts: at one retry that
+ * is 4812 ms, so the second attempt finishes before the collector stops waiting
+ * and dead-letters a batch the client would still land. Request compression is
+ * off in this client by default and insert bodies are large, repetitive JSON,
+ * so it is turned on. Both yield to `settings.clickhouse`.
+ */
+declare const init: Flow.StepExample;
+/**
+ * What happens to the raw `settings.clickhouse` passthrough.
+ *
+ * The merge runs in four layers, lowest first: the passthrough, then the
+ * derived defaults where the passthrough left their key unset, then `url` and
+ * `database`, then `credentials`. An insert names only the table, so a
+ * passthrough `database` that disagreed with the configured one would move
+ * every row to a different database in silence; the configured one wins. The
+ * credentials slot wins over a username or password left in the passthrough,
+ * and an option the destination does not name travels through untouched.
+ */
+declare const initPassthrough: Flow.StepExample;
+/**
+ * One event, one row, one insert. Every payload travels as a JSON-encoded
+ * string and the timestamp as a `YYYY-MM-DD hh:mm:ss.SSS` UTC literal, the
+ * format every ClickHouse date parser reads identically.
+ */
+declare const pageView: Flow.StepExample;
+/**
+ * An order carries an array of items. The whole `data` object, array
+ * included, is JSON-encoded into the single `data` column, so a flow can send
+ * any shape without the table knowing about it.
+ */
+declare const purchase: Flow.StepExample;
+
+declare const step_init: typeof init;
+declare const step_initPassthrough: typeof initPassthrough;
+declare const step_pageView: typeof pageView;
+declare const step_purchase: typeof purchase;
+declare namespace step {
+  export { step_init as init, step_initPassthrough as initPassthrough, step_pageView as pageView, step_purchase as purchase };
+}
+
+export { env, step };

package/dist/examples/index.js

@@ -0,0 +1,279 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/examples/index.ts
+var examples_exports = {};
+__export(examples_exports, {
+  env: () => env_exports,
+  step: () => step_exports
+});
+module.exports = __toCommonJS(examples_exports);
+
+// src/examples/env.ts
+var env_exports = {};
+__export(env_exports, {
+  push: () => push,
+  simulation: () => simulation
+});
+
+// src/__mocks__/@clickhouse/client.ts
+var client_exports = {};
+__export(client_exports, {
+  __failInsert: () => __failInsert,
+  __getCalls: () => __getCalls,
+  __reset: () => __reset,
+  createClient: () => createClient
+});
+var import_client = require("@clickhouse/client");
+__reExport(client_exports, require("@clickhouse/client"));
+var calls = [];
+var failure;
+function __getCalls() {
+  return calls;
+}
+function __reset() {
+  calls.length = 0;
+  failure = void 0;
+}
+function __failInsert(times, options = {}) {
+  failure = {
+    remaining: times,
+    code: options.code,
+    message: options.message ?? "clickhouse insert failed",
+    type: options.type
+  };
+}
+function nextFailure() {
+  if (!failure || failure.remaining < 1) return void 0;
+  failure.remaining -= 1;
+  const { code, message, type } = failure;
+  return code === void 0 ? new Error(message) : new import_client.ClickHouseError({ message, code, type });
+}
+var createClient = (config) => {
+  calls.push(["createClient", config]);
+  return {
+    async insert(params) {
+      calls.push(["insert", params]);
+      const error = nextFailure();
+      if (error) throw error;
+      return {
+        executed: true,
+        query_id: `mock-insert-${calls.length}`,
+        response_headers: {}
+      };
+    },
+    async close() {
+      calls.push(["close"]);
+    }
+  };
+};
+
+// src/examples/env.ts
+var push = {
+  ClickHouseClient: createClient
+};
+var simulation = ["ClickHouseClient"];
+
+// src/examples/step.ts
+var step_exports = {};
+__export(step_exports, {
+  init: () => init,
+  initPassthrough: () => initPassthrough,
+  pageView: () => pageView,
+  purchase: () => purchase
+});
+var import_core = require("@walkeros/core");
+
+// src/eventToRow.ts
+function pad(value, length) {
+  return String(value).padStart(length, "0");
+}
+function toDateTime(timestamp) {
+  const date = new Date(timestamp);
+  const day = [
+    pad(date.getUTCFullYear(), 4),
+    pad(date.getUTCMonth() + 1, 2),
+    pad(date.getUTCDate(), 2)
+  ].join("-");
+  const time = [
+    pad(date.getUTCHours(), 2),
+    pad(date.getUTCMinutes(), 2),
+    pad(date.getUTCSeconds(), 2)
+  ].join(":");
+  return `${day} ${time}.${pad(date.getUTCMilliseconds(), 3)}`;
+}
+function jsonOrEmpty(value) {
+  if (value === void 0 || value === null) return "";
+  if (typeof value === "object") {
+    if (Array.isArray(value)) {
+      if (value.length === 0) return "";
+    } else if (Object.keys(value).length === 0) return "";
+  }
+  return JSON.stringify(value);
+}
+function eventToRow(event) {
+  return {
+    name: event.name,
+    data: jsonOrEmpty(event.data),
+    context: jsonOrEmpty(event.context),
+    globals: jsonOrEmpty(event.globals),
+    custom: jsonOrEmpty(event.custom),
+    user: jsonOrEmpty(event.user),
+    nested: jsonOrEmpty(event.nested),
+    consent: jsonOrEmpty(event.consent),
+    id: event.id,
+    trigger: event.trigger,
+    entity: event.entity,
+    action: event.action,
+    timestamp: toDateTime(event.timestamp),
+    timing: event.timing,
+    source: jsonOrEmpty(event.source)
+  };
+}
+
+// src/insert.ts
+var import_node_crypto = require("crypto");
+
+// src/config.ts
+var import_client3 = require("@clickhouse/client");
+
+// src/insert.ts
+function deduplicationToken(ids) {
+  return (0, import_node_crypto.createHash)("sha256").update(JSON.stringify(ids)).digest("hex");
+}
+
+// src/examples/step.ts
+var settings = {
+  url: "https://clickhouse.example.com:8443",
+  database: "analytics",
+  table: "events"
+};
+var init = {
+  title: "Initialization",
+  description: "Init creates a single ClickHouse client for the configured endpoint, database and credentials. The client is held on the resolved config and reused by every insert. Its request timeout is derived from the collector timeout so a retried insert cannot outlive the delivery it belongs to, and request compression is enabled.",
+  in: {
+    settings,
+    credentials: {
+      username: "walkeros_writer",
+      password: "$secret.CLICKHOUSE_PASSWORD"
+    }
+  },
+  out: [
+    [
+      "createClient",
+      {
+        url: "https://clickhouse.example.com:8443",
+        database: "analytics",
+        username: "walkeros_writer",
+        password: "$secret.CLICKHOUSE_PASSWORD",
+        request_timeout: 4812,
+        compression: { request: true }
+      }
+    ]
+  ]
+};
+var initPassthrough = {
+  title: "Client option passthrough",
+  description: "The raw settings.clickhouse options reach the client underneath the destination's own settings: url, database and the credentials slot win over them, an explicit request_timeout or compression wins over the derived default, and any other option travels through untouched.",
+  public: false,
+  in: {
+    settings: {
+      url: "https://clickhouse.example.com:8443",
+      database: "analytics",
+      clickhouse: {
+        url: "https://ignored.example.com:8443",
+        database: "ignored",
+        username: "ignored_user",
+        request_timeout: 6e4,
+        max_open_connections: 25,
+        application: "walkeros"
+      }
+    },
+    credentials: {
+      username: "walkeros_writer",
+      password: "$secret.CLICKHOUSE_PASSWORD"
+    }
+  },
+  out: [
+    [
+      "createClient",
+      {
+        url: "https://clickhouse.example.com:8443",
+        database: "analytics",
+        username: "walkeros_writer",
+        password: "$secret.CLICKHOUSE_PASSWORD",
+        request_timeout: 6e4,
+        max_open_connections: 25,
+        application: "walkeros",
+        compression: { request: true }
+      }
+    ]
+  ]
+};
+function expectedInsert(event) {
+  return [
+    [
+      "insert",
+      {
+        table: settings.table,
+        values: [eventToRow(event)],
+        format: "JSONEachRow",
+        clickhouse_settings: {
+          async_insert: 0,
+          input_format_skip_unknown_fields: 0,
+          input_format_null_as_default: 0,
+          insert_deduplication_token: deduplicationToken([event.id])
+        }
+      }
+    ]
+  ];
+}
+var pageViewEvent = (0, import_core.getEvent)("page view", { timestamp: 1700000000123 });
+var pageView = {
+  title: "Page view",
+  description: "A page view is inserted as a single JSONEachRow row. The payload columns (data, context, globals, custom, user, nested, consent, source) are JSON-encoded strings and the timestamp is a UTC datetime literal, both produced by eventToRow.",
+  in: pageViewEvent,
+  mapping: void 0,
+  out: expectedInsert(pageViewEvent)
+};
+var purchaseEvent = (0, import_core.getEvent)("order complete", {
+  timestamp: 1700000000456,
+  data: {
+    id: "ORD-500",
+    currency: "EUR",
+    total: 199.99,
+    items: [
+      { sku: "SKU-1", quantity: 2 },
+      { sku: "SKU-2", quantity: 1 }
+    ]
+  }
+});
+var purchase = {
+  title: "Purchase",
+  description: "An order event is inserted as one row. The nested items array is JSON-encoded into the data column, so a richer payload needs no table change. Promoting a hot key to its own column is a MATERIALIZED expression in the table DDL, not a destination setting.",
+  in: purchaseEvent,
+  mapping: void 0,
+  out: expectedInsert(purchaseEvent)
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  env,
+  step
+});