@walkeros/server-destination-clickhouse 0.0.0-bootstrap

package/dist/index.mjs

@@ -0,0 +1 @@
+import{createClient as t}from"@clickhouse/client";function e(t){return Number.isFinite(t)?Math.max(0,Math.floor(t)):0}function n(t,e=1.5){return 250*2**t*e}function i(t,i){const s=e(i),o=s+1;return Math.max(1e3,Math.floor((r(t)-function(t){const i=e(t);let r=0;for(let t=0;t<i;t+=1)if(r+=n(t),!Number.isFinite(r))return r;return r}(s))/o))}function r(t){return void 0===t||!Number.isFinite(t)||t<=0?1e4:t}var s={async_insert:0,input_format_skip_unknown_fields:0,input_format_null_as_default:0};function o(t){return{...s,...t}}function a(t){try{return new URL(t).host}catch{return"unparsed"}}function c(t,e){const{url:n,clickhouse:i}=t;return n?{url:n,database:t.database||"default",table:t.table||"events",maxRetries:t.maxRetries??1,...i&&{clickhouse:i},clickhouseSettings:o(t.clickhouseSettings)}:e.throw("Config settings url missing")}function u(e,n,s){const{settings:o}=e;if(!o)return s.throw("Config settings missing");const u=c(o,s),l=function(t,e,n){const r={...t.clickhouse};return void 0===r.request_timeout&&(r.request_timeout=i(n,t.maxRetries)),void 0===r.compression&&(r.compression={request:!0}),r.url=t.url,r.database=t.database,e&&(r.username=e.username,r.password=e.password),r}(u,e.credentials,r(e.timeout)),g=(n?.ClickHouseClient||t)(l);return s.debug("ClickHouse client created",{host:a(u.url),database:u.database,table:u.table}),{...e,settings:{...u,client:g}}}import{isObject as l}from"@walkeros/core";var g=new Set(["159","202","203","209","210","242","394","439","745","999","1017"]),f={252:"ClickHouse has too many parts in the partition and rejected the insert. Send fewer, larger batches: raise config.batch.size so the same rows arrive in fewer inserts. Raising parts_to_throw_insert is not the fix, it only hides the merge backlog until it is worse.",60:"ClickHouse does not know the table. Create it with the table DDL at https://www.walkeros.io/docs/destinations/server/clickhouse#create-the-table before sending events."};function d(t){if("object"!=typeof t||null===t)return;if(!("code"in t))return;const{code:e}=t;return"string"==typeof e&&/^\d+$/.test(e)?e:void 0}function h(t,e){const n=d(t);if(void 0===n||g.has(n))return"retryable";const i=f[n];return i&&e.error(i,{code:n}),"terminal"}function m(t,e){return String(t).padStart(e,"0")}function w(t){const e=new Date(t);return`${[m(e.getUTCFullYear(),4),m(e.getUTCMonth()+1,2),m(e.getUTCDate(),2)].join("-")} ${[m(e.getUTCHours(),2),m(e.getUTCMinutes(),2),m(e.getUTCSeconds(),2)].join(":")}.${m(e.getUTCMilliseconds(),3)}`}function b(t){if(null==t)return"";if("object"==typeof t)if(Array.isArray(t)){if(0===t.length)return""}else if(0===Object.keys(t).length)return"";return JSON.stringify(t)}function p(t){return{name:t.name,data:b(t.data),context:b(t.context),globals:b(t.globals),custom:b(t.custom),user:b(t.user),nested:b(t.nested),consent:b(t.consent),id:t.id,trigger:t.trigger,entity:t.entity,action:t.action,timestamp:w(t.timestamp),timing:t.timing,source:b(t.source)}}import{createHash as k}from"crypto";function y(){return 1+.5*Math.random()}function C(t){return new Promise(e=>{setTimeout(e,t)})}function v({event:t,data:e}){return{row:l(e)?e:p(t),id:t.id}}async function _(t,i,r){const s=function(t,e){const{settings:n}=t;if(!n)return e.throw("Config settings missing, init() not run");const{client:i}=n;return i?{...c(n,e),client:i}:e.throw("ClickHouse client missing, init() not run")}(i,r);try{await async function(t,i,r){const{client:s,table:a}=i,c=e(i.maxRetries),u={...o(i.clickhouseSettings),insert_deduplication_token:(l=t.map(t=>t.id),k("sha256").update(JSON.stringify(l)).digest("hex"))};var l;const g={table:a,values:t.map(t=>t.row),format:"JSONEachRow",clickhouse_settings:u};for(let e=0;;e+=1){r.debug("ClickHouse insert",{table:a,rows:t.length,attempt:e});try{return void await s.insert(g)}catch(t){if("retryable"!==h(t,r)||e>=c)throw t;const i=n(e,y());if(!Number.isFinite(i))throw t;r.warn("ClickHouse insert failed, retrying",{code:d(t),attempt:e,delay:i}),await C(i)}}}(t.map(v),s,r)}catch(e){throw r.error("ClickHouse insert failed",{code:d(e),message:e instanceof Error?e.message:String(e),rows:t.length}),e}r.debug("ClickHouse insert ok",{rows:t.length})}var H={},S={type:"clickhouse",config:{},init:async({config:t,env:e,logger:n})=>u(t,e,n),push:async(t,{config:e,data:n,logger:i})=>{await _([{event:t,data:n}],e,i)},pushBatch:async(t,{config:e,logger:n})=>{0!==t.entries.length&&await _(t.entries,e,n)},async destroy({config:t,logger:e}){const n=t.settings,i=n?.client;n&&i&&(n.client=void 0,await i.close(),e.debug("ClickHouse client closed"))}},x=S;export{H as DestinationClickHouse,x as default,S as destinationClickHouse};//# sourceMappingURL=index.mjs.map

package/dist/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/config.ts","../src/deliver.ts","../src/classify.ts","../src/eventToRow.ts","../src/insert.ts","../src/push.ts","../src/pushBatch.ts","../src/types/index.ts","../src/index.ts"],"sourcesContent":["import type {\n  ClickHouseClientConfigOptions,\n  ClickHouseSettings,\n} from '@clickhouse/client';\nimport type { Logger } from '@walkeros/core';\nimport { createClient } from '@clickhouse/client';\nimport type {\n  ClickHouseClientFactory,\n  Config,\n  Credentials,\n  Env,\n  InitSettings,\n  PartialConfig,\n  Settings,\n} from './types';\n\n/** Database written to when a flow names none. */\nexport const DEFAULT_DATABASE = 'default';\n\n/** Table written to when a flow names none. */\nexport const DEFAULT_TABLE = 'events';\n\n/** Retries added on top of the first attempt when a flow names none. */\nexport const DEFAULT_MAX_RETRIES = 1;\n\n/** The collector's own per-delivery race, used when `config.timeout` is unset. */\nexport const DEFAULT_TIMEOUT_MS = 10_000;\n\n/** Delay before the first retry. Every further retry doubles it. */\nexport const RETRY_BASE_DELAY_MS = 250;\n\n/** Highest multiplier the retry jitter can apply to a single delay. */\nexport const RETRY_JITTER_MAX = 1.5;\n\n/** Floor for the derived per-attempt timeout. */\nexport const MIN_REQUEST_TIMEOUT_MS = 1_000;\n\n/**\n * `maxRetries` as the retry mechanism agrees to read it: a whole, finite count,\n * never negative. Everything else means no retries at all.\n *\n * A flow's config arrives as JSON and reaches the resolution helpers without\n * passing the schema, so `int().min(0)` does not stand between a config file and\n * this code. `JSON.parse('{\"maxRetries\": 1e999}')` yields `Infinity`, which\n * would spin the budget loop below inside `init`, on the collector's startup\n * path. `NaN` is quieter and reaches the client as a `request_timeout` of `NaN`.\n *\n * Every part of the retry mechanism normalizes through this one function, so the\n * budget the timeout was derived from and the loop that spends it cannot\n * disagree about what a valid count is.\n */\nexport function normalizeMaxRetries(maxRetries: number): number {\n  return Number.isFinite(maxRetries) ? Math.max(0, Math.floor(maxRetries)) : 0;\n}\n\n/**\n * How long to wait before the retry at `retry`, counting the first retry as 0:\n * the base delay doubled once per retry, scaled by the jitter.\n *\n * The retry loop and the budget below call this same function, so what the\n * per-attempt timeout was derived from and what the loop actually waits cannot\n * drift apart. A loop passes its own drawn jitter; the budget takes the default,\n * which is the largest jitter that can be drawn.\n */\nexport function retryDelay(\n  retry: number,\n  jitter: number = RETRY_JITTER_MAX,\n): number {\n  return RETRY_BASE_DELAY_MS * 2 ** retry * jitter;\n}\n\n/**\n * Longest total time the retry delays alone can take: every one of the\n * `maxRetries` delays at its longest jittered length. The delays sit between\n * the `1 + maxRetries` attempts, so they spend the same budget the attempts do.\n *\n * The delays double, so they pass what a float can hold after about a thousand\n * of them and the total stops changing. Returning at that point bounds the loop\n * for any count at all, without inventing a maximum number of retries.\n */\nexport function retryDelayBudget(maxRetries: number): number {\n  const retries = normalizeMaxRetries(maxRetries);\n  let budget = 0;\n\n  for (let retry = 0; retry < retries; retry += 1) {\n    budget += retryDelay(retry);\n\n    if (!Number.isFinite(budget)) return budget;\n  }\n\n  return budget;\n}\n\n/**\n * Per-request timeout for the client, derived so that every attempt and every\n * retry delay finishes inside the collector's race.\n *\n * The client's own default is 30 s, three times the collector's 10 s. Left\n * there, the collector stops waiting at 10 s, dead-letters the batch and counts\n * it lost while the client is still sending, and the rows land at 12 s anyway:\n * a batch counted lost and actually written. Splitting what the delays leave\n * over across the attempts keeps the last attempt finishing first.\n *\n * The floor keeps a short timeout or a high retry count from deriving a useless\n * or negative value. Such a configuration cannot fit its own retries either\n * way, and the collector's race is what bounds it there. Once the floor fires,\n * the attempts and their delays add up to more than `timeout`, and the\n * collector cuts the later attempts off when it stops waiting.\n */\nexport function resolveRequestTimeout(\n  timeout: number,\n  maxRetries: number,\n): number {\n  // Both arguments are normalized rather than trusted. A retry count of NaN\n  // would otherwise carry straight through the arithmetic into a\n  // `request_timeout` of NaN, and a non-finite timeout into one of Infinity:\n  // a hung insert the collector can only end by dead-lettering the batch.\n  const retries = normalizeMaxRetries(maxRetries);\n  const attempts = retries + 1;\n\n  return Math.max(\n    MIN_REQUEST_TIMEOUT_MS,\n    Math.floor(\n      (resolveTimeout(timeout) - retryDelayBudget(retries)) / attempts,\n    ),\n  );\n}\n\n/**\n * The collector's per-delivery race, as this destination reads it: a positive,\n * finite number of milliseconds, or the collector's own default.\n */\nexport function resolveTimeout(timeout?: number): number {\n  if (timeout === undefined || !Number.isFinite(timeout) || timeout <= 0)\n    return DEFAULT_TIMEOUT_MS;\n\n  return timeout;\n}\n\n/**\n * ClickHouse settings sent with every insert.\n *\n * `async_insert: 0` keeps the server's acknowledgement tied to a durable write.\n * An async insert is acknowledged before the data reaches the table, and an\n * error raised at the later flush has no way back to the caller, so a lost\n * write would neither retry nor reach the dead letter queue. The server-side\n * default is documented inconsistently, so the value travels explicitly.\n *\n * `input_format_skip_unknown_fields: 0` and `input_format_null_as_default: 0`\n * turn the server's silent repairs off. Both are on by default: the first drops\n * a JSON key the table does not know, the second rewrites an explicit null into\n * the column default. A walkerOS field missing from a partner's table would\n * disappear with no error and no warning, found weeks later in reporting. At 0\n * the whole batch fails loudly, is attributed and reaches the counters.\n *\n * A flow's own `clickhouseSettings` sit on top of all three.\n */\nconst INSERT_SETTINGS: ClickHouseSettings = {\n  async_insert: 0,\n  input_format_skip_unknown_fields: 0,\n  input_format_null_as_default: 0,\n};\n\n/** The per-insert settings, with a flow's own `clickhouseSettings` on top. */\nexport function resolveInsertSettings(\n  custom?: ClickHouseSettings,\n): ClickHouseSettings {\n  return { ...INSERT_SETTINGS, ...custom };\n}\n\n/**\n * The host of an endpoint, with any userinfo left behind.\n *\n * A url is one of the places a password can legitimately sit\n * (`https://user:pass@host:port`), so the configured string is never safe to\n * put in a log line and only the host ever is. A url the client would reject\n * still reaches here when the factory comes from `env`, which parses nothing.\n */\nfunction endpointHost(url: string): string {\n  try {\n    return new URL(url).host;\n  } catch {\n    return 'unparsed';\n  }\n}\n\n/** The resolved settings before init attaches the live client. */\ntype ResolvedSettings = Omit<Settings, 'client'>;\n\nfunction withDefaults(\n  settings: InitSettings,\n  logger: Logger.Instance,\n): ResolvedSettings {\n  const { url, clickhouse } = settings;\n\n  if (!url) return logger.throw('Config settings url missing');\n\n  return {\n    url,\n    database: settings.database || DEFAULT_DATABASE,\n    table: settings.table || DEFAULT_TABLE,\n    maxRetries: settings.maxRetries ?? DEFAULT_MAX_RETRIES,\n    ...(clickhouse && { clickhouse }),\n    clickhouseSettings: resolveInsertSettings(settings.clickhouseSettings),\n  };\n}\n\n/**\n * The options `createClient` receives, merged lowest layer first:\n *\n * 1. `settings.clickhouse`, the raw SDK passthrough.\n * 2. The derived defaults, `request_timeout` and request compression, each\n *    applied only where the passthrough left its key unset.\n * 3. `url` and `database`, the destination's own first-class settings. They\n *    always win: an insert names only the table, so a passthrough `database`\n *    that disagreed would move every row to a different database in silence.\n * 4. `credentials`, the strictly typed slot backed by a managed secret. It wins\n *    over a username or password left in the passthrough; with the slot empty\n *    the passthrough reaches the client untouched.\n *\n * The table, the retry count and the per-insert settings are not client\n * options and stay off this object.\n */\nexport function resolveClientOptions(\n  settings: ResolvedSettings,\n  credentials: Credentials | undefined,\n  timeout: number,\n): ClickHouseClientConfigOptions {\n  const options: ClickHouseClientConfigOptions = { ...settings.clickhouse };\n\n  if (options.request_timeout === undefined)\n    options.request_timeout = resolveRequestTimeout(\n      timeout,\n      settings.maxRetries,\n    );\n\n  // Both directions are off by default in this client. Compressing the request\n  // is the one that pays here: insert bodies are large, repetitive JSON.\n  if (options.compression === undefined)\n    options.compression = { request: true };\n\n  options.url = settings.url;\n  options.database = settings.database;\n\n  if (credentials) {\n    options.username = credentials.username;\n    options.password = credentials.password;\n  }\n\n  return options;\n}\n\n/**\n * Resolves the config init hands forward, and creates the one client every\n * insert reuses. Nothing is queried: a missing table surfaces at the first\n * insert, as a loud, attributable failure.\n */\nexport function getConfig(\n  config: PartialConfig,\n  env: Env | undefined,\n  logger: Logger.Instance,\n): Config {\n  const { settings } = config;\n\n  if (!settings) return logger.throw('Config settings missing');\n\n  const resolved = withDefaults(settings, logger);\n  const options = resolveClientOptions(\n    resolved,\n    config.credentials,\n    resolveTimeout(config.timeout),\n  );\n\n  const factory: ClickHouseClientFactory =\n    env?.ClickHouseClient || createClient;\n  const client = factory(options);\n\n  // Every credential stays out of this line deliberately, the url included:\n  // init is the one place that holds them, and a debug log is the easiest way\n  // to spill them.\n  logger.debug('ClickHouse client created', {\n    host: endpointHost(resolved.url),\n    database: resolved.database,\n    table: resolved.table,\n  });\n\n  return { ...config, settings: { ...resolved, client } };\n}\n\n/**\n * The settings the insert path works with.\n *\n * Core resolves `Config.settings` from the init slot, so a push receives the\n * settings as a user wrote them: the client optional, the defaults unapplied.\n * This narrows them once. It guards that init ran and re-applies the defaults,\n * so the insert path holds a live client and a complete configuration without\n * guarding field by field.\n */\nexport function resolveSettings(\n  config: PartialConfig,\n  logger: Logger.Instance,\n): Settings {\n  const { settings } = config;\n\n  if (!settings) return logger.throw('Config settings missing, init() not run');\n\n  const { client } = settings;\n\n  if (!client) return logger.throw('ClickHouse client missing, init() not run');\n\n  return { ...withDefaults(settings, logger), client };\n}\n","import type { Destination, Logger } from '@walkeros/core';\nimport { isObject } from '@walkeros/core';\nimport { errorCode } from './classify';\nimport { resolveSettings } from './config';\nimport { eventToRow } from './eventToRow';\nimport type { InsertEntry } from './insert';\nimport { insertRows } from './insert';\nimport type { Mapping, PartialConfig } from './types';\n\n/**\n * One event on its way to the table, with whatever a mapping produced for it.\n * The collector's own batch entry, so a flushed batch travels here unchanged\n * and a single push builds the same shape for its one event.\n */\ntype Entry = Destination.BatchEntry<Mapping>;\n\n/**\n * The row one entry contributes, and the event id that identifies it.\n *\n * A flow that mapped the event hands the row it wants in `data`, and that\n * object travels verbatim: it replaces the canonical shape wholesale, so a\n * partner whose table is not the reference DDL configures its columns in the\n * flow. Anything that is not an object means no mapping produced a row, and\n * the event is converted.\n *\n * The id comes from the event either way. A mapped row need not carry one, and\n * the deduplication token cannot be read from a value that may be absent.\n */\nfunction toInsertEntry({ event, data }: Entry): InsertEntry {\n  return {\n    row: isObject(data) ? data : eventToRow(event),\n    id: event.id,\n  };\n}\n\n/**\n * Sends one set of entries and says what happened to them.\n *\n * Both entry points arrive here, so a single event and a flushed batch take the\n * same path: one insert, one retry budget, one line per outcome, and one place\n * that decides what a row looks like. The settings are resolved here rather\n * than in each caller because core carries them in the slot a user writes, the\n * client optional and the defaults unapplied, so they have to be narrowed once\n * per delivery.\n *\n * A failure is logged and rethrown as it arrived. The collector reads a throw\n * as the whole delivery failing: every event goes to the dead letter queue and\n * one transport failure is recorded against the destination's breaker.\n * Swallowing the rejection would report lost events as delivered.\n */\nexport async function deliver(\n  entries: readonly Entry[],\n  config: PartialConfig,\n  logger: Logger.Instance,\n): Promise<void> {\n  const settings = resolveSettings(config, logger);\n\n  try {\n    await insertRows(entries.map(toInsertEntry), settings, logger);\n  } catch (error) {\n    // The message travels beside the code because the code is often absent: a\n    // transport failure carries a Node errno, which is not a ClickHouse code\n    // and is deliberately not read as one. `ECONNRESET` is the commonest\n    // retryable failure there is, and a line that named neither it nor the\n    // server's own text would leave an operator grepping for nothing.\n    logger.error('ClickHouse insert failed', {\n      code: errorCode(error),\n      message: error instanceof Error ? error.message : String(error),\n      rows: entries.length,\n    });\n\n    throw error;\n  }\n\n  // A landed delivery stays at debug. Every event that arrives produces one of\n  // these, so anything louder buries the line that matters.\n  logger.debug('ClickHouse insert ok', { rows: entries.length });\n}\n","import type { Logger } from '@walkeros/core';\n\n/** Whether another attempt at the same insert can plausibly succeed. */\nexport type Classification = 'retryable' | 'terminal';\n\n/**\n * The codes worth another attempt: server-side timeouts, a saturated query or\n * connection pool, a transport failure, a table held read-only while keeper\n * works. Each of them describes a condition that ends on its own in seconds.\n *\n * TIMEOUT_EXCEEDED and QUERY_WAS_CANCELLED are the two that may have landed\n * anyway, the server having been writing when the answer stopped coming.\n * Retrying them is only safe because every attempt of a batch carries the same\n * deduplication token.\n */\nconst RETRYABLE: ReadonlySet<string> = new Set([\n  '159', // TIMEOUT_EXCEEDED\n  '202', // TOO_MANY_SIMULTANEOUS_QUERIES\n  '203', // NO_FREE_CONNECTION\n  '209', // SOCKET_TIMEOUT\n  '210', // NETWORK_ERROR\n  '242', // TABLE_IS_READ_ONLY\n  '394', // QUERY_WAS_CANCELLED\n  '439', // CANNOT_SCHEDULE_TASK\n  '745', // SERVER_OVERLOADED\n  '999', // KEEPER_EXCEPTION\n  '1017', // ASYNC_INSERT_FLUSH_TIMEOUT\n]);\n\n/**\n * What an operator has to change, for the two terminal failures where the fix\n * is neither obvious nor in the data.\n *\n * TOO_MANY_PARTS is the classic failure of naive event ingestion: too many\n * small inserts leave more active parts in a partition than merging can keep\n * up with. Retrying inside a ten second window cannot help a merge backlog and\n * adds parts to it, and the setting that names the limit is not the lever.\n *\n * Everything else terminal explains itself in the server's own message, and the\n * collector already logs every failed push, so a second line would only repeat\n * it.\n */\nconst ADVICE: Readonly<Record<string, string>> = {\n  '252':\n    'ClickHouse has too many parts in the partition and rejected the insert. ' +\n    'Send fewer, larger batches: raise config.batch.size so the same rows ' +\n    'arrive in fewer inserts. Raising parts_to_throw_insert is not the fix, ' +\n    'it only hides the merge backlog until it is worse.',\n  '60':\n    'ClickHouse does not know the table. Create it with the table DDL at ' +\n    'https://www.walkeros.io/docs/destinations/server/clickhouse#create-the-table ' +\n    'before sending events.',\n};\n\n/**\n * The ClickHouse error code carried by a rejection of unknown shape.\n *\n * Read structurally rather than through `instanceof ClickHouseError`: a bundled\n * or duplicated copy of the client is a different class object and would fail\n * that check while carrying a perfectly good code.\n *\n * Gated on what a ClickHouse code is rather than on what a `code` property\n * might hold. The client parses the code out of the server's message with a\n * decimal-digit group and returns anything that does not match unchanged, so a\n * ClickHouse code is always a run of digits in a string. A socket-level\n * rejection reaches here with its Node errno still on it, `ECONNRESET` for the\n * keep-alive a restarting server drops, `ENOTFOUND`, `ETIMEDOUT`. Those are not\n * ClickHouse codes, and reading them as one would send the single most common\n * transient failure straight to the dead letter queue.\n */\nexport function errorCode(error: unknown): string | undefined {\n  if (typeof error !== 'object' || error === null) return undefined;\n  if (!('code' in error)) return undefined;\n\n  const { code } = error;\n\n  return typeof code === 'string' && /^\\d+$/.test(code) ? code : undefined;\n}\n\n/**\n * Whether a failed insert is worth another attempt, and the one place an\n * operator is told what to change when it is not.\n *\n * The code is the classification input, never the HTTP status. ClickHouse maps\n * a few dozen codes to a status of their own and sends everything else as a\n * plain 500, so a saturated queue and an unknown column arrive\n * indistinguishable; it emits no 429 and no Retry-After at all, those come from\n * a proxy in front of it.\n *\n * A rejection carrying no ClickHouse code never got an answer from the server.\n * A keep-alive reset by a restarting node, a DNS failure, a connection refused,\n * a request that timed out on the way out: each arrives with a Node errno and\n * no code the server assigned. Nothing was rejected, so the batch retries, and\n * this is the failure class the retry exists for.\n *\n * An unreadable code errs the same way on purpose: one wasted attempt costs a\n * second, while a batch wrongly called terminal is lost for good, the collector\n * having no retry of its own and never draining its dead letter queue.\n */\nexport function classify(\n  error: unknown,\n  logger: Logger.Instance,\n): Classification {\n  const code = errorCode(error);\n\n  if (code === undefined || RETRYABLE.has(code)) return 'retryable';\n\n  const advice = ADVICE[code];\n\n  if (advice) logger.error(advice, { code });\n\n  return 'terminal';\n}\n","import type { WalkerOS } from '@walkeros/core';\n\n/**\n * One insert row, in the canonical walkerOS Event v4 column order: name, data,\n * context, globals, custom, user, nested, consent, id, trigger, entity,\n * action, timestamp, timing, source.\n *\n * Payload columns are JSON-encoded strings rather than nested structures, so\n * the table needs no schema knowledge of what a flow sends. Hot keys become\n * real columns in the table's own DDL through `MATERIALIZED` expressions over\n * these strings, which costs the destination nothing.\n *\n * No column is nullable. Inserts run with `input_format_null_as_default = 0`,\n * which turns a stray `null` into a loud server-side error instead of a silent\n * default, so an absent payload has to arrive as an empty string. A non-finite\n * `timing` is the one value the type still admits that reaches the wire as a\n * null, `JSON.stringify` writing `NaN` and `Infinity` that way: the server\n * rejects the batch, the rejection is terminal and is not retried, and the\n * wrong number stays visible in the event rather than becoming a plausible `0`\n * in the table.\n */\nexport interface ClickHouseRow {\n  [column: string]: string | number;\n  name: string;\n  data: string;\n  context: string;\n  globals: string;\n  custom: string;\n  user: string;\n  nested: string;\n  consent: string;\n  id: string;\n  trigger: string;\n  entity: string;\n  action: string;\n  timestamp: string;\n  timing: number;\n  source: string;\n}\n\n/**\n * A row as the insert path receives it: the canonical shape above, or the\n * object a flow's mapping placed in `context.data`.\n *\n * A mapped row replaces the canonical one wholesale and travels verbatim, so a\n * partner whose table is not the reference DDL configures its columns in the\n * flow. Its shape is whatever the flow wrote, and this type claims no more\n * than that.\n */\nexport type InsertRow = ClickHouseRow | WalkerOS.AnyObject;\n\nfunction pad(value: number, length: number): string {\n  return String(value).padStart(length, '0');\n}\n\n/**\n * `YYYY-MM-DD hh:mm:ss.SSS` in UTC, the one literal every ClickHouse server\n * reads the same way. The `basic` date parser rejects the ISO 8601 `T` and `Z`\n * markers, and a bare number is read as Unix seconds by newer servers but as\n * the raw value at the column's precision by older ones, so epoch\n * milliseconds would land off by a factor of 1000 depending on the version.\n * The reference DDL declares the column `DateTime64(3, 'UTC')`, and a string\n * is read in the column's own timezone.\n */\nfunction toDateTime(timestamp: number): string {\n  const date = new Date(timestamp);\n\n  const day = [\n    pad(date.getUTCFullYear(), 4),\n    pad(date.getUTCMonth() + 1, 2),\n    pad(date.getUTCDate(), 2),\n  ].join('-');\n\n  const time = [\n    pad(date.getUTCHours(), 2),\n    pad(date.getUTCMinutes(), 2),\n    pad(date.getUTCSeconds(), 2),\n  ].join(':');\n\n  return `${day} ${time}.${pad(date.getUTCMilliseconds(), 3)}`;\n}\n\n/**\n * JSON for anything carrying content, `''` for anything not. Absent values,\n * empty objects and empty arrays collapse to the same empty string so no\n * column ever holds a null.\n */\nfunction jsonOrEmpty(value: unknown): string {\n  if (value === undefined || value === null) return '';\n\n  if (typeof value === 'object') {\n    if (Array.isArray(value)) {\n      if (value.length === 0) return '';\n    } else if (Object.keys(value).length === 0) return '';\n  }\n\n  return JSON.stringify(value);\n}\n\n/** Convert one walkerOS event into one ClickHouse row. */\nexport function eventToRow(event: WalkerOS.Event): ClickHouseRow {\n  return {\n    name: event.name,\n    data: jsonOrEmpty(event.data),\n    context: jsonOrEmpty(event.context),\n    globals: jsonOrEmpty(event.globals),\n    custom: jsonOrEmpty(event.custom),\n    user: jsonOrEmpty(event.user),\n    nested: jsonOrEmpty(event.nested),\n    consent: jsonOrEmpty(event.consent),\n    id: event.id,\n    trigger: event.trigger,\n    entity: event.entity,\n    action: event.action,\n    timestamp: toDateTime(event.timestamp),\n    timing: event.timing,\n    source: jsonOrEmpty(event.source),\n  };\n}\n","import type { Readable } from 'stream';\nimport type { ClickHouseSettings, InsertParams } from '@clickhouse/client';\nimport type { Logger } from '@walkeros/core';\nimport { createHash } from 'node:crypto';\nimport { classify, errorCode } from './classify';\nimport {\n  RETRY_JITTER_MAX,\n  normalizeMaxRetries,\n  resolveInsertSettings,\n  retryDelay,\n} from './config';\nimport type { InsertRow } from './eventToRow';\nimport type { Settings } from './types';\n\n/**\n * One row on its way to the table, beside the id of the event it came from.\n *\n * The id travels beside the row rather than inside it because a row is not\n * guaranteed to carry one. A mapped row is whatever a flow wrote, and `id` is\n * a column a partner's table need not have. Reading the token out of the row\n * would give every id-less mapped row the same token, which\n * `deduplicationToken` explains is silent, total data loss.\n */\nexport interface InsertEntry {\n  /** The row as it travels: a mapped object verbatim, or the conversion. */\n  row: InsertRow;\n  /** The walkerOS event id. The only input the token is derived from. */\n  id: string;\n}\n\n/**\n * The deduplication token sent with a batch: a digest over its event ids, in\n * the order they are inserted.\n *\n * Retrying an insert without one double-counts every row of a batch the server\n * had already written when the answer stopped coming. Synchronous inserts\n * deduplicate on the block contents by themselves, but only where the window\n * is on: `non_replicated_deduplication_window` defaults to 0, so a plain\n * MergeTree does nothing unless the table says otherwise. An explicit token\n * identifies the block for both MergeTree families, but it prevents duplicates\n * only where insert deduplication is enabled: by default on a\n * ReplicatedMergeTree, and on a plain MergeTree only with a positive window.\n *\n * A digest rather than a random uuid, so a redelivery of the same events\n * produces the same token and deduplicates too, as long as it lands inside the\n * table's deduplication window, which counts blocks and not time. The ids\n * travel as a JSON array so that no id can imitate the separator between two\n * of them, and the order is part of the digest because the same rows in\n * another order are another block to ClickHouse.\n *\n * Event ids and nothing else. The server deduplicates on this token INSTEAD of\n * the block contents, and skips a repeated block while reporting success, so\n * two deliveries that share a token are one delivery with no error, no dead\n * letter entry and no counter anywhere: a token that could repeat across\n * different events is silent data loss. The ids are the one input that is\n * always present and unique per event, which the row is not, and they identify\n * the events rather than describing them, so two different events that a\n * mapping happened to shape identically stay two blocks.\n */\nexport function deduplicationToken(ids: readonly string[]): string {\n  return createHash('sha256').update(JSON.stringify(ids)).digest('hex');\n}\n\n/**\n * How much of the base delay this wait takes: all of it, plus up to half again.\n * Two runtimes that failed on the same server-side condition would otherwise\n * retry in lockstep and meet it together a second time.\n */\nfunction drawJitter(): number {\n  return 1 + Math.random() * (RETRY_JITTER_MAX - 1);\n}\n\nfunction wait(ms: number): Promise<void> {\n  return new Promise((resolve) => {\n    setTimeout(resolve, ms);\n  });\n}\n\n/**\n * Inserts one batch, retrying a transient failure up to `maxRetries` times.\n *\n * One flushed batch is one insert, whatever its size: the rows travel as a\n * single JSONEachRow body, and `format` is fixed because a table written in two\n * formats is a table nobody can read back.\n *\n * The parameters are built once and every attempt sends that same object, so\n * the deduplication token cannot be regenerated halfway through a retry, which\n * is the one thing that would turn a safe retry into duplicated rows.\n *\n * The delays come from the same function the per-attempt request timeout was\n * derived from, so what the loop waits and what the client was configured for\n * cannot drift apart and leave an attempt running past the collector's race.\n *\n * A failure that survives the retries is rethrown as it arrived, ClickHouse\n * code included: the collector dead-letters the whole batch on a throw and\n * keeps the error beside it, and the code is what makes that entry readable.\n */\nexport async function insertRows(\n  entries: readonly InsertEntry[],\n  settings: Settings,\n  logger: Logger.Instance,\n): Promise<void> {\n  const { client, table } = settings;\n\n  // What counts as a usable retry count is decided in one place for both halves\n  // of the mechanism, the loop here and the budget the request timeout was\n  // derived from. A count that is not a finite number would otherwise never end\n  // this loop, `attempt >= NaN` and `attempt >= Infinity` being false forever,\n  // and nothing cancels a destination's work once the collector has stopped\n  // waiting for it.\n  const retries = normalizeMaxRetries(settings.maxRetries);\n\n  // Our defaults first, then what the flow set, then the token: a token an\n  // operator could pin would collapse every insert into one deduplicated block\n  // and drop every batch after the first without an error anywhere.\n  const clickhouse_settings: ClickHouseSettings = {\n    ...resolveInsertSettings(settings.clickhouseSettings),\n    insert_deduplication_token: deduplicationToken(\n      entries.map((entry) => entry.id),\n    ),\n  };\n\n  const params: InsertParams<Readable, InsertRow> = {\n    table,\n    values: entries.map((entry) => entry.row),\n    format: 'JSONEachRow',\n    clickhouse_settings,\n  };\n\n  for (let attempt = 0; ; attempt += 1) {\n    logger.debug('ClickHouse insert', { table, rows: entries.length, attempt });\n\n    try {\n      await client.insert(params);\n\n      return;\n    } catch (error) {\n      // Classified before the budget is read, so the operator hears about a\n      // terminal failure even where there was never a retry to skip.\n      const retryable = classify(error, logger) === 'retryable';\n\n      if (!retryable || attempt >= retries) throw error;\n\n      const delay = retryDelay(attempt, drawJitter());\n\n      // A finite but absurd retry count passes the clamp above and would spend\n      // it on insert attempts against a live server. The delays double, so they\n      // leave the finite range on their own after about a thousand retries, and\n      // reading that as the end of the loop bounds it without inventing a\n      // maximum attempt count or restating the schedule. The budget the request\n      // timeout comes from ends on the matching condition: it stops when its\n      // running sum stops being finite, this loop when a single delay does.\n      if (!Number.isFinite(delay)) throw error;\n\n      logger.warn('ClickHouse insert failed, retrying', {\n        code: errorCode(error),\n        attempt,\n        delay,\n      });\n\n      await wait(delay);\n    }\n  }\n}\n","import { deliver } from './deliver';\nimport type { PushFn } from './types';\n\n/**\n * Sends one event as a one-row insert.\n *\n * What the row looks like is decided in `deliver`, beside the same decision for\n * a flushed batch, so a mapped event takes the same path whether batching is on\n * or off.\n */\nexport const push: PushFn = async (event, { config, data, logger }) => {\n  await deliver([{ event, data }], config, logger);\n};\n","import { deliver } from './deliver';\nimport type { PushBatchFn } from './types';\n\n/**\n * Sends one flushed batch as a single insert, its rows in entry order.\n *\n * There are two answers and no third one. Resolving tells the collector the\n * whole batch landed; throwing tells it the whole batch did not, and it dead\n * letters every entry and records one transport failure.\n *\n * The per-entry outcome the contract also allows has no meaning here. A\n * ClickHouse insert succeeds or fails as a whole and names no row, so such an\n * outcome would have to claim every index, and the collector counts an outcome\n * where nothing succeeded as neither a success nor a failure: a total outage\n * would pass the breaker unseen. This destination never returns one.\n */\nexport const pushBatch: PushBatchFn = async (batch, { config, logger }) => {\n  // Nothing to send and nothing to report. An insert of no rows would spend a\n  // round trip, and a deduplication token, on an empty block.\n  if (batch.entries.length === 0) return;\n\n  await deliver(batch.entries, config, logger);\n};\n","import type { Destination as CoreDestination } from '@walkeros/core';\nimport type { DestinationServer } from '@walkeros/server-core';\nimport type {\n  ClickHouseClient,\n  ClickHouseClientConfigOptions,\n  ClickHouseSettings,\n} from '@clickhouse/client';\n\n/**\n * The only two client methods this package is allowed to reach for. Narrowing\n * the seam to them makes the INSERT-only rule a compile-time property rather\n * than a convention: `query`, `command`, `exec` and `ping` are not reachable\n * from a client typed this way, so no code path can read `system.*` or issue\n * DDL. The SDK's own `createClient` stays assignable by structural typing.\n */\nexport type ClickHouseClientSurface = Pick<\n  ClickHouseClient,\n  'insert' | 'close'\n>;\n\n/** Creates the client this destination writes through. Matches `createClient`. */\nexport type ClickHouseClientFactory = (\n  config: ClickHouseClientConfigOptions,\n) => ClickHouseClientSurface;\n\n/**\n * Resolved settings, as they exist after init() applied the defaults and\n * created the client. Every field is present.\n */\nexport interface Settings {\n  /** ClickHouse HTTP endpoint, like https://host.clickhouse.cloud:8443. */\n  url: string;\n  /** Database holding the events table. */\n  database: string;\n  /** Table every event is inserted into. */\n  table: string;\n  /** Retries added on top of the first attempt: total attempts = 1 + maxRetries. */\n  maxRetries: number;\n  /** Raw @clickhouse/client options, merged into the created client. */\n  clickhouse?: ClickHouseClientConfigOptions;\n  /** ClickHouse settings applied to every insert. */\n  clickhouseSettings?: ClickHouseSettings;\n  /**\n   * The live client created by init(). Carried on the resolved config so the\n   * insert path receives it as part of its settings argument. Runtime-only,\n   * not user-facing.\n   */\n  client: ClickHouseClientSurface;\n}\n\n/**\n * Settings as a user writes them in a flow. Only `url` is required; init()\n * fills the rest from the documented defaults.\n */\nexport interface InitSettings {\n  /** ClickHouse HTTP endpoint, like https://host.clickhouse.cloud:8443. */\n  url: string;\n  /** Database holding the events table. Default `default`. */\n  database?: string;\n  /** Table every event is inserted into. Default `events`. */\n  table?: string;\n  /**\n   * Retries added on top of the first attempt: total attempts = 1 + maxRetries.\n   * Default 1, so two attempts.\n   */\n  maxRetries?: number;\n  /** Raw @clickhouse/client options, merged into the created client. */\n  clickhouse?: ClickHouseClientConfigOptions;\n  /** ClickHouse settings applied to every insert. */\n  clickhouseSettings?: ClickHouseSettings;\n  /**\n   * The live client, populated by init(). Runtime-only, not user-facing.\n   *\n   * Optional because this is the type a user writes. Core resolves\n   * `Config.settings` from this slot, so the push path receives it optional\n   * too and must guard before use (see `__tests__/schemas.test.ts`).\n   */\n  client?: ClickHouseClientSurface;\n}\n\n/**\n * ClickHouse receives raw events, so there is no event-level mapping\n * configuration to declare.\n */\nexport interface Mapping {}\n\n/** Username and password for the ClickHouse user the destination writes as. */\nexport interface Credentials {\n  username: string;\n  password: string;\n}\n\n/**\n * Env carries the client factory so tests and simulations inject a recording\n * stand-in instead of opening a socket. Production leaves it undefined and the\n * destination uses the SDK's own `createClient`.\n */\nexport interface Env extends DestinationServer.Env {\n  ClickHouseClient?: ClickHouseClientFactory;\n}\n\n/**\n * The setup slot stays `unknown`: this destination issues INSERT and nothing\n * else, so there is no provisioning lifecycle to configure.\n */\nexport type Types = CoreDestination.Types<\n  Settings,\n  Mapping,\n  Env,\n  InitSettings,\n  unknown,\n  Credentials\n>;\n\n/**\n * `pushBatch` is required, not optional: the collector engages batching only\n * when the destination exports it, and bulk inserts are the whole point of\n * this package. A build that drops it would silently insert row by row.\n */\nexport interface Destination extends DestinationServer.Destination<Types> {\n  init: InitFn;\n  pushBatch: PushBatchFn;\n}\n\nexport type Config = {\n  settings: Settings;\n} & DestinationServer.Config<Types>;\n\nexport type InitFn = DestinationServer.InitFn<Types>;\nexport type PushFn = DestinationServer.PushFn<Types>;\nexport type PushBatchFn = CoreDestination.PushBatchFn<Types>;\n\nexport type PartialConfig = DestinationServer.PartialConfig<Types>;\n\nexport type PushEvents = DestinationServer.PushEvents<Mapping>;\n","import type { Destination } from './types';\nimport { getConfig } from './config';\nimport { push } from './push';\nimport { pushBatch } from './pushBatch';\n\n// Types\nexport * as DestinationClickHouse from './types';\n\n/**\n * The ClickHouse destination.\n *\n * `pushBatch` is exported beside `push` because the collector engages batching\n * only for a destination that carries it, and bulk inserts are the whole point\n * of this package.\n */\nexport const destinationClickHouse: Destination = {\n  type: 'clickhouse',\n\n  config: {},\n\n  async init({ config, env, logger }) {\n    return getConfig(config, env, logger);\n  },\n\n  push,\n\n  pushBatch,\n\n  async destroy({ config, logger }) {\n    const settings = config.settings;\n    const client = settings?.client;\n\n    if (!settings || !client) return;\n\n    // Release the handle before closing. The SDK documents close as a call to\n    // make once per lifecycle and promises nothing about a second one, so a\n    // repeated destroy has to end here rather than in the connection pool.\n    settings.client = undefined;\n\n    // The collector flushes every batch before it calls destroy, so normally\n    // there is nothing left to send and closing the connection pool is the\n    // whole job. A flush that outran the collector's shutdown race is the\n    // exception: its insert is still in flight, and closing destroys the\n    // sockets under it.\n    await client.close();\n\n    logger.debug('ClickHouse client closed');\n  },\n};\n\nexport default destinationClickHouse;\n"],"mappings":";AAKA,SAAS,oBAAoB;AAYtB,IAAM,mBAAmB;AAGzB,IAAM,gBAAgB;AAGtB,IAAM,sBAAsB;AAG5B,IAAM,qBAAqB;AAG3B,IAAM,sBAAsB;AAG5B,IAAM,mBAAmB;AAGzB,IAAM,yBAAyB;AAgB/B,SAAS,oBAAoB,YAA4B;AAC9D,SAAO,OAAO,SAAS,UAAU,IAAI,KAAK,IAAI,GAAG,KAAK,MAAM,UAAU,CAAC,IAAI;AAC7E;AAWO,SAAS,WACd,OACA,SAAiB,kBACT;AACR,SAAO,sBAAsB,KAAK,QAAQ;AAC5C;AAWO,SAAS,iBAAiB,YAA4B;AAC3D,QAAM,UAAU,oBAAoB,UAAU;AAC9C,MAAI,SAAS;AAEb,WAAS,QAAQ,GAAG,QAAQ,SAAS,SAAS,GAAG;AAC/C,cAAU,WAAW,KAAK;AAE1B,QAAI,CAAC,OAAO,SAAS,MAAM,EAAG,QAAO;AAAA,EACvC;AAEA,SAAO;AACT;AAkBO,SAAS,sBACd,SACA,YACQ;AAKR,QAAM,UAAU,oBAAoB,UAAU;AAC9C,QAAM,WAAW,UAAU;AAE3B,SAAO,KAAK;AAAA,IACV;AAAA,IACA,KAAK;AAAA,OACF,eAAe,OAAO,IAAI,iBAAiB,OAAO,KAAK;AAAA,IAC1D;AAAA,EACF;AACF;AAMO,SAAS,eAAe,SAA0B;AACvD,MAAI,YAAY,UAAa,CAAC,OAAO,SAAS,OAAO,KAAK,WAAW;AACnE,WAAO;AAET,SAAO;AACT;AAoBA,IAAM,kBAAsC;AAAA,EAC1C,cAAc;AAAA,EACd,kCAAkC;AAAA,EAClC,8BAA8B;AAChC;AAGO,SAAS,sBACd,QACoB;AACpB,SAAO,EAAE,GAAG,iBAAiB,GAAG,OAAO;AACzC;AAUA,SAAS,aAAa,KAAqB;AACzC,MAAI;AACF,WAAO,IAAI,IAAI,GAAG,EAAE;AAAA,EACtB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAKA,SAAS,aACP,UACA,QACkB;AAClB,QAAM,EAAE,KAAK,WAAW,IAAI;AAE5B,MAAI,CAAC,IAAK,QAAO,OAAO,MAAM,6BAA6B;AAE3D,SAAO;AAAA,IACL;AAAA,IACA,UAAU,SAAS,YAAY;AAAA,IAC/B,OAAO,SAAS,SAAS;AAAA,IACzB,YAAY,SAAS,cAAc;AAAA,IACnC,GAAI,cAAc,EAAE,WAAW;AAAA,IAC/B,oBAAoB,sBAAsB,SAAS,kBAAkB;AAAA,EACvE;AACF;AAkBO,SAAS,qBACd,UACA,aACA,SAC+B;AAC/B,QAAM,UAAyC,EAAE,GAAG,SAAS,WAAW;AAExE,MAAI,QAAQ,oBAAoB;AAC9B,YAAQ,kBAAkB;AAAA,MACxB;AAAA,MACA,SAAS;AAAA,IACX;AAIF,MAAI,QAAQ,gBAAgB;AAC1B,YAAQ,cAAc,EAAE,SAAS,KAAK;AAExC,UAAQ,MAAM,SAAS;AACvB,UAAQ,WAAW,SAAS;AAE5B,MAAI,aAAa;AACf,YAAQ,WAAW,YAAY;AAC/B,YAAQ,WAAW,YAAY;AAAA,EACjC;AAEA,SAAO;AACT;AAOO,SAAS,UACd,QACA,KACA,QACQ;AACR,QAAM,EAAE,SAAS,IAAI;AAErB,MAAI,CAAC,SAAU,QAAO,OAAO,MAAM,yBAAyB;AAE5D,QAAM,WAAW,aAAa,UAAU,MAAM;AAC9C,QAAM,UAAU;AAAA,IACd;AAAA,IACA,OAAO;AAAA,IACP,eAAe,OAAO,OAAO;AAAA,EAC/B;AAEA,QAAM,UACJ,KAAK,oBAAoB;AAC3B,QAAM,SAAS,QAAQ,OAAO;AAK9B,SAAO,MAAM,6BAA6B;AAAA,IACxC,MAAM,aAAa,SAAS,GAAG;AAAA,IAC/B,UAAU,SAAS;AAAA,IACnB,OAAO,SAAS;AAAA,EAClB,CAAC;AAED,SAAO,EAAE,GAAG,QAAQ,UAAU,EAAE,GAAG,UAAU,OAAO,EAAE;AACxD;AAWO,SAAS,gBACd,QACA,QACU;AACV,QAAM,EAAE,SAAS,IAAI;AAErB,MAAI,CAAC,SAAU,QAAO,OAAO,MAAM,yCAAyC;AAE5E,QAAM,EAAE,OAAO,IAAI;AAEnB,MAAI,CAAC,OAAQ,QAAO,OAAO,MAAM,2CAA2C;AAE5E,SAAO,EAAE,GAAG,aAAa,UAAU,MAAM,GAAG,OAAO;AACrD;;;ACtTA,SAAS,gBAAgB;;;ACczB,IAAM,YAAiC,oBAAI,IAAI;AAAA,EAC7C;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AAAA,EACA;AAAA;AACF,CAAC;AAeD,IAAM,SAA2C;AAAA,EAC/C,OACE;AAAA,EAIF,MACE;AAGJ;AAkBO,SAAS,UAAU,OAAoC;AAC5D,MAAI,OAAO,UAAU,YAAY,UAAU,KAAM,QAAO;AACxD,MAAI,EAAE,UAAU,OAAQ,QAAO;AAE/B,QAAM,EAAE,KAAK,IAAI;AAEjB,SAAO,OAAO,SAAS,YAAY,QAAQ,KAAK,IAAI,IAAI,OAAO;AACjE;AAsBO,SAAS,SACd,OACA,QACgB;AAChB,QAAM,OAAO,UAAU,KAAK;AAE5B,MAAI,SAAS,UAAa,UAAU,IAAI,IAAI,EAAG,QAAO;AAEtD,QAAM,SAAS,OAAO,IAAI;AAE1B,MAAI,OAAQ,QAAO,MAAM,QAAQ,EAAE,KAAK,CAAC;AAEzC,SAAO;AACT;;;AC7DA,SAAS,IAAI,OAAe,QAAwB;AAClD,SAAO,OAAO,KAAK,EAAE,SAAS,QAAQ,GAAG;AAC3C;AAWA,SAAS,WAAW,WAA2B;AAC7C,QAAM,OAAO,IAAI,KAAK,SAAS;AAE/B,QAAM,MAAM;AAAA,IACV,IAAI,KAAK,eAAe,GAAG,CAAC;AAAA,IAC5B,IAAI,KAAK,YAAY,IAAI,GAAG,CAAC;AAAA,IAC7B,IAAI,KAAK,WAAW,GAAG,CAAC;AAAA,EAC1B,EAAE,KAAK,GAAG;AAEV,QAAM,OAAO;AAAA,IACX,IAAI,KAAK,YAAY,GAAG,CAAC;AAAA,IACzB,IAAI,KAAK,cAAc,GAAG,CAAC;AAAA,IAC3B,IAAI,KAAK,cAAc,GAAG,CAAC;AAAA,EAC7B,EAAE,KAAK,GAAG;AAEV,SAAO,GAAG,GAAG,IAAI,IAAI,IAAI,IAAI,KAAK,mBAAmB,GAAG,CAAC,CAAC;AAC5D;AAOA,SAAS,YAAY,OAAwB;AAC3C,MAAI,UAAU,UAAa,UAAU,KAAM,QAAO;AAElD,MAAI,OAAO,UAAU,UAAU;AAC7B,QAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,UAAI,MAAM,WAAW,EAAG,QAAO;AAAA,IACjC,WAAW,OAAO,KAAK,KAAK,EAAE,WAAW,EAAG,QAAO;AAAA,EACrD;AAEA,SAAO,KAAK,UAAU,KAAK;AAC7B;AAGO,SAAS,WAAW,OAAsC;AAC/D,SAAO;AAAA,IACL,MAAM,MAAM;AAAA,IACZ,MAAM,YAAY,MAAM,IAAI;AAAA,IAC5B,SAAS,YAAY,MAAM,OAAO;AAAA,IAClC,SAAS,YAAY,MAAM,OAAO;AAAA,IAClC,QAAQ,YAAY,MAAM,MAAM;AAAA,IAChC,MAAM,YAAY,MAAM,IAAI;AAAA,IAC5B,QAAQ,YAAY,MAAM,MAAM;AAAA,IAChC,SAAS,YAAY,MAAM,OAAO;AAAA,IAClC,IAAI,MAAM;AAAA,IACV,SAAS,MAAM;AAAA,IACf,QAAQ,MAAM;AAAA,IACd,QAAQ,MAAM;AAAA,IACd,WAAW,WAAW,MAAM,SAAS;AAAA,IACrC,QAAQ,MAAM;AAAA,IACd,QAAQ,YAAY,MAAM,MAAM;AAAA,EAClC;AACF;;;ACnHA,SAAS,kBAAkB;AAwDpB,SAAS,mBAAmB,KAAgC;AACjE,SAAO,WAAW,QAAQ,EAAE,OAAO,KAAK,UAAU,GAAG,CAAC,EAAE,OAAO,KAAK;AACtE;AAOA,SAAS,aAAqB;AAC5B,SAAO,IAAI,KAAK,OAAO,KAAK,mBAAmB;AACjD;AAEA,SAAS,KAAK,IAA2B;AACvC,SAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,eAAW,SAAS,EAAE;AAAA,EACxB,CAAC;AACH;AAqBA,eAAsB,WACpB,SACA,UACA,QACe;AACf,QAAM,EAAE,QAAQ,MAAM,IAAI;AAQ1B,QAAM,UAAU,oBAAoB,SAAS,UAAU;AAKvD,QAAM,sBAA0C;AAAA,IAC9C,GAAG,sBAAsB,SAAS,kBAAkB;AAAA,IACpD,4BAA4B;AAAA,MAC1B,QAAQ,IAAI,CAAC,UAAU,MAAM,EAAE;AAAA,IACjC;AAAA,EACF;AAEA,QAAM,SAA4C;AAAA,IAChD;AAAA,IACA,QAAQ,QAAQ,IAAI,CAAC,UAAU,MAAM,GAAG;AAAA,IACxC,QAAQ;AAAA,IACR;AAAA,EACF;AAEA,WAAS,UAAU,KAAK,WAAW,GAAG;AACpC,WAAO,MAAM,qBAAqB,EAAE,OAAO,MAAM,QAAQ,QAAQ,QAAQ,CAAC;AAE1E,QAAI;AACF,YAAM,OAAO,OAAO,MAAM;AAE1B;AAAA,IACF,SAAS,OAAO;AAGd,YAAM,YAAY,SAAS,OAAO,MAAM,MAAM;AAE9C,UAAI,CAAC,aAAa,WAAW,QAAS,OAAM;AAE5C,YAAM,QAAQ,WAAW,SAAS,WAAW,CAAC;AAS9C,UAAI,CAAC,OAAO,SAAS,KAAK,EAAG,OAAM;AAEnC,aAAO,KAAK,sCAAsC;AAAA,QAChD,MAAM,UAAU,KAAK;AAAA,QACrB;AAAA,QACA;AAAA,MACF,CAAC;AAED,YAAM,KAAK,KAAK;AAAA,IAClB;AAAA,EACF;AACF;;;AHvIA,SAAS,cAAc,EAAE,OAAO,KAAK,GAAuB;AAC1D,SAAO;AAAA,IACL,KAAK,SAAS,IAAI,IAAI,OAAO,WAAW,KAAK;AAAA,IAC7C,IAAI,MAAM;AAAA,EACZ;AACF;AAiBA,eAAsB,QACpB,SACA,QACA,QACe;AACf,QAAM,WAAW,gBAAgB,QAAQ,MAAM;AAE/C,MAAI;AACF,UAAM,WAAW,QAAQ,IAAI,aAAa,GAAG,UAAU,MAAM;AAAA,EAC/D,SAAS,OAAO;AAMd,WAAO,MAAM,4BAA4B;AAAA,MACvC,MAAM,UAAU,KAAK;AAAA,MACrB,SAAS,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,MAAM,QAAQ;AAAA,IAChB,CAAC;AAED,UAAM;AAAA,EACR;AAIA,SAAO,MAAM,wBAAwB,EAAE,MAAM,QAAQ,OAAO,CAAC;AAC/D;;;AInEO,IAAM,OAAe,OAAO,OAAO,EAAE,QAAQ,MAAM,OAAO,MAAM;AACrE,QAAM,QAAQ,CAAC,EAAE,OAAO,KAAK,CAAC,GAAG,QAAQ,MAAM;AACjD;;;ACIO,IAAM,YAAyB,OAAO,OAAO,EAAE,QAAQ,OAAO,MAAM;AAGzE,MAAI,MAAM,QAAQ,WAAW,EAAG;AAEhC,QAAM,QAAQ,MAAM,SAAS,QAAQ,MAAM;AAC7C;;;ACtBA;;;ACeO,IAAM,wBAAqC;AAAA,EAChD,MAAM;AAAA,EAEN,QAAQ,CAAC;AAAA,EAET,MAAM,KAAK,EAAE,QAAQ,KAAK,OAAO,GAAG;AAClC,WAAO,UAAU,QAAQ,KAAK,MAAM;AAAA,EACtC;AAAA,EAEA;AAAA,EAEA;AAAA,EAEA,MAAM,QAAQ,EAAE,QAAQ,OAAO,GAAG;AAChC,UAAM,WAAW,OAAO;AACxB,UAAM,SAAS,UAAU;AAEzB,QAAI,CAAC,YAAY,CAAC,OAAQ;AAK1B,aAAS,SAAS;AAOlB,UAAM,OAAO,MAAM;AAEnB,WAAO,MAAM,0BAA0B;AAAA,EACzC;AACF;AAEA,IAAO,gBAAQ;","names":[]}

package/dist/walkerOS.json

@@ -0,0 +1,379 @@
+{
+  "$meta": {
+    "package": "@walkeros/server-destination-clickhouse",
+    "version": "4.6.1",
+    "type": "destination",
+    "platform": [
+      "server"
+    ],
+    "docs": "https://www.walkeros.io/docs/destinations/server/clickhouse",
+    "source": "https://github.com/elbwalker/walkerOS/tree/main/packages/server/destinations/clickhouse/src"
+  },
+  "schemas": {
+    "mapping": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "type": "object",
+      "properties": {},
+      "additionalProperties": false
+    },
+    "settings": {
+      "$schema": "http://json-schema.org/draft-07/schema#",
+      "type": "object",
+      "properties": {
+        "url": {
+          "type": "string",
+          "minLength": 1,
+          "description": "ClickHouse HTTP endpoint (like https://host.clickhouse.cloud:8443)"
+        },
+        "database": {
+          "default": "default",
+          "description": "Database holding the events table (like analytics)",
+          "type": "string",
+          "minLength": 1
+        },
+        "table": {
+          "default": "events",
+          "description": "Table every event is inserted into (like events)",
+          "type": "string",
+          "minLength": 1
+        },
+        "maxRetries": {
+          "default": 1,
+          "description": "Retries added on top of the first attempt, so total attempts = 1 + maxRetries (like 1 for two attempts)",
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "clickhouse": {
+          "description": "Raw @clickhouse/client options merged into the created client (like { request_timeout: 60000 })",
+          "type": "object",
+          "propertyNames": {
+            "type": "string"
+          },
+          "additionalProperties": {}
+        },
+        "clickhouseSettings": {
+          "description": "ClickHouse settings applied to every insert (like { async_insert: 1 })",
+          "type": "object",
+          "propertyNames": {
+            "type": "string"
+          },
+          "additionalProperties": {}
+        }
+      },
+      "required": [
+        "url"
+      ],
+      "additionalProperties": false
+    }
+  },
+  "examples": {
+    "env": {
+      "push": {
+        "ClickHouseClient": {
+          "$code": "e=>(_.push([\"createClient\",e]),{async insert(e){_.push([\"insert\",e]);const t=function(){if(!v||v.remaining<1)return;v.remaining-=1;const{code:e,message:t,type:i}=v;return void 0===e?new Error(t):new w({message:t,code:e,type:i})}();if(t)throw t;return{executed:!0,query_id:`mock-insert-${_.length}`,response_headers:{}}},async close(){_.push([\"close\"])}})"
+        }
+      },
+      "simulation": [
+        "ClickHouseClient"
+      ]
+    },
+    "step": {
+      "init": {
+        "title": "Initialization",
+        "description": "Init creates a single ClickHouse client for the configured endpoint, database and credentials. The client is held on the resolved config and reused by every insert. Its request timeout is derived from the collector timeout so a retried insert cannot outlive the delivery it belongs to, and request compression is enabled.",
+        "in": {
+          "settings": {
+            "url": "https://clickhouse.example.com:8443",
+            "database": "analytics",
+            "table": "events"
+          },
+          "credentials": {
+            "username": "walkeros_writer",
+            "password": "$secret.CLICKHOUSE_PASSWORD"
+          }
+        },
+        "out": [
+          [
+            "createClient",
+            {
+              "url": "https://clickhouse.example.com:8443",
+              "database": "analytics",
+              "username": "walkeros_writer",
+              "password": "$secret.CLICKHOUSE_PASSWORD",
+              "request_timeout": 4812,
+              "compression": {
+                "request": true
+              }
+            }
+          ]
+        ]
+      },
+      "initPassthrough": {
+        "title": "Client option passthrough",
+        "description": "The raw settings.clickhouse options reach the client underneath the destination's own settings: url, database and the credentials slot win over them, an explicit request_timeout or compression wins over the derived default, and any other option travels through untouched.",
+        "public": false,
+        "in": {
+          "settings": {
+            "url": "https://clickhouse.example.com:8443",
+            "database": "analytics",
+            "clickhouse": {
+              "url": "https://ignored.example.com:8443",
+              "database": "ignored",
+              "username": "ignored_user",
+              "request_timeout": 60000,
+              "max_open_connections": 25,
+              "application": "walkeros"
+            }
+          },
+          "credentials": {
+            "username": "walkeros_writer",
+            "password": "$secret.CLICKHOUSE_PASSWORD"
+          }
+        },
+        "out": [
+          [
+            "createClient",
+            {
+              "url": "https://clickhouse.example.com:8443",
+              "database": "analytics",
+              "username": "walkeros_writer",
+              "password": "$secret.CLICKHOUSE_PASSWORD",
+              "request_timeout": 60000,
+              "max_open_connections": 25,
+              "application": "walkeros",
+              "compression": {
+                "request": true
+              }
+            }
+          ]
+        ]
+      },
+      "pageView": {
+        "title": "Page view",
+        "description": "A page view is inserted as a single JSONEachRow row. The payload columns (data, context, globals, custom, user, nested, consent, source) are JSON-encoded strings and the timestamp is a UTC datetime literal, both produced by eventToRow.",
+        "in": {
+          "name": "page view",
+          "data": {
+            "domain": "www.example.com",
+            "title": "walkerOS documentation",
+            "referrer": "https://www.walkeros.io/",
+            "search": "?foo=bar",
+            "hash": "#hash",
+            "id": "/docs/"
+          },
+          "context": {
+            "dev": [
+              "test",
+              1
+            ]
+          },
+          "globals": {
+            "pagegroup": "docs"
+          },
+          "custom": {
+            "completely": "random"
+          },
+          "user": {
+            "id": "us3r",
+            "device": "c00k13",
+            "session": "s3ss10n"
+          },
+          "nested": [
+            {
+              "entity": "child",
+              "data": {
+                "is": "subordinated"
+              }
+            }
+          ],
+          "consent": {
+            "functional": true
+          },
+          "id": "294fcda5e2876f7e",
+          "trigger": "load",
+          "entity": "page",
+          "action": "view",
+          "timestamp": 1700000000123,
+          "timing": 3.14,
+          "source": {
+            "count": 1,
+            "trace": "0a1b2c3d4e5f60718293a4b5c6d7e8f9",
+            "type": "collector",
+            "schema": "4"
+          }
+        },
+        "out": [
+          [
+            "insert",
+            {
+              "table": "events",
+              "values": [
+                {
+                  "name": "page view",
+                  "data": "{\"domain\":\"www.example.com\",\"title\":\"walkerOS documentation\",\"referrer\":\"https://www.walkeros.io/\",\"search\":\"?foo=bar\",\"hash\":\"#hash\",\"id\":\"/docs/\"}",
+                  "context": "{\"dev\":[\"test\",1]}",
+                  "globals": "{\"pagegroup\":\"docs\"}",
+                  "custom": "{\"completely\":\"random\"}",
+                  "user": "{\"id\":\"us3r\",\"device\":\"c00k13\",\"session\":\"s3ss10n\"}",
+                  "nested": "[{\"entity\":\"child\",\"data\":{\"is\":\"subordinated\"}}]",
+                  "consent": "{\"functional\":true}",
+                  "id": "294fcda5e2876f7e",
+                  "trigger": "load",
+                  "entity": "page",
+                  "action": "view",
+                  "timestamp": "2023-11-14 22:13:20.123",
+                  "timing": 3.14,
+                  "source": "{\"count\":1,\"trace\":\"0a1b2c3d4e5f60718293a4b5c6d7e8f9\",\"type\":\"collector\",\"schema\":\"4\"}"
+                }
+              ],
+              "format": "JSONEachRow",
+              "clickhouse_settings": {
+                "async_insert": 0,
+                "input_format_skip_unknown_fields": 0,
+                "input_format_null_as_default": 0,
+                "insert_deduplication_token": "a065eedc371c98741e4269d65b72e490a837ea19896a827b9dd91d6ea4cb1406"
+              }
+            }
+          ]
+        ]
+      },
+      "purchase": {
+        "title": "Purchase",
+        "description": "An order event is inserted as one row. The nested items array is JSON-encoded into the data column, so a richer payload needs no table change. Promoting a hot key to its own column is a MATERIALIZED expression in the table DDL, not a destination setting.",
+        "in": {
+          "name": "order complete",
+          "data": {
+            "id": "ORD-500",
+            "currency": "EUR",
+            "total": 199.99,
+            "items": [
+              {
+                "sku": "SKU-1",
+                "quantity": 2
+              },
+              {
+                "sku": "SKU-2",
+                "quantity": 1
+              }
+            ]
+          },
+          "context": {
+            "shopping": [
+              "complete",
+              0
+            ]
+          },
+          "globals": {
+            "pagegroup": "shop"
+          },
+          "custom": {
+            "completely": "random"
+          },
+          "user": {
+            "id": "us3r",
+            "device": "c00k13",
+            "session": "s3ss10n"
+          },
+          "nested": [
+            {
+              "entity": "product",
+              "data": {
+                "id": "ers",
+                "name": "Everyday Ruck Snack",
+                "color": "black",
+                "size": "l",
+                "price": 420
+              },
+              "context": {
+                "shopping": [
+                  "complete",
+                  0
+                ]
+              },
+              "nested": []
+            },
+            {
+              "entity": "product",
+              "data": {
+                "id": "cc",
+                "name": "Cool Cap",
+                "size": "one size",
+                "price": 42
+              },
+              "context": {
+                "shopping": [
+                  "complete",
+                  0
+                ]
+              },
+              "nested": []
+            },
+            {
+              "entity": "gift",
+              "data": {
+                "name": "Surprise"
+              },
+              "context": {
+                "shopping": [
+                  "complete",
+                  0
+                ]
+              },
+              "nested": []
+            }
+          ],
+          "consent": {
+            "functional": true
+          },
+          "id": "f8f3095091ad89fc",
+          "trigger": "load",
+          "entity": "order",
+          "action": "complete",
+          "timestamp": 1700000000456,
+          "timing": 3.14,
+          "source": {
+            "count": 1,
+            "trace": "0a1b2c3d4e5f60718293a4b5c6d7e8f9",
+            "type": "collector",
+            "schema": "4"
+          }
+        },
+        "out": [
+          [
+            "insert",
+            {
+              "table": "events",
+              "values": [
+                {
+                  "name": "order complete",
+                  "data": "{\"id\":\"ORD-500\",\"currency\":\"EUR\",\"total\":199.99,\"items\":[{\"sku\":\"SKU-1\",\"quantity\":2},{\"sku\":\"SKU-2\",\"quantity\":1}]}",
+                  "context": "{\"shopping\":[\"complete\",0]}",
+                  "globals": "{\"pagegroup\":\"shop\"}",
+                  "custom": "{\"completely\":\"random\"}",
+                  "user": "{\"id\":\"us3r\",\"device\":\"c00k13\",\"session\":\"s3ss10n\"}",
+                  "nested": "[{\"entity\":\"product\",\"data\":{\"id\":\"ers\",\"name\":\"Everyday Ruck Snack\",\"color\":\"black\",\"size\":\"l\",\"price\":420},\"context\":{\"shopping\":[\"complete\",0]},\"nested\":[]},{\"entity\":\"product\",\"data\":{\"id\":\"cc\",\"name\":\"Cool Cap\",\"size\":\"one size\",\"price\":42},\"context\":{\"shopping\":[\"complete\",0]},\"nested\":[]},{\"entity\":\"gift\",\"data\":{\"name\":\"Surprise\"},\"context\":{\"shopping\":[\"complete\",0]},\"nested\":[]}]",
+                  "consent": "{\"functional\":true}",
+                  "id": "f8f3095091ad89fc",
+                  "trigger": "load",
+                  "entity": "order",
+                  "action": "complete",
+                  "timestamp": "2023-11-14 22:13:20.456",
+                  "timing": 3.14,
+                  "source": "{\"count\":1,\"trace\":\"0a1b2c3d4e5f60718293a4b5c6d7e8f9\",\"type\":\"collector\",\"schema\":\"4\"}"
+                }
+              ],
+              "format": "JSONEachRow",
+              "clickhouse_settings": {
+                "async_insert": 0,
+                "input_format_skip_unknown_fields": 0,
+                "input_format_null_as_default": 0,
+                "insert_deduplication_token": "7349cbacb4af8deb89e4bebef480abee592232fc783c3e7acade79d1bbb4ec3f"
+              }
+            }
+          ]
+        ]
+      }
+    }
+  }
+}

package/package.json

@@ -0,0 +1,73 @@
+{
+  "name": "@walkeros/server-destination-clickhouse",
+  "description": "ClickHouse server destination for walkerOS (bulk JSONEachRow inserts, bounded retry, insert deduplication)",
+  "version": "0.0.0-bootstrap",
+  "license": "MIT",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.mjs",
+      "require": "./dist/index.js"
+    },
+    "./dev": {
+      "types": "./dist/dev.d.ts",
+      "import": "./dist/dev.mjs",
+      "require": "./dist/dev.js"
+    },
+    "./walkerOS.json": "./dist/walkerOS.json",
+    "./examples": {
+      "types": "./dist/examples/index.d.ts",
+      "import": "./dist/examples/index.mjs",
+      "require": "./dist/examples/index.js"
+    }
+  },
+  "files": [
+    "dist/**",
+    "CHANGELOG.md"
+  ],
+  "scripts": {
+    "build": "tsup --silent",
+    "clean": "rm -rf .turbo && rm -rf dist",
+    "dev": "jest --watchAll --colors",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint \"**/*.ts*\"",
+    "test": "jest",
+    "update": "npx npm-check-updates -u && npm update"
+  },
+  "dependencies": {
+    "@clickhouse/client": "1.23.1",
+    "@walkeros/core": "4.6.1",
+    "@walkeros/server-core": "4.6.1"
+  },
+  "repository": {
+    "url": "git+https://github.com/elbwalker/walkerOS.git",
+    "directory": "packages/server/destinations/clickhouse"
+  },
+  "author": "elbwalker <hello@elbwalker.com>",
+  "homepage": "https://github.com/elbwalker/walkerOS#readme",
+  "bugs": {
+    "url": "https://github.com/elbwalker/walkerOS/issues"
+  },
+  "walkerOS": {
+    "type": "destination",
+    "platform": [
+      "server"
+    ],
+    "docs": "https://www.walkeros.io/docs/destinations/server/clickhouse"
+  },
+  "keywords": [
+    "walkerOS",
+    "walkerOS-destination",
+    "destination",
+    "server",
+    "clickhouse",
+    "warehouse",
+    "analytics"
+  ],
+  "funding": [
+    {
+      "type": "GitHub Sponsors",
+      "url": "https://github.com/sponsors/elbwalker"
+    }
+  ]
+}