mongo.do 0.1.0

package/dist/olap/cdc/cdc-schema.d.ts

@@ -0,0 +1,119 @@
+/**
+ * CDC Event Schema
+ *
+ * Defines the schema for Change Data Capture events.
+ * These events represent document changes (insert, update, delete)
+ * that can be emitted to downstream systems like Pipelines/ClickHouse.
+ */
+import { ObjectId } from '../../types/objectid';
+/**
+ * Namespace containing database and collection information
+ */
+export interface CDCNamespace {
+    db: string;
+    coll: string;
+}
+/**
+ * Document key containing the _id field
+ */
+export interface DocumentKey {
+    _id: ObjectId;
+}
+/**
+ * Update description with changed and removed fields
+ */
+export interface UpdateDescription {
+    updatedFields: Record<string, unknown>;
+    removedFields: string[];
+}
+/**
+ * Base interface for all CDC events
+ */
+export interface CDCEventBase {
+    eventId: string;
+    operationType: 'insert' | 'update' | 'delete';
+    ns: CDCNamespace;
+    documentKey: DocumentKey;
+    timestamp: Date;
+}
+/**
+ * Insert event - emitted when a document is inserted
+ */
+export interface InsertEvent extends CDCEventBase {
+    operationType: 'insert';
+    fullDocument: Record<string, unknown>;
+}
+/**
+ * Update event - emitted when a document is updated
+ */
+export interface UpdateEvent extends CDCEventBase {
+    operationType: 'update';
+    fullDocumentBeforeChange: Record<string, unknown> | null;
+    fullDocument: Record<string, unknown>;
+    updateDescription: UpdateDescription;
+}
+/**
+ * Delete event - emitted when a document is deleted
+ */
+export interface DeleteEvent extends CDCEventBase {
+    operationType: 'delete';
+    fullDocumentBeforeChange: Record<string, unknown> | null;
+}
+/**
+ * Union type for all CDC events
+ */
+export type CDCEvent = InsertEvent | UpdateEvent | DeleteEvent;
+/**
+ * Create insert event parameters
+ */
+export interface CreateInsertEventParams {
+    database: string;
+    collection: string;
+    document: Record<string, unknown> & {
+        _id: ObjectId;
+    };
+}
+/**
+ * Create an insert event for a new document
+ */
+export declare function createInsertEvent(params: CreateInsertEventParams): InsertEvent;
+/**
+ * Create update event parameters
+ */
+export interface CreateUpdateEventParams {
+    database: string;
+    collection: string;
+    documentKey: DocumentKey;
+    before: Record<string, unknown> | null;
+    after: Record<string, unknown>;
+}
+/**
+ * Create an update event for a document change
+ */
+export declare function createUpdateEvent(params: CreateUpdateEventParams): UpdateEvent;
+/**
+ * Create delete event parameters
+ */
+export interface CreateDeleteEventParams {
+    database: string;
+    collection: string;
+    documentKey: DocumentKey;
+    deletedDocument: Record<string, unknown> | null;
+}
+/**
+ * Create a delete event for a removed document
+ */
+export declare function createDeleteEvent(params: CreateDeleteEventParams): DeleteEvent;
+/**
+ * Serialize a CDC event to JSON string
+ */
+export declare function serializeToJSON(event: CDCEvent): string;
+/**
+ * Serialize a CDC event to BSON format
+ */
+export declare function serializeToBSON(event: CDCEvent): Uint8Array;
+/**
+ * Validate a CDC event structure
+ */
+export declare function validateEvent(event: CDCEvent): boolean;
+//# sourceMappingURL=cdc-schema.d.ts.map

package/dist/olap/cdc/cdc-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cdc-schema.d.ts","sourceRoot":"","sources":["../../../src/olap/cdc/cdc-schema.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAOhD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;CACd;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,GAAG,EAAE,QAAQ,CAAC;CACf;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACvC,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IAC9C,EAAE,EAAE,YAAY,CAAC;IACjB,WAAW,EAAE,WAAW,CAAC;IACzB,SAAS,EAAE,IAAI,CAAC;CACjB;AAED;;GAEG;AACH,MAAM,WAAW,WAAY,SAAQ,YAAY;IAC/C,aAAa,EAAE,QAAQ,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACvC;AAED;;GAEG;AACH,MAAM,WAAW,WAAY,SAAQ,YAAY;IAC/C,aAAa,EAAE,QAAQ,CAAC;IACxB,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACzD,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACtC,iBAAiB,EAAE,iBAAiB,CAAC;CACtC;AAED;;GAEG;AACH,MAAM,WAAW,WAAY,SAAQ,YAAY;IAC/C,aAAa,EAAE,QAAQ,CAAC;IACxB,wBAAwB,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1D;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,WAAW,GAAG,WAAW,GAAG,WAAW,CAAC;AAa/D;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;QAAE,GAAG,EAAE,QAAQ,CAAA;KAAE,CAAC;CACvD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,uBAAuB,GAAG,WAAW,CAW9E;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,WAAW,CAAC;IACzB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;IACvC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAgFD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,uBAAuB,GAAG,WAAW,CAa9E;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,WAAW,CAAC;IACzB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CACjD;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,uBAAuB,GAAG,WAAW,CAW9E;AAkDD;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM,CAGvD;AAkCD;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,QAAQ,GAAG,UAAU,CAG3D;AAWD;;GAEG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CA8BtD"}

package/dist/olap/cdc/cdc-schema.js

@@ -0,0 +1,253 @@
+/**
+ * CDC Event Schema
+ *
+ * Defines the schema for Change Data Capture events.
+ * These events represent document changes (insert, update, delete)
+ * that can be emitted to downstream systems like Pipelines/ClickHouse.
+ */
+import { randomUUID } from 'crypto';
+import { ObjectId } from '../../types/objectid';
+import { serialize } from 'bson';
+// ============================================================================
+// Event Creation Functions
+// ============================================================================
+/**
+ * Generate a unique event ID
+ */
+function generateEventId() {
+    return randomUUID();
+}
+/**
+ * Create an insert event for a new document
+ */
+export function createInsertEvent(params) {
+    const { database, collection, document } = params;
+    return {
+        eventId: generateEventId(),
+        operationType: 'insert',
+        ns: { db: database, coll: collection },
+        documentKey: { _id: document._id },
+        fullDocument: document,
+        timestamp: new Date(),
+    };
+}
+/**
+ * Compute the difference between before and after documents
+ */
+function computeUpdateDescription(before, after) {
+    const updatedFields = {};
+    const removedFields = [];
+    if (before === null) {
+        // If no before document, all fields in after are considered updated (except _id)
+        for (const key of Object.keys(after)) {
+            if (key !== '_id') {
+                updatedFields[key] = after[key];
+            }
+        }
+        return { updatedFields, removedFields };
+    }
+    // Find updated fields (changed or added)
+    for (const key of Object.keys(after)) {
+        if (key === '_id')
+            continue;
+        if (!(key in before) || !deepEquals(before[key], after[key])) {
+            updatedFields[key] = after[key];
+        }
+    }
+    // Find removed fields
+    for (const key of Object.keys(before)) {
+        if (key === '_id')
+            continue;
+        if (!(key in after)) {
+            removedFields.push(key);
+        }
+    }
+    return { updatedFields, removedFields };
+}
+/**
+ * Deep equality check for values
+ */
+function deepEquals(a, b) {
+    if (a === b)
+        return true;
+    if (a === null || b === null)
+        return a === b;
+    if (typeof a !== typeof b)
+        return false;
+    if (a instanceof ObjectId && b instanceof ObjectId) {
+        return a.equals(b);
+    }
+    if (a instanceof Date && b instanceof Date) {
+        return a.getTime() === b.getTime();
+    }
+    if (Array.isArray(a) && Array.isArray(b)) {
+        if (a.length !== b.length)
+            return false;
+        for (let i = 0; i < a.length; i++) {
+            if (!deepEquals(a[i], b[i]))
+                return false;
+        }
+        return true;
+    }
+    if (typeof a === 'object' && typeof b === 'object') {
+        const aKeys = Object.keys(a);
+        const bKeys = Object.keys(b);
+        if (aKeys.length !== bKeys.length)
+            return false;
+        for (const key of aKeys) {
+            if (!deepEquals(a[key], b[key])) {
+                return false;
+            }
+        }
+        return true;
+    }
+    return false;
+}
+/**
+ * Create an update event for a document change
+ */
+export function createUpdateEvent(params) {
+    const { database, collection, documentKey, before, after } = params;
+    return {
+        eventId: generateEventId(),
+        operationType: 'update',
+        ns: { db: database, coll: collection },
+        documentKey,
+        fullDocumentBeforeChange: before,
+        fullDocument: after,
+        updateDescription: computeUpdateDescription(before, after),
+        timestamp: new Date(),
+    };
+}
+/**
+ * Create a delete event for a removed document
+ */
+export function createDeleteEvent(params) {
+    const { database, collection, documentKey, deletedDocument } = params;
+    return {
+        eventId: generateEventId(),
+        operationType: 'delete',
+        ns: { db: database, coll: collection },
+        documentKey,
+        fullDocumentBeforeChange: deletedDocument,
+        timestamp: new Date(),
+    };
+}
+// ============================================================================
+// Serialization Functions
+// ============================================================================
+/**
+ * Custom JSON replacer for ObjectId and Date values
+ */
+function jsonReplacer(_key, value) {
+    if (value instanceof ObjectId) {
+        return value.toString();
+    }
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    return value;
+}
+/**
+ * Recursively convert ObjectId and Date instances to serializable format
+ */
+function convertForJSON(value) {
+    if (value === null || value === undefined) {
+        return value;
+    }
+    if (value instanceof ObjectId) {
+        return value.toString();
+    }
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    if (Array.isArray(value)) {
+        return value.map(convertForJSON);
+    }
+    if (typeof value === 'object') {
+        const result = {};
+        for (const [key, val] of Object.entries(value)) {
+            result[key] = convertForJSON(val);
+        }
+        return result;
+    }
+    return value;
+}
+/**
+ * Serialize a CDC event to JSON string
+ */
+export function serializeToJSON(event) {
+    const converted = convertForJSON(event);
+    return JSON.stringify(converted, jsonReplacer);
+}
+/**
+ * Recursively convert ObjectId instances to BSON-compatible format
+ */
+function convertForBSON(value) {
+    if (value === null || value === undefined) {
+        return value;
+    }
+    if (value instanceof ObjectId) {
+        // Convert to a plain object that BSON can serialize
+        return { $oid: value.toString() };
+    }
+    if (value instanceof Date) {
+        return value;
+    }
+    if (Array.isArray(value)) {
+        return value.map(convertForBSON);
+    }
+    if (typeof value === 'object') {
+        const result = {};
+        for (const [key, val] of Object.entries(value)) {
+            result[key] = convertForBSON(val);
+        }
+        return result;
+    }
+    return value;
+}
+/**
+ * Serialize a CDC event to BSON format
+ */
+export function serializeToBSON(event) {
+    const converted = convertForBSON(event);
+    return serialize(converted);
+}
+// ============================================================================
+// Validation Functions
+// ============================================================================
+/**
+ * Valid operation types for CDC events
+ */
+const VALID_OPERATION_TYPES = ['insert', 'update', 'delete'];
+/**
+ * Validate a CDC event structure
+ */
+export function validateEvent(event) {
+    if (!event || typeof event !== 'object') {
+        return false;
+    }
+    // Check operationType
+    if (!event.operationType || !VALID_OPERATION_TYPES.includes(event.operationType)) {
+        return false;
+    }
+    // Check namespace
+    if (!event.ns || typeof event.ns !== 'object') {
+        return false;
+    }
+    if (!event.ns.db || typeof event.ns.db !== 'string') {
+        return false;
+    }
+    if (!event.ns.coll || typeof event.ns.coll !== 'string') {
+        return false;
+    }
+    // Check documentKey
+    if (!event.documentKey || typeof event.documentKey !== 'object') {
+        return false;
+    }
+    if (!event.documentKey._id) {
+        return false;
+    }
+    return true;
+}
+//# sourceMappingURL=cdc-schema.js.map

package/dist/olap/cdc/cdc-schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cdc-schema.js","sourceRoot":"","sources":["../../../src/olap/cdc/cdc-schema.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,EAAE,SAAS,EAAiB,MAAM,MAAM,CAAC;AAuEhD,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E;;GAEG;AACH,SAAS,eAAe;IACtB,OAAO,UAAU,EAAE,CAAC;AACtB,CAAC;AAWD;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAA+B;IAC/D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAElD,OAAO;QACL,OAAO,EAAE,eAAe,EAAE;QAC1B,aAAa,EAAE,QAAQ;QACvB,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE;QACtC,WAAW,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,EAAE;QAClC,YAAY,EAAE,QAAQ;QACtB,SAAS,EAAE,IAAI,IAAI,EAAE;KACtB,CAAC;AACJ,CAAC;AAaD;;GAEG;AACH,SAAS,wBAAwB,CAC/B,MAAsC,EACtC,KAA8B;IAE9B,MAAM,aAAa,GAA4B,EAAE,CAAC;IAClD,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,iFAAiF;QACjF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACrC,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;gBAClB,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;IAC1C,CAAC;IAED,yCAAyC;IACzC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,GAAG,KAAK,KAAK;YAAE,SAAS;QAC5B,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC7D,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,sBAAsB;IACtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACtC,IAAI,GAAG,KAAK,KAAK;YAAE,SAAS;QAC5B,IAAI,CAAC,CAAC,GAAG,IAAI,KAAK,CAAC,EAAE,CAAC;YACpB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,CAAC;AAC1C,CAAC;AAED;;GAEG;AACH,SAAS,UAAU,CAAC,CAAU,EAAE,CAAU;IACxC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACzB,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7C,IAAI,OAAO,CAAC,KAAK,OAAO,CAAC;QAAE,OAAO,KAAK,CAAC;IAExC,IAAI,CAAC,YAAY,QAAQ,IAAI,CAAC,YAAY,QAAQ,EAAE,CAAC;QACnD,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IAED,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,EAAE,CAAC;QAC3C,OAAO,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACrC,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAAE,OAAO,KAAK,CAAC;QAC5C,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAW,CAAC,CAAC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAW,CAAC,CAAC;QACvC,IAAI,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,IAAI,CAAC,UAAU,CAAE,CAA6B,CAAC,GAAG,CAAC,EAAG,CAA6B,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBAC1F,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAA+B;IAC/D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IAEpE,OAAO;QACL,OAAO,EAAE,eAAe,EAAE;QAC1B,aAAa,EAAE,QAAQ;QACvB,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE;QACtC,WAAW;QACX,wBAAwB,EAAE,MAAM;QAChC,YAAY,EAAE,KAAK;QACnB,iBAAiB,EAAE,wBAAwB,CAAC,MAAM,EAAE,KAAK,CAAC;QAC1D,SAAS,EAAE,IAAI,IAAI,EAAE;KACtB,CAAC;AACJ,CAAC;AAYD;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAA+B;IAC/D,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;IAEtE,OAAO;QACL,OAAO,EAAE,eAAe,EAAE;QAC1B,aAAa,EAAE,QAAQ;QACvB,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE;QACtC,WAAW;QACX,wBAAwB,EAAE,eAAe;QACzC,SAAS,EAAE,IAAI,IAAI,EAAE;KACtB,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,0BAA0B;AAC1B,+EAA+E;AAE/E;;GAEG;AACH,SAAS,YAAY,CAAC,IAAY,EAAE,KAAc;IAChD,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC1B,CAAC;IAED,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAe;IAC7C,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACxC,OAAO,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;AACjD,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;QAC9B,oDAAoD;QACpD,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;IACpC,CAAC;IAED,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;QAC1B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAe;IAC7C,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACxC,OAAO,SAAS,CAAC,SAAqB,CAAC,CAAC;AAC1C,CAAC;AAED,+EAA+E;AAC/E,uBAAuB;AACvB,+EAA+E;AAE/E;;GAEG;AACH,MAAM,qBAAqB,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AAE7D;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,KAAe;IAC3C,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC;IACf,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC;QACjF,OAAO,KAAK,CAAC;IACf,CAAC;IAED,kBAAkB;IAClB,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,OAAO,KAAK,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;QACpD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACxD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,oBAAoB;IACpB,IAAI,CAAC,KAAK,CAAC,WAAW,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,CAAC;QAC3B,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/olap/cdc/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * CDC (Change Data Capture) Module
+ *
+ * Provides infrastructure for capturing and emitting document changes
+ * to downstream OLAP systems like ClickHouse via Cloudflare Pipelines.
+ */
+export * from './cdc-schema';
+export * from './cdc-emitter';
+export * from './cdc-buffer';
+//# sourceMappingURL=index.d.ts.map

package/dist/olap/cdc/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/olap/cdc/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC"}

package/dist/olap/cdc/index.js

@@ -0,0 +1,10 @@
+/**
+ * CDC (Change Data Capture) Module
+ *
+ * Provides infrastructure for capturing and emitting document changes
+ * to downstream OLAP systems like ClickHouse via Cloudflare Pipelines.
+ */
+export * from './cdc-schema';
+export * from './cdc-emitter';
+export * from './cdc-buffer';
+//# sourceMappingURL=index.js.map

package/dist/olap/cdc/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/olap/cdc/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,cAAc,cAAc,CAAC;AAC7B,cAAc,eAAe,CAAC;AAC9B,cAAc,cAAc,CAAC"}

package/dist/olap/clickhouse/iceberg.d.ts

@@ -0,0 +1,164 @@
+/**
+ * ClickHouse Iceberg Integration
+ *
+ * Provides connection and catalog discovery for ClickHouse with Apache Iceberg.
+ * This module enables querying Iceberg tables stored in object storage
+ * (S3, GCS, Azure Blob) through ClickHouse's native Iceberg support.
+ *
+ * Issue: mongo.do-vyf4
+ */
+/**
+ * Configuration for connecting to ClickHouse with Iceberg support
+ */
+export interface IcebergConnectionConfig {
+    /** ClickHouse server hostname */
+    host: string;
+    /** ClickHouse server port (defaults to 8443 for HTTPS, 8123 for HTTP) */
+    port?: number;
+    /** Database name */
+    database: string;
+    /** Username for authentication */
+    username?: string;
+    /** Password for authentication */
+    password?: string;
+    /** JWT token for authentication (alternative to username/password) */
+    jwtToken?: string;
+    /** Use HTTPS connection */
+    secure?: boolean;
+    /** Iceberg catalog name */
+    icebergCatalog: string;
+    /** Connection timeout in milliseconds */
+    connectionTimeout?: number;
+    /** Query timeout in milliseconds */
+    queryTimeout?: number;
+    /** Connection pool size */
+    poolSize?: number;
+    /** Maximum retry attempts for transient errors */
+    maxRetries?: number;
+}
+/**
+ * Represents an Iceberg catalog in ClickHouse
+ */
+export interface IcebergCatalog {
+    /** Catalog name */
+    name: string;
+    /** Catalog type (iceberg) */
+    type: string;
+    /** Warehouse location (S3/GCS/Azure path) */
+    warehouse: string;
+    /** Metadata path */
+    metadataPath?: string;
+    /** Creation timestamp */
+    createdAt?: Date;
+}
+/**
+ * Represents an Iceberg table
+ */
+export interface IcebergTable {
+    /** Table name */
+    name: string;
+    /** Namespace/schema */
+    namespace: string;
+    /** Table format */
+    format: string;
+    /** Storage location */
+    location?: string;
+    /** Partition specification */
+    partitionSpec?: string;
+    /** Current snapshot ID */
+    currentSnapshotId?: string;
+    /** Total record count */
+    totalRecords?: number;
+    /** Total size in bytes */
+    totalSizeBytes?: number;
+}
+/**
+ * Represents an Iceberg table schema
+ */
+export interface IcebergSchema {
+    /** Schema columns */
+    columns: IcebergColumn[];
+}
+/**
+ * Represents a column in an Iceberg table
+ */
+export interface IcebergColumn {
+    /** Column name */
+    name: string;
+    /** ClickHouse data type */
+    type: string;
+    /** Position in schema */
+    position: number;
+    /** Whether the column is nullable */
+    nullable: boolean;
+    /** Default value */
+    default?: unknown;
+    /** Column comment/description */
+    comment?: string;
+    /** Whether this column is part of the partition key */
+    isPartitionKey?: boolean;
+}
+/**
+ * Client for interacting with ClickHouse Iceberg tables
+ */
+export declare class ClickHouseIcebergClient {
+    private _closed;
+    private _config;
+    private _baseUrl;
+    constructor(config: IcebergConnectionConfig);
+    /**
+     * Get the configuration
+     */
+    getConfig(): IcebergConnectionConfig;
+    /**
+     * Get the base URL
+     */
+    getBaseUrl(): string;
+    /**
+     * Check if the connection is authenticated
+     */
+    isAuthenticated(): Promise<boolean>;
+    /**
+     * Get the connection timeout
+     */
+    getTimeout(): number;
+    /**
+     * Get the connection timeout
+     */
+    getConnectionTimeout(): number;
+    /**
+     * Get the query timeout
+     */
+    getQueryTimeout(): number;
+    /**
+     * Check if the connection is closed
+     */
+    isClosed(): boolean;
+    /**
+     * Close the connection
+     */
+    close(): Promise<void>;
+    /**
+     * Get the connection pool size
+     */
+    getPoolSize(): number;
+}
+/**
+ * Create a new Iceberg connection to ClickHouse
+ */
+export declare function createIcebergConnection(config: IcebergConnectionConfig): Promise<ClickHouseIcebergClient>;
+/**
+ * Discover an Iceberg catalog in ClickHouse
+ */
+export declare function discoverCatalog(_connection: ClickHouseIcebergClient, _catalogName: string): Promise<IcebergCatalog>;
+/**
+ * Discover tables in an Iceberg catalog
+ */
+export declare function discoverTables(_connection: ClickHouseIcebergClient, _options?: {
+    namespace?: string;
+}): Promise<IcebergTable[]>;
+/**
+ * Get the schema of an Iceberg table
+ */
+export declare function getTableSchema(_connection: ClickHouseIcebergClient, _tableName: string): Promise<IcebergSchema>;
+//# sourceMappingURL=iceberg.d.ts.map

package/dist/olap/clickhouse/iceberg.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"iceberg.d.ts","sourceRoot":"","sources":["../../../src/olap/clickhouse/iceberg.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACtC,iCAAiC;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oBAAoB;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2BAA2B;IAC3B,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,2BAA2B;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,yCAAyC;IACzC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,oCAAoC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,mBAAmB;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAC;IACb,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,yBAAyB;IACzB,SAAS,CAAC,EAAE,IAAI,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,uBAAuB;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,mBAAmB;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,uBAAuB;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,8BAA8B;IAC9B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,0BAA0B;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yBAAyB;IACzB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0BAA0B;IAC1B,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,qBAAqB;IACrB,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,kBAAkB;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,2BAA2B;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,qCAAqC;IACrC,QAAQ,EAAE,OAAO,CAAC;IAClB,oBAAoB;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uDAAuD;IACvD,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAMD;;GAEG;AACH,qBAAa,uBAAuB;IAClC,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,EAAE,uBAAuB;IAe3C;;OAEG;IACH,SAAS,IAAI,uBAAuB;IAIpC;;OAEG;IACH,UAAU,IAAI,MAAM;IAIpB;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,OAAO,CAAC;IAIzC;;OAEG;IACH,UAAU,IAAI,MAAM;IAIpB;;OAEG;IACH,oBAAoB,IAAI,MAAM;IAI9B;;OAEG;IACH,eAAe,IAAI,MAAM;IAIzB;;OAEG;IACH,QAAQ,IAAI,OAAO;IAInB;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;OAEG;IACH,WAAW,IAAI,MAAM;CAGtB;AAMD;;GAEG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,uBAAuB,CAAC,CA8BlC;AAED;;GAEG;AACH,wBAAsB,eAAe,CACnC,WAAW,EAAE,uBAAuB,EACpC,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,cAAc,CAAC,CAEzB;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,WAAW,EAAE,uBAAuB,EACpC,QAAQ,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAChC,OAAO,CAAC,YAAY,EAAE,CAAC,CAEzB;AAED;;GAEG;AACH,wBAAsB,cAAc,CAClC,WAAW,EAAE,uBAAuB,EACpC,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,aAAa,CAAC,CAExB"}