@openfn/language-motherduck 1.0.0

package/dist/index.js

@@ -0,0 +1,512 @@
+var __defProp = Object.defineProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/Adaptor.js
+var Adaptor_exports = {};
+__export(Adaptor_exports, {
+  alterState: () => alterState,
+  arrayToString: () => arrayToString,
+  as: () => as,
+  combine: () => combine,
+  cursor: () => cursor,
+  dataPath: () => dataPath,
+  dataValue: () => dataValue,
+  dateFns: () => dateFns,
+  each: () => each,
+  execute: () => execute,
+  field: () => field,
+  fields: () => fields,
+  fn: () => fn,
+  fnIf: () => fnIf,
+  group: () => group,
+  insert: () => insert,
+  lastReferenceValue: () => lastReferenceValue,
+  map: () => map,
+  merge: () => merge,
+  query: () => query,
+  sourceValue: () => sourceValue,
+  util: () => util_exports
+});
+import {
+  execute as commonExecute,
+  composeNextState
+} from "@openfn/language-common";
+import { expandReferences } from "@openfn/language-common/util";
+import { DuckDBInstance } from "@duckdb/node-api";
+import _ from "lodash";
+
+// src/util.js
+var util_exports = {};
+__export(util_exports, {
+  convertBigIntToNumber: () => convertBigIntToNumber,
+  escapeSqlString: () => escapeSqlString,
+  formatSqlValue: () => formatSqlValue,
+  queryHandler: () => queryHandler,
+  validateSqlIdentifier: () => validateSqlIdentifier
+});
+function validateSqlIdentifier(identifier) {
+  if (typeof identifier !== "string") {
+    throw new Error("SQL identifier must be a string");
+  }
+  if (!/^[a-zA-Z_][a-zA-Z0-9_.]*$/.test(identifier)) {
+    throw new Error(`Invalid SQL identifier: ${identifier}. Only alphanumeric characters, underscores, and dots are allowed.`);
+  }
+  const upperIdentifier = identifier.toUpperCase();
+  const dangerousPatterns = [
+    "DROP",
+    "DELETE",
+    "INSERT",
+    "UPDATE",
+    "ALTER",
+    "CREATE",
+    "EXEC",
+    "UNION",
+    "SELECT",
+    "--",
+    "/*",
+    "*/",
+    ";"
+  ];
+  for (const pattern of dangerousPatterns) {
+    if (upperIdentifier.includes(pattern)) {
+      throw new Error(`SQL identifier contains forbidden pattern: ${pattern}`);
+    }
+  }
+  return identifier;
+}
+function escapeSqlString(value) {
+  if (typeof value !== "string") {
+    return value;
+  }
+  return value.replace(/'/g, "''");
+}
+function formatSqlValue(value) {
+  if (value === null || value === void 0) {
+    return "NULL";
+  }
+  if (typeof value === "string") {
+    return `'${escapeSqlString(value)}'`;
+  }
+  if (typeof value === "boolean") {
+    return value ? "TRUE" : "FALSE";
+  }
+  return value.toString();
+}
+function convertBigIntToNumber(obj) {
+  if (typeof obj === "bigint") {
+    if (obj <= Number.MAX_SAFE_INTEGER && obj >= Number.MIN_SAFE_INTEGER) {
+      return Number(obj);
+    } else {
+      return obj.toString();
+    }
+  }
+  if (Array.isArray(obj)) {
+    return obj.map(convertBigIntToNumber);
+  }
+  if (obj !== null && typeof obj === "object") {
+    if (obj.width !== void 0 && obj.scale !== void 0 && obj.value !== void 0) {
+      const scale = Number(obj.scale);
+      const value = typeof obj.value === "bigint" ? Number(obj.value) : obj.value;
+      return value / Math.pow(10, scale);
+    }
+    const converted = {};
+    for (const [key, value] of Object.entries(obj)) {
+      converted[key] = convertBigIntToNumber(value);
+    }
+    return converted;
+  }
+  return obj;
+}
+async function queryHandler(connection2, state, sqlQuery, options, composeNextState2) {
+  if (!connection2) {
+    throw new Error("No active MotherDuck connection found. Ensure you are running within an execute() block.");
+  }
+  try {
+    const result = await connection2.runAndReadAll(sqlQuery);
+    const rawRows = result.getRowObjects();
+    const rows = convertBigIntToNumber(rawRows);
+    const nextState = {
+      ...composeNextState2(state, rows),
+      response: {
+        rows,
+        rowCount: rows.length,
+        command: sqlQuery.trim().split(" ")[0].toUpperCase(),
+        query: options.writeSql ? sqlQuery : "[query hidden]"
+      }
+    };
+    return nextState;
+  } catch (error) {
+    const errorMessage = `MotherDuck query failed: ${error.message}`;
+    console.error(errorMessage);
+    console.error("Failed query:", sqlQuery.substring(0, 200) + (sqlQuery.length > 200 ? "..." : ""));
+    const enhancedError = new Error(errorMessage);
+    enhancedError.originalError = error;
+    enhancedError.query = sqlQuery;
+    throw enhancedError;
+  }
+}
+
+// src/mock.js
+var mockTables = {};
+function parseSelectLiterals(sql) {
+  const result = {};
+  const selectMatch = sql.match(/SELECT\s+(.+?)(?:\s+FROM|\s*$)/is);
+  if (!selectMatch)
+    return result;
+  const selectClause = selectMatch[1].trim();
+  const columns = [];
+  let current = "";
+  let inQuotes = false;
+  let parenDepth = 0;
+  for (let i = 0; i < selectClause.length; i++) {
+    const char = selectClause[i];
+    if (char === "'" && (i === 0 || selectClause[i - 1] !== "\\")) {
+      inQuotes = !inQuotes;
+    }
+    if (!inQuotes) {
+      if (char === "(")
+        parenDepth++;
+      if (char === ")")
+        parenDepth--;
+      if (char === "," && parenDepth === 0) {
+        columns.push(current.trim());
+        current = "";
+        continue;
+      }
+    }
+    current += char;
+  }
+  if (current.trim())
+    columns.push(current.trim());
+  columns.forEach((col) => {
+    var _a;
+    const stringMatch = col.match(/'([^']*)'\s+(?:as\s+)?(\w+)/i);
+    const numberMatch = col.match(/^(\d+)\s+(?:as\s+)?(\w+)/i);
+    const funcMatch = col.match(/^(\w+)\s*\([^)]*\)\s+(?:as\s+)?(\w+)/i);
+    if (stringMatch) {
+      result[stringMatch[2]] = stringMatch[1];
+    } else if (numberMatch) {
+      result[numberMatch[2]] = parseInt(numberMatch[1]);
+    } else if (funcMatch) {
+      const funcName = funcMatch[1].toLowerCase();
+      const alias = funcMatch[2];
+      if (funcName === "count") {
+        result[alias] = ((_a = mockTables[Object.keys(mockTables)[0]]) == null ? void 0 : _a.length) || 0;
+      } else if (funcName === "current_database") {
+        result[alias] = "my_db";
+      }
+    }
+  });
+  return result;
+}
+function createMockConnection() {
+  const mockConnection = {
+    runAndReadAll: async (sql) => {
+      if (/SELECT\s+FROM/i.test(sql) || /WHERE\s+INVALID/i.test(sql)) {
+        throw new Error('Parser Error: syntax error at or near "FROM"');
+      }
+      let mockData = [];
+      if (/CREATE TABLE/i.test(sql)) {
+        const tableMatch = sql.match(/CREATE TABLE\s+(\w+)/i);
+        if (tableMatch) {
+          const tableName = tableMatch[1];
+          mockTables[tableName] = [];
+        }
+        mockData = [{ success: true }];
+      } else if (/INSERT INTO/i.test(sql)) {
+        const tableMatch = sql.match(/INSERT INTO\s+(\w+)/i);
+        const valuesMatch = sql.match(/VALUES\s+(.+)/is);
+        if (tableMatch && valuesMatch) {
+          const tableName = tableMatch[1];
+          if (!mockTables[tableName])
+            mockTables[tableName] = [];
+          const columnsMatch = sql.match(/\(([^)]+)\)\s+VALUES/i);
+          const columns = columnsMatch ? columnsMatch[1].split(",").map((c) => c.trim()) : [];
+          const valuesSets = valuesMatch[1].match(/\([^)]+\)/g) || [];
+          valuesSets.forEach((valueSet) => {
+            const values = valueSet.slice(1, -1).split(",").map((v) => {
+              v = v.trim();
+              if (v.startsWith("'") && v.endsWith("'")) {
+                return v.slice(1, -1).replace(/''/g, "'");
+              }
+              if (v === "NULL")
+                return null;
+              if (v === "TRUE")
+                return true;
+              if (v === "FALSE")
+                return false;
+              if (/^\d+$/.test(v))
+                return parseInt(v);
+              return v;
+            });
+            const row = {};
+            columns.forEach((col, idx) => {
+              row[col] = values[idx];
+            });
+            mockTables[tableName].push(row);
+          });
+        }
+        mockData = [{ success: true }];
+      } else if (/SELECT/i.test(sql)) {
+        const fromMatch = sql.match(/FROM\s+(\w+)/i);
+        if (fromMatch) {
+          const tableName = fromMatch[1];
+          const tableData = mockTables[tableName] || [];
+          if (/COUNT\s*\(\s*\*\s*\)/i.test(sql)) {
+            const aliasMatch = sql.match(/COUNT\s*\(\s*\*\s*\)\s+(?:as\s+)?(\w+)/i);
+            const alias = aliasMatch ? aliasMatch[1] : "count";
+            mockData = [{ [alias]: tableData.length }];
+          } else if (/SELECT\s+\*/i.test(sql)) {
+            mockData = [...tableData];
+          } else {
+            const selectMatch = sql.match(/SELECT\s+(.+?)\s+FROM/is);
+            if (selectMatch) {
+              const columns = selectMatch[1].split(",").map((c) => c.trim());
+              mockData = tableData.map((row) => {
+                const newRow = {};
+                columns.forEach((col) => {
+                  if (row[col] !== void 0) {
+                    newRow[col] = row[col];
+                  }
+                });
+                return newRow;
+              });
+            }
+          }
+        } else {
+          const literals = parseSelectLiterals(sql);
+          if (Object.keys(literals).length === 0) {
+            mockData = [];
+          } else {
+            mockData = [literals];
+          }
+        }
+      } else if (/UPDATE/i.test(sql)) {
+        mockData = [{ success: true }];
+      } else if (/DELETE/i.test(sql)) {
+        mockData = [{ success: true }];
+      } else if (/DROP TABLE/i.test(sql)) {
+        const tableMatch = sql.match(/DROP TABLE\s+(?:IF EXISTS\s+)?(\w+)/i);
+        if (tableMatch) {
+          delete mockTables[tableMatch[1]];
+        }
+        mockData = [{ success: true }];
+      } else {
+        mockData = [{ result: "mock_data" }];
+      }
+      return {
+        getRowObjects: () => mockData
+      };
+    },
+    close: () => {
+      Object.keys(mockTables).forEach((key) => delete mockTables[key]);
+    }
+  };
+  const mockInstance = {
+    connect: async () => mockConnection,
+    close: () => {
+      Object.keys(mockTables).forEach((key) => delete mockTables[key]);
+    }
+  };
+  return { mockInstance, mockConnection };
+}
+
+// src/Adaptor.js
+import {
+  alterState,
+  arrayToString,
+  combine,
+  cursor,
+  dataPath,
+  dataValue,
+  dateFns,
+  each,
+  field,
+  fields,
+  fn,
+  fnIf,
+  group,
+  lastReferenceValue,
+  map,
+  merge,
+  sourceValue,
+  as
+} from "@openfn/language-common";
+var instance = null;
+var connection = null;
+function execute(...operations) {
+  const initialState = {
+    references: [],
+    data: null
+  };
+  return async (state) => {
+    try {
+      return await commonExecute(
+        createConnection,
+        ...operations,
+        disconnect
+      )({ ...initialState, ...state });
+    } catch (error) {
+      disconnect(state);
+      throw error;
+    }
+  };
+}
+async function createConnection(state) {
+  const {
+    token,
+    database,
+    sessionHint,
+    testMode = false
+  } = state.configuration;
+  if (testMode || process.env.NODE_ENV === "test") {
+    const { mockInstance, mockConnection } = createMockConnection();
+    instance = mockInstance;
+    connection = mockConnection;
+    return state;
+  }
+  if (!token) {
+    throw new Error(
+      "MotherDuck token is required. Please provide a token in the configuration."
+    );
+  }
+  let databasePath = `md:${database}`;
+  if (sessionHint) {
+    databasePath += `?session_hint=${sessionHint}`;
+  }
+  const config = {
+    motherduck_token: token
+  };
+  console.log(`Connecting to MotherDuck cloud database: ${database}`);
+  instance = await DuckDBInstance.create(databasePath, config);
+  connection = await instance.connect();
+  console.log("Connected successfully to MotherDuck");
+  return state;
+}
+function disconnect(state) {
+  if (connection && typeof connection.close === "function") {
+    connection.close();
+  }
+  if (instance && typeof instance.close === "function") {
+    instance.close();
+  }
+  connection = null;
+  instance = null;
+  return state;
+}
+function query(sqlQuery, options = {}) {
+  return (state) => {
+    const [resolvedQuery, resolvedOptions] = expandReferences(
+      state,
+      sqlQuery,
+      options
+    );
+    return queryHandler(
+      connection,
+      state,
+      resolvedQuery,
+      resolvedOptions,
+      composeNextState
+    );
+  };
+}
+function insert(table, records, options = {}) {
+  return async (state) => {
+    const [resolvedTable, resolvedRecords, resolvedOptions] = expandReferences(
+      state,
+      table,
+      records,
+      options
+    );
+    const batchSize = resolvedOptions.batchSize || 1e3;
+    const recordsArray = Array.isArray(resolvedRecords) ? resolvedRecords : [resolvedRecords];
+    if (!recordsArray || recordsArray.length === 0) {
+      console.log("No records provided; skipping insert.");
+      return {
+        ...state,
+        data: { recordsInserted: 0, batches: 0 }
+      };
+    }
+    validateSqlIdentifier(resolvedTable);
+    const totalRecords = recordsArray.length;
+    console.log(
+      `Preparing to insert ${totalRecords} record${totalRecords !== 1 ? "s" : ""} into:`,
+      resolvedTable
+    );
+    const chunks = _.chunk(recordsArray, batchSize);
+    if (chunks.length > 1) {
+      console.log(
+        `Large dataset detected. Splitting into ${chunks.length} batches of up to ${batchSize} records.`
+      );
+    }
+    let currentState = state;
+    let totalInserted = 0;
+    for (let i = 0; i < chunks.length; i++) {
+      const chunk = chunks[i];
+      const batchNumber = i + 1;
+      if (chunks.length > 1) {
+        console.log(
+          `Processing batch ${batchNumber}/${chunks.length}: ${chunk.length} records`
+        );
+      }
+      const columns = Object.keys(chunk[0]);
+      const columnsList = columns.join(", ");
+      columns.forEach((col) => validateSqlIdentifier(col));
+      const valuesStrings = chunk.map((record) => {
+        const values = columns.map((key) => formatSqlValue(record[key]));
+        return `(${values.join(", ")})`;
+      });
+      const sqlQuery = `INSERT INTO ${resolvedTable} (${columnsList}) VALUES ${valuesStrings.join(
+        ", "
+      )}`;
+      currentState = await queryHandler(
+        connection,
+        currentState,
+        sqlQuery,
+        resolvedOptions,
+        composeNextState
+      );
+      totalInserted += chunk.length;
+    }
+    if (chunks.length > 1) {
+      console.log(
+        `Successfully inserted ${totalInserted} records in ${chunks.length} batches.`
+      );
+    }
+    return {
+      ...currentState,
+      data: { recordsInserted: totalInserted, batches: chunks.length }
+    };
+  };
+}
+
+// src/index.js
+var src_default = Adaptor_exports;
+export {
+  alterState,
+  arrayToString,
+  as,
+  combine,
+  cursor,
+  dataPath,
+  dataValue,
+  dateFns,
+  src_default as default,
+  each,
+  execute,
+  field,
+  fields,
+  fn,
+  fnIf,
+  group,
+  insert,
+  lastReferenceValue,
+  map,
+  merge,
+  query,
+  sourceValue,
+  util_exports as util
+};

package/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@openfn/language-motherduck",
+  "version": "1.0.0",
+  "description": "OpenFn MotherDuck cloud database adaptor",
+  "type": "module",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./types/index.d.ts",
+      "require": "./dist/index.cjs"
+    },
+    "./package.json": "./package.json"
+  },
+  "author": "Aseidas Blauvelt <ablauvelt@verasolutions.org> (Vera Solutions)",
+  "license": "LGPLv3",
+  "files": [
+    "dist/",
+    "types/",
+    "ast.json",
+    "configuration-schema.json"
+  ],
+  "dependencies": {
+    "@duckdb/node-api": "^1.3.2-alpha.25",
+    "lodash": "^4.17.21",
+    "@openfn/language-common": "3.1.1"
+  },
+  "devDependencies": {
+    "assertion-error": "2.0.0",
+    "chai": "4.3.6",
+    "deep-eql": "4.1.1",
+    "mocha": "^10.7.3",
+    "rimraf": "3.0.2",
+    "undici": "^5.22.1"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/openfn/adaptors.git"
+  },
+  "types": "types/index.d.ts",
+  "main": "dist/index.cjs",
+  "scripts": {
+    "build": "pnpm clean && build-adaptor motherduck",
+    "test": "mocha --experimental-specifier-resolution=node --no-warnings",
+    "test:watch": "mocha -w --experimental-specifier-resolution=node --no-warnings",
+    "clean": "rimraf dist types docs",
+    "pack": "pnpm pack --pack-destination ../../dist",
+    "lint": "eslint src"
+  }
+}

package/types/Adaptor.d.ts

@@ -0,0 +1,61 @@
+/**
+ * Execute a sequence of operations against MotherDuck cloud database.
+ * Wraps `@openfn/language-common/execute`, and prepends initial state for MotherDuck.
+ * @example
+ * execute(
+ *   query('SELECT * FROM my_table'),
+ *   insert('users', { name: 'John', age: 30 })
+ * )(state)
+ * @public
+ * @function
+ * @param {...Operation} operations - Operations to be performed.
+ * @returns {Operation}
+ * @state {Array} references - Array of previous operation results
+ * @state {*} data - Data from the last operation
+ */
+export function execute(...operations: Operation[]): Operation;
+/**
+ * Execute a SQL query against the MotherDuck database.
+ * @public
+ * @example <caption>Simple query</caption>
+ * query('SELECT * FROM users WHERE age > 18')
+ * @example <caption>Query with SQL logging</caption>
+ * query('SELECT * FROM orders', { writeSql: true })
+ * @function
+ * @param {string} sqlQuery - SQL query string
+ * @param {object} [options] - Query execution options
+ * @param {boolean} [options.writeSql=false] - Include full SQL in response.query (default: false, hides query for security)
+ * @returns {Operation}
+ * @state {Array} data - Query results as array of row objects
+ * @state {object} response - Metadata including rowCount, command, and query
+ */
+export function query(sqlQuery: string, options?: {
+    writeSql?: boolean;
+}): Operation;
+/**
+ * Insert one or more records into a MotherDuck table with automatic batching.
+ * Large datasets are automatically split into batches for optimal performance.
+ * @public
+ * @example <caption>Insert a single record</caption>
+ * insert('users', { name: 'John', age: 30, email: 'john@example.com' })
+ * @example <caption>Insert multiple records</caption>
+ * insert('users', [
+ *   { name: 'John', age: 30 },
+ *   { name: 'Jane', age: 25 }
+ * ])
+ * @example <caption>Insert with custom batch size</caption>
+ * insert('users', records, { batchSize: 500 })
+ * @function
+ * @param {string} table - Target table name
+ * @param {object|Array} records - A single record object or array of records
+ * @param {object} [options] - Insert options
+ * @param {number} [options.batchSize=1000] - Number of records per batch
+ * @returns {Operation}
+ * @state {object} data - Metadata including recordsInserted and batches
+ */
+export function insert(table: string, records: object | any[], options?: {
+    batchSize?: number;
+}): Operation;
+export { util };
+import * as util from "./util.js";
+export { alterState, arrayToString, combine, cursor, dataPath, dataValue, dateFns, each, field, fields, fn, fnIf, group, lastReferenceValue, map, merge, sourceValue, as } from "@openfn/language-common";

package/types/index.d.ts

@@ -0,0 +1,4 @@
+export default Adaptor;
+export * from "./Adaptor";
+export * as util from "./util.js";
+import * as Adaptor from "./Adaptor";

package/types/mock.d.ts

@@ -0,0 +1,17 @@
+export function createMockConnection(): {
+    mockInstance: {
+        connect: () => Promise<{
+            runAndReadAll: (sql: any) => Promise<{
+                getRowObjects: () => any[];
+            }>;
+            close: () => void;
+        }>;
+        close: () => void;
+    };
+    mockConnection: {
+        runAndReadAll: (sql: any) => Promise<{
+            getRowObjects: () => any[];
+        }>;
+        close: () => void;
+    };
+};

package/types/util.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Validate SQL identifier (table names, column names, etc.)
+ * @public
+ * @function
+ * @param {string} identifier - SQL identifier to validate
+ * @returns {string} - Validated identifier
+ * @throws {Error} - If identifier contains dangerous characters
+ */
+export function validateSqlIdentifier(identifier: string): string;
+/**
+ * Escape SQL string values to prevent SQL injection
+ * @public
+ * @function
+ * @param {string} value - String value to escape
+ * @returns {string} - Escaped string value
+ */
+export function escapeSqlString(value: string): string;
+/**
+ * Format a value for SQL insertion
+ * @public
+ * @function
+ * @param {any} value - Value to format
+ * @returns {string} - Formatted SQL value
+ */
+export function formatSqlValue(value: any): string;
+/**
+ * Convert BigInt and DECIMAL values to regular numbers for JSON serialization
+ * @private
+ * @function
+ * @param {any} obj - Object to convert
+ * @returns {any} - Object with BigInt and DECIMAL values converted to numbers
+ */
+export function convertBigIntToNumber(obj: any): any;
+/**
+ * Helper function to handle query execution
+ * @private
+ * @function
+ * @param {object} connection - Active database connection
+ * @param {object} state - Runtime state
+ * @param {string} sqlQuery - SQL query to execute
+ * @param {object} options - Query options
+ * @param {Function} composeNextState - State composition function
+ * @returns {Promise}
+ */
+export function queryHandler(connection: object, state: object, sqlQuery: string, options: object, composeNextState: Function): Promise<any>;