@openfn/language-motherduck 1.0.0

package/configuration-schema.json

@@ -0,0 +1,27 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "properties": {
+    "token": {
+      "title": "MotherDuck Token",
+      "type": "string",
+      "description": "MotherDuck authentication token",
+      "writeOnly": true,
+      "examples": ["token_abc123..."]
+    },
+    "database": {
+      "title": "Database Name",
+      "type": "string",
+      "description": "MotherDuck database name",
+      "examples": ["my_database", "analytics_warehouse", "production_db"]
+    },
+    "sessionHint": {
+      "title": "Session Hint",
+      "type": "string",
+      "description": "Optional session hint for MotherDuck read scaling",
+      "examples": ["primary", "readonly", "analytics"]
+    }
+  },
+  "type": "object",
+  "additionalProperties": true,
+  "required": ["token"]
+}

package/dist/index.cjs

@@ -0,0 +1,537 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.js
+var src_exports = {};
+__export(src_exports, {
+  alterState: () => import_language_common2.alterState,
+  arrayToString: () => import_language_common2.arrayToString,
+  as: () => import_language_common2.as,
+  combine: () => import_language_common2.combine,
+  cursor: () => import_language_common2.cursor,
+  dataPath: () => import_language_common2.dataPath,
+  dataValue: () => import_language_common2.dataValue,
+  dateFns: () => import_language_common2.dateFns,
+  default: () => src_default,
+  each: () => import_language_common2.each,
+  execute: () => execute,
+  field: () => import_language_common2.field,
+  fields: () => import_language_common2.fields,
+  fn: () => import_language_common2.fn,
+  fnIf: () => import_language_common2.fnIf,
+  group: () => import_language_common2.group,
+  insert: () => insert,
+  lastReferenceValue: () => import_language_common2.lastReferenceValue,
+  map: () => import_language_common2.map,
+  merge: () => import_language_common2.merge,
+  query: () => query,
+  sourceValue: () => import_language_common2.sourceValue,
+  util: () => util_exports
+});
+module.exports = __toCommonJS(src_exports);
+
+// src/Adaptor.js
+var Adaptor_exports = {};
+__export(Adaptor_exports, {
+  alterState: () => import_language_common2.alterState,
+  arrayToString: () => import_language_common2.arrayToString,
+  as: () => import_language_common2.as,
+  combine: () => import_language_common2.combine,
+  cursor: () => import_language_common2.cursor,
+  dataPath: () => import_language_common2.dataPath,
+  dataValue: () => import_language_common2.dataValue,
+  dateFns: () => import_language_common2.dateFns,
+  each: () => import_language_common2.each,
+  execute: () => execute,
+  field: () => import_language_common2.field,
+  fields: () => import_language_common2.fields,
+  fn: () => import_language_common2.fn,
+  fnIf: () => import_language_common2.fnIf,
+  group: () => import_language_common2.group,
+  insert: () => insert,
+  lastReferenceValue: () => import_language_common2.lastReferenceValue,
+  map: () => import_language_common2.map,
+  merge: () => import_language_common2.merge,
+  query: () => query,
+  sourceValue: () => import_language_common2.sourceValue,
+  util: () => util_exports
+});
+var import_language_common = require("@openfn/language-common");
+var import_util = require("@openfn/language-common/util");
+var import_node_api = require("@duckdb/node-api");
+var import_lodash = __toESM(require("lodash"), 1);
+
+// src/util.js
+var util_exports = {};
+__export(util_exports, {
+  convertBigIntToNumber: () => convertBigIntToNumber,
+  escapeSqlString: () => escapeSqlString,
+  formatSqlValue: () => formatSqlValue,
+  queryHandler: () => queryHandler,
+  validateSqlIdentifier: () => validateSqlIdentifier
+});
+function validateSqlIdentifier(identifier) {
+  if (typeof identifier !== "string") {
+    throw new Error("SQL identifier must be a string");
+  }
+  if (!/^[a-zA-Z_][a-zA-Z0-9_.]*$/.test(identifier)) {
+    throw new Error(`Invalid SQL identifier: ${identifier}. Only alphanumeric characters, underscores, and dots are allowed.`);
+  }
+  const upperIdentifier = identifier.toUpperCase();
+  const dangerousPatterns = [
+    "DROP",
+    "DELETE",
+    "INSERT",
+    "UPDATE",
+    "ALTER",
+    "CREATE",
+    "EXEC",
+    "UNION",
+    "SELECT",
+    "--",
+    "/*",
+    "*/",
+    ";"
+  ];
+  for (const pattern of dangerousPatterns) {
+    if (upperIdentifier.includes(pattern)) {
+      throw new Error(`SQL identifier contains forbidden pattern: ${pattern}`);
+    }
+  }
+  return identifier;
+}
+function escapeSqlString(value) {
+  if (typeof value !== "string") {
+    return value;
+  }
+  return value.replace(/'/g, "''");
+}
+function formatSqlValue(value) {
+  if (value === null || value === void 0) {
+    return "NULL";
+  }
+  if (typeof value === "string") {
+    return `'${escapeSqlString(value)}'`;
+  }
+  if (typeof value === "boolean") {
+    return value ? "TRUE" : "FALSE";
+  }
+  return value.toString();
+}
+function convertBigIntToNumber(obj) {
+  if (typeof obj === "bigint") {
+    if (obj <= Number.MAX_SAFE_INTEGER && obj >= Number.MIN_SAFE_INTEGER) {
+      return Number(obj);
+    } else {
+      return obj.toString();
+    }
+  }
+  if (Array.isArray(obj)) {
+    return obj.map(convertBigIntToNumber);
+  }
+  if (obj !== null && typeof obj === "object") {
+    if (obj.width !== void 0 && obj.scale !== void 0 && obj.value !== void 0) {
+      const scale = Number(obj.scale);
+      const value = typeof obj.value === "bigint" ? Number(obj.value) : obj.value;
+      return value / Math.pow(10, scale);
+    }
+    const converted = {};
+    for (const [key, value] of Object.entries(obj)) {
+      converted[key] = convertBigIntToNumber(value);
+    }
+    return converted;
+  }
+  return obj;
+}
+async function queryHandler(connection2, state, sqlQuery, options, composeNextState2) {
+  if (!connection2) {
+    throw new Error("No active MotherDuck connection found. Ensure you are running within an execute() block.");
+  }
+  try {
+    const result = await connection2.runAndReadAll(sqlQuery);
+    const rawRows = result.getRowObjects();
+    const rows = convertBigIntToNumber(rawRows);
+    const nextState = {
+      ...composeNextState2(state, rows),
+      response: {
+        rows,
+        rowCount: rows.length,
+        command: sqlQuery.trim().split(" ")[0].toUpperCase(),
+        query: options.writeSql ? sqlQuery : "[query hidden]"
+      }
+    };
+    return nextState;
+  } catch (error) {
+    const errorMessage = `MotherDuck query failed: ${error.message}`;
+    console.error(errorMessage);
+    console.error("Failed query:", sqlQuery.substring(0, 200) + (sqlQuery.length > 200 ? "..." : ""));
+    const enhancedError = new Error(errorMessage);
+    enhancedError.originalError = error;
+    enhancedError.query = sqlQuery;
+    throw enhancedError;
+  }
+}
+
+// src/mock.js
+var mockTables = {};
+function parseSelectLiterals(sql) {
+  const result = {};
+  const selectMatch = sql.match(/SELECT\s+(.+?)(?:\s+FROM|\s*$)/is);
+  if (!selectMatch)
+    return result;
+  const selectClause = selectMatch[1].trim();
+  const columns = [];
+  let current = "";
+  let inQuotes = false;
+  let parenDepth = 0;
+  for (let i = 0; i < selectClause.length; i++) {
+    const char = selectClause[i];
+    if (char === "'" && (i === 0 || selectClause[i - 1] !== "\\")) {
+      inQuotes = !inQuotes;
+    }
+    if (!inQuotes) {
+      if (char === "(")
+        parenDepth++;
+      if (char === ")")
+        parenDepth--;
+      if (char === "," && parenDepth === 0) {
+        columns.push(current.trim());
+        current = "";
+        continue;
+      }
+    }
+    current += char;
+  }
+  if (current.trim())
+    columns.push(current.trim());
+  columns.forEach((col) => {
+    var _a;
+    const stringMatch = col.match(/'([^']*)'\s+(?:as\s+)?(\w+)/i);
+    const numberMatch = col.match(/^(\d+)\s+(?:as\s+)?(\w+)/i);
+    const funcMatch = col.match(/^(\w+)\s*\([^)]*\)\s+(?:as\s+)?(\w+)/i);
+    if (stringMatch) {
+      result[stringMatch[2]] = stringMatch[1];
+    } else if (numberMatch) {
+      result[numberMatch[2]] = parseInt(numberMatch[1]);
+    } else if (funcMatch) {
+      const funcName = funcMatch[1].toLowerCase();
+      const alias = funcMatch[2];
+      if (funcName === "count") {
+        result[alias] = ((_a = mockTables[Object.keys(mockTables)[0]]) == null ? void 0 : _a.length) || 0;
+      } else if (funcName === "current_database") {
+        result[alias] = "my_db";
+      }
+    }
+  });
+  return result;
+}
+function createMockConnection() {
+  const mockConnection = {
+    runAndReadAll: async (sql) => {
+      if (/SELECT\s+FROM/i.test(sql) || /WHERE\s+INVALID/i.test(sql)) {
+        throw new Error('Parser Error: syntax error at or near "FROM"');
+      }
+      let mockData = [];
+      if (/CREATE TABLE/i.test(sql)) {
+        const tableMatch = sql.match(/CREATE TABLE\s+(\w+)/i);
+        if (tableMatch) {
+          const tableName = tableMatch[1];
+          mockTables[tableName] = [];
+        }
+        mockData = [{ success: true }];
+      } else if (/INSERT INTO/i.test(sql)) {
+        const tableMatch = sql.match(/INSERT INTO\s+(\w+)/i);
+        const valuesMatch = sql.match(/VALUES\s+(.+)/is);
+        if (tableMatch && valuesMatch) {
+          const tableName = tableMatch[1];
+          if (!mockTables[tableName])
+            mockTables[tableName] = [];
+          const columnsMatch = sql.match(/\(([^)]+)\)\s+VALUES/i);
+          const columns = columnsMatch ? columnsMatch[1].split(",").map((c) => c.trim()) : [];
+          const valuesSets = valuesMatch[1].match(/\([^)]+\)/g) || [];
+          valuesSets.forEach((valueSet) => {
+            const values = valueSet.slice(1, -1).split(",").map((v) => {
+              v = v.trim();
+              if (v.startsWith("'") && v.endsWith("'")) {
+                return v.slice(1, -1).replace(/''/g, "'");
+              }
+              if (v === "NULL")
+                return null;
+              if (v === "TRUE")
+                return true;
+              if (v === "FALSE")
+                return false;
+              if (/^\d+$/.test(v))
+                return parseInt(v);
+              return v;
+            });
+            const row = {};
+            columns.forEach((col, idx) => {
+              row[col] = values[idx];
+            });
+            mockTables[tableName].push(row);
+          });
+        }
+        mockData = [{ success: true }];
+      } else if (/SELECT/i.test(sql)) {
+        const fromMatch = sql.match(/FROM\s+(\w+)/i);
+        if (fromMatch) {
+          const tableName = fromMatch[1];
+          const tableData = mockTables[tableName] || [];
+          if (/COUNT\s*\(\s*\*\s*\)/i.test(sql)) {
+            const aliasMatch = sql.match(/COUNT\s*\(\s*\*\s*\)\s+(?:as\s+)?(\w+)/i);
+            const alias = aliasMatch ? aliasMatch[1] : "count";
+            mockData = [{ [alias]: tableData.length }];
+          } else if (/SELECT\s+\*/i.test(sql)) {
+            mockData = [...tableData];
+          } else {
+            const selectMatch = sql.match(/SELECT\s+(.+?)\s+FROM/is);
+            if (selectMatch) {
+              const columns = selectMatch[1].split(",").map((c) => c.trim());
+              mockData = tableData.map((row) => {
+                const newRow = {};
+                columns.forEach((col) => {
+                  if (row[col] !== void 0) {
+                    newRow[col] = row[col];
+                  }
+                });
+                return newRow;
+              });
+            }
+          }
+        } else {
+          const literals = parseSelectLiterals(sql);
+          if (Object.keys(literals).length === 0) {
+            mockData = [];
+          } else {
+            mockData = [literals];
+          }
+        }
+      } else if (/UPDATE/i.test(sql)) {
+        mockData = [{ success: true }];
+      } else if (/DELETE/i.test(sql)) {
+        mockData = [{ success: true }];
+      } else if (/DROP TABLE/i.test(sql)) {
+        const tableMatch = sql.match(/DROP TABLE\s+(?:IF EXISTS\s+)?(\w+)/i);
+        if (tableMatch) {
+          delete mockTables[tableMatch[1]];
+        }
+        mockData = [{ success: true }];
+      } else {
+        mockData = [{ result: "mock_data" }];
+      }
+      return {
+        getRowObjects: () => mockData
+      };
+    },
+    close: () => {
+      Object.keys(mockTables).forEach((key) => delete mockTables[key]);
+    }
+  };
+  const mockInstance = {
+    connect: async () => mockConnection,
+    close: () => {
+      Object.keys(mockTables).forEach((key) => delete mockTables[key]);
+    }
+  };
+  return { mockInstance, mockConnection };
+}
+
+// src/Adaptor.js
+var import_language_common2 = require("@openfn/language-common");
+var instance = null;
+var connection = null;
+function execute(...operations) {
+  const initialState = {
+    references: [],
+    data: null
+  };
+  return async (state) => {
+    try {
+      return await (0, import_language_common.execute)(
+        createConnection,
+        ...operations,
+        disconnect
+      )({ ...initialState, ...state });
+    } catch (error) {
+      disconnect(state);
+      throw error;
+    }
+  };
+}
+async function createConnection(state) {
+  const {
+    token,
+    database,
+    sessionHint,
+    testMode = false
+  } = state.configuration;
+  if (testMode || process.env.NODE_ENV === "test") {
+    const { mockInstance, mockConnection } = createMockConnection();
+    instance = mockInstance;
+    connection = mockConnection;
+    return state;
+  }
+  if (!token) {
+    throw new Error(
+      "MotherDuck token is required. Please provide a token in the configuration."
+    );
+  }
+  let databasePath = `md:${database}`;
+  if (sessionHint) {
+    databasePath += `?session_hint=${sessionHint}`;
+  }
+  const config = {
+    motherduck_token: token
+  };
+  console.log(`Connecting to MotherDuck cloud database: ${database}`);
+  instance = await import_node_api.DuckDBInstance.create(databasePath, config);
+  connection = await instance.connect();
+  console.log("Connected successfully to MotherDuck");
+  return state;
+}
+function disconnect(state) {
+  if (connection && typeof connection.close === "function") {
+    connection.close();
+  }
+  if (instance && typeof instance.close === "function") {
+    instance.close();
+  }
+  connection = null;
+  instance = null;
+  return state;
+}
+function query(sqlQuery, options = {}) {
+  return (state) => {
+    const [resolvedQuery, resolvedOptions] = (0, import_util.expandReferences)(
+      state,
+      sqlQuery,
+      options
+    );
+    return queryHandler(
+      connection,
+      state,
+      resolvedQuery,
+      resolvedOptions,
+      import_language_common.composeNextState
+    );
+  };
+}
+function insert(table, records, options = {}) {
+  return async (state) => {
+    const [resolvedTable, resolvedRecords, resolvedOptions] = (0, import_util.expandReferences)(
+      state,
+      table,
+      records,
+      options
+    );
+    const batchSize = resolvedOptions.batchSize || 1e3;
+    const recordsArray = Array.isArray(resolvedRecords) ? resolvedRecords : [resolvedRecords];
+    if (!recordsArray || recordsArray.length === 0) {
+      console.log("No records provided; skipping insert.");
+      return {
+        ...state,
+        data: { recordsInserted: 0, batches: 0 }
+      };
+    }
+    validateSqlIdentifier(resolvedTable);
+    const totalRecords = recordsArray.length;
+    console.log(
+      `Preparing to insert ${totalRecords} record${totalRecords !== 1 ? "s" : ""} into:`,
+      resolvedTable
+    );
+    const chunks = import_lodash.default.chunk(recordsArray, batchSize);
+    if (chunks.length > 1) {
+      console.log(
+        `Large dataset detected. Splitting into ${chunks.length} batches of up to ${batchSize} records.`
+      );
+    }
+    let currentState = state;
+    let totalInserted = 0;
+    for (let i = 0; i < chunks.length; i++) {
+      const chunk = chunks[i];
+      const batchNumber = i + 1;
+      if (chunks.length > 1) {
+        console.log(
+          `Processing batch ${batchNumber}/${chunks.length}: ${chunk.length} records`
+        );
+      }
+      const columns = Object.keys(chunk[0]);
+      const columnsList = columns.join(", ");
+      columns.forEach((col) => validateSqlIdentifier(col));
+      const valuesStrings = chunk.map((record) => {
+        const values = columns.map((key) => formatSqlValue(record[key]));
+        return `(${values.join(", ")})`;
+      });
+      const sqlQuery = `INSERT INTO ${resolvedTable} (${columnsList}) VALUES ${valuesStrings.join(
+        ", "
+      )}`;
+      currentState = await queryHandler(
+        connection,
+        currentState,
+        sqlQuery,
+        resolvedOptions,
+        import_language_common.composeNextState
+      );
+      totalInserted += chunk.length;
+    }
+    if (chunks.length > 1) {
+      console.log(
+        `Successfully inserted ${totalInserted} records in ${chunks.length} batches.`
+      );
+    }
+    return {
+      ...currentState,
+      data: { recordsInserted: totalInserted, batches: chunks.length }
+    };
+  };
+}
+
+// src/index.js
+var src_default = Adaptor_exports;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  alterState,
+  arrayToString,
+  as,
+  combine,
+  cursor,
+  dataPath,
+  dataValue,
+  dateFns,
+  each,
+  execute,
+  field,
+  fields,
+  fn,
+  fnIf,
+  group,
+  insert,
+  lastReferenceValue,
+  map,
+  merge,
+  query,
+  sourceValue,
+  util
+});