@devrev/meerkat-node 0.0.88 → 0.0.90

package/package.json

@@ -1,10 +1,12 @@
 {
   "name": "@devrev/meerkat-node",
-  "version": "0.0.88",
+  "version": "0.0.90",
   "dependencies": {
     "@swc/helpers": "~0.5.0",
     "@devrev/meerkat-core": "*",
-    "duckdb": "^0.10.2"
+    "axios": "^1.6.0",
+    "lodash": "^4.17.21",
+    "duckdb": "^1.0.0"
   },
   "scripts": {
     "release": "semantic-release"

package/src/cube-to-sql/cube-to-sql.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../meerkat-node/src/cube-to-sql/cube-to-sql.ts"],"sourcesContent":["import {\n  BASE_TABLE_NAME,\n  ContextParams,\n  Query,\n  TableSchema,\n  applyFilterParamsToBaseSQL,\n  applyProjectionToSQLQuery,\n  astDeserializerQuery,\n  cubeToDuckdbAST,\n  deserializeQuery,\n  detectApplyContextParamsToBaseSQL,\n  getCombinedTableSchema,\n  getFilterParamsSQL,\n  getFinalBaseSQL\n} from '@devrev/meerkat-core';\nimport { duckdbExec } from '../duckdb-exec';\n\n\n\ninterface CubeQueryToSQLParams {\n  query: Query,\n  tableSchemas: TableSchema[],\n  contextParams?: ContextParams\n}\n\nexport const cubeQueryToSQL = async ({\n  query,\n  tableSchemas,\n  contextParams,\n}: CubeQueryToSQLParams) => {\n  const updatedTableSchemas: TableSchema[] = await Promise.all(\n    tableSchemas.map(async (schema: TableSchema) => {\n      const baseFilterParamsSQL = await getFinalBaseSQL({query, tableSchema: schema, getQueryOutput: duckdbExec });\n      return {\n        ...schema,\n        sql: baseFilterParamsSQL,\n      };\n    })\n  );\n\n  const updatedTableSchema = await getCombinedTableSchema(\n    updatedTableSchemas,\n    query\n  );\n\n  const ast = cubeToDuckdbAST(query, updatedTableSchema);\n  if (!ast) {\n    throw new Error('Could not generate AST');\n  }\n\n  const queryTemp = astDeserializerQuery(ast);\n\n  const queryOutput = await duckdbExec<\n    {\n      [key: string]: string;\n    }[]\n  >(queryTemp);\n  const preBaseQuery = deserializeQuery(queryOutput);\n\n  const filterParamsSQL = await getFilterParamsSQL({\n    query,\n    tableSchema: updatedTableSchema,\n    getQueryOutput: duckdbExec,\n  });\n\n  const filterParamQuery = applyFilterParamsToBaseSQL(\n    updatedTableSchema.sql,\n    filterParamsSQL\n  );\n\n  /**\n   * Replace CONTEXT_PARAMS with context params\n   */\n  const baseQuery = detectApplyContextParamsToBaseSQL(\n    filterParamQuery,\n    contextParams || {}\n  );\n\n  /**\n   * Replace BASE_TABLE_NAME with cube query\n   */\n  const replaceBaseTableName = preBaseQuery.replace(\n    BASE_TABLE_NAME,\n    `(${baseQuery}) AS ${updatedTableSchema.name}`\n  );\n\n  /**\n   * Add measures to the query\n   */\n  const measures = query.measures;\n  const dimensions = query.dimensions || [];\n  const finalQuery = applyProjectionToSQLQuery(\n    dimensions,\n    measures,\n    updatedTableSchema,\n    replaceBaseTableName\n  );\n\n  return finalQuery;\n};\n"],"names":["cubeQueryToSQL","query","tableSchemas","contextParams","updatedTableSchemas","Promise","all","map","schema","baseFilterParamsSQL","getFinalBaseSQL","tableSchema","getQueryOutput","duckdbExec","sql","updatedTableSchema","getCombinedTableSchema","ast","cubeToDuckdbAST","Error","queryTemp","astDeserializerQuery","queryOutput","preBaseQuery","deserializeQuery","filterParamsSQL","getFilterParamsSQL","filterParamQuery","applyFilterParamsToBaseSQL","baseQuery","detectApplyContextParamsToBaseSQL","replaceBaseTableName","replace","BASE_TABLE_NAME","name","measures","dimensions","finalQuery","applyProjectionToSQLQuery"],"mappings":";+BAyBaA;;;eAAAA;;;;6BAXN;4BACoB;AAUpB,MAAMA,iBAAiB,OAAO,EACnCC,KAAK,EACLC,YAAY,EACZC,aAAa,EACQ;IACrB,MAAMC,sBAAqC,MAAMC,QAAQC,GAAG,CAC1DJ,aAAaK,GAAG,CAAC,OAAOC;QACtB,MAAMC,sBAAsB,MAAMC,IAAAA,4BAAe,EAAC;YAACT;YAAOU,aAAaH;YAAQI,gBAAgBC,sBAAU;QAAC;QAC1G,OAAO,eACFL;YACHM,KAAKL;;IAET;IAGF,MAAMM,qBAAqB,MAAMC,IAAAA,mCAAsB,EACrDZ,qBACAH;IAGF,MAAMgB,MAAMC,IAAAA,4BAAe,EAACjB,OAAOc;IACnC,IAAI,CAACE,KAAK;QACR,MAAM,IAAIE,MAAM;IAClB;IAEA,MAAMC,YAAYC,IAAAA,iCAAoB,EAACJ;IAEvC,MAAMK,cAAc,MAAMT,IAAAA,sBAAU,EAIlCO;IACF,MAAMG,eAAeC,IAAAA,6BAAgB,EAACF;IAEtC,MAAMG,kBAAkB,MAAMC,IAAAA,+BAAkB,EAAC;QAC/CzB;QACAU,aAAaI;QACbH,gBAAgBC,sBAAU;IAC5B;IAEA,MAAMc,mBAAmBC,IAAAA,uCAA0B,EACjDb,mBAAmBD,GAAG,EACtBW;IAGF;;GAEC,GACD,MAAMI,YAAYC,IAAAA,8CAAiC,EACjDH,kBACAxB,iBAAiB,CAAC;IAGpB;;GAEC,GACD,MAAM4B,uBAAuBR,aAAaS,OAAO,CAC/CC,4BAAe,EACf,CAAC,CAAC,EAAEJ,UAAU,KAAK,EAAEd,mBAAmBmB,IAAI,CAAC,CAAC;IAGhD;;GAEC,GACD,MAAMC,WAAWlC,MAAMkC,QAAQ;IAC/B,MAAMC,aAAanC,MAAMmC,UAAU,IAAI,EAAE;IACzC,MAAMC,aAAaC,IAAAA,sCAAyB,EAC1CF,YACAD,UACApB,oBACAgB;IAGF,OAAOM;AACT"}
+{"version":3,"sources":["../../../../meerkat-node/src/cube-to-sql/cube-to-sql.ts"],"sourcesContent":["import {\n  BASE_TABLE_NAME,\n  ContextParams,\n  Query,\n  TableSchema,\n  applyFilterParamsToBaseSQL,\n  applyProjectionToSQLQuery,\n  astDeserializerQuery,\n  cubeToDuckdbAST,\n  deserializeQuery,\n  detectApplyContextParamsToBaseSQL,\n  getCombinedTableSchema,\n  getFilterParamsSQL,\n  getFinalBaseSQL,\n} from '@devrev/meerkat-core';\nimport { duckdbExec } from '../duckdb-exec';\n\ninterface CubeQueryToSQLParams {\n  query: Query;\n  tableSchemas: TableSchema[];\n  contextParams?: ContextParams;\n}\n\nexport const cubeQueryToSQL = async ({\n  query,\n  tableSchemas,\n  contextParams,\n}: CubeQueryToSQLParams) => {\n  const updatedTableSchemas: TableSchema[] = await Promise.all(\n    tableSchemas.map(async (schema: TableSchema) => {\n      const baseFilterParamsSQL = await getFinalBaseSQL({\n        query,\n        tableSchema: schema,\n        getQueryOutput: duckdbExec,\n      });\n      return {\n        ...schema,\n        sql: baseFilterParamsSQL,\n      };\n    })\n  );\n\n  const updatedTableSchema = await getCombinedTableSchema(\n    updatedTableSchemas,\n    query\n  );\n\n  const ast = cubeToDuckdbAST(query, updatedTableSchema);\n  if (!ast) {\n    throw new Error('Could not generate AST');\n  }\n\n  const queryTemp = astDeserializerQuery(ast);\n\n  const queryOutput = (await duckdbExec(queryTemp)) as Record<string, string>[];\n  const preBaseQuery = deserializeQuery(queryOutput);\n\n  const filterParamsSQL = await getFilterParamsSQL({\n    query,\n    tableSchema: updatedTableSchema,\n    getQueryOutput: duckdbExec,\n  });\n\n  const filterParamQuery = applyFilterParamsToBaseSQL(\n    updatedTableSchema.sql,\n    filterParamsSQL\n  );\n\n  /**\n   * Replace CONTEXT_PARAMS with context params\n   */\n  const baseQuery = detectApplyContextParamsToBaseSQL(\n    filterParamQuery,\n    contextParams || {}\n  );\n\n  /**\n   * Replace BASE_TABLE_NAME with cube query\n   */\n  const replaceBaseTableName = preBaseQuery.replace(\n    BASE_TABLE_NAME,\n    `(${baseQuery}) AS ${updatedTableSchema.name}`\n  );\n\n  /**\n   * Add measures to the query\n   */\n  const measures = query.measures;\n  const dimensions = query.dimensions || [];\n  const finalQuery = applyProjectionToSQLQuery(\n    dimensions,\n    measures,\n    updatedTableSchema,\n    replaceBaseTableName\n  );\n\n  return finalQuery;\n};\n"],"names":["cubeQueryToSQL","query","tableSchemas","contextParams","updatedTableSchemas","Promise","all","map","schema","baseFilterParamsSQL","getFinalBaseSQL","tableSchema","getQueryOutput","duckdbExec","sql","updatedTableSchema","getCombinedTableSchema","ast","cubeToDuckdbAST","Error","queryTemp","astDeserializerQuery","queryOutput","preBaseQuery","deserializeQuery","filterParamsSQL","getFilterParamsSQL","filterParamQuery","applyFilterParamsToBaseSQL","baseQuery","detectApplyContextParamsToBaseSQL","replaceBaseTableName","replace","BASE_TABLE_NAME","name","measures","dimensions","finalQuery","applyProjectionToSQLQuery"],"mappings":";+BAuBaA;;;eAAAA;;;;6BATN;4BACoB;AAQpB,MAAMA,iBAAiB,OAAO,EACnCC,KAAK,EACLC,YAAY,EACZC,aAAa,EACQ;IACrB,MAAMC,sBAAqC,MAAMC,QAAQC,GAAG,CAC1DJ,aAAaK,GAAG,CAAC,OAAOC;QACtB,MAAMC,sBAAsB,MAAMC,IAAAA,4BAAe,EAAC;YAChDT;YACAU,aAAaH;YACbI,gBAAgBC,sBAAU;QAC5B;QACA,OAAO,eACFL;YACHM,KAAKL;;IAET;IAGF,MAAMM,qBAAqB,MAAMC,IAAAA,mCAAsB,EACrDZ,qBACAH;IAGF,MAAMgB,MAAMC,IAAAA,4BAAe,EAACjB,OAAOc;IACnC,IAAI,CAACE,KAAK;QACR,MAAM,IAAIE,MAAM;IAClB;IAEA,MAAMC,YAAYC,IAAAA,iCAAoB,EAACJ;IAEvC,MAAMK,cAAe,MAAMT,IAAAA,sBAAU,EAACO;IACtC,MAAMG,eAAeC,IAAAA,6BAAgB,EAACF;IAEtC,MAAMG,kBAAkB,MAAMC,IAAAA,+BAAkB,EAAC;QAC/CzB;QACAU,aAAaI;QACbH,gBAAgBC,sBAAU;IAC5B;IAEA,MAAMc,mBAAmBC,IAAAA,uCAA0B,EACjDb,mBAAmBD,GAAG,EACtBW;IAGF;;GAEC,GACD,MAAMI,YAAYC,IAAAA,8CAAiC,EACjDH,kBACAxB,iBAAiB,CAAC;IAGpB;;GAEC,GACD,MAAM4B,uBAAuBR,aAAaS,OAAO,CAC/CC,4BAAe,EACf,CAAC,CAAC,EAAEJ,UAAU,KAAK,EAAEd,mBAAmBmB,IAAI,CAAC,CAAC;IAGhD;;GAEC,GACD,MAAMC,WAAWlC,MAAMkC,QAAQ;IAC/B,MAAMC,aAAanC,MAAMmC,UAAU,IAAI,EAAE;IACzC,MAAMC,aAAaC,IAAAA,sCAAyB,EAC1CF,YACAD,UACApB,oBACAgB;IAGF,OAAOM;AACT"}

package/src/duckdb-manager/duckdb-manager.d.ts

@@ -0,0 +1,28 @@
+import { ColumnInfo, Connection, Database, TableData } from 'duckdb';
+export declare class DuckDBManager {
+    private db;
+    private connection;
+    private initPromise;
+    constructor({ onInitialize, }: {
+        onInitialize?: (db: Database) => Promise<void>;
+    });
+    /**
+     * Initialize the DuckDB instance
+     */
+    private initialize;
+    /**
+     * Get a DuckDB connection instance.
+     */
+    getConnection(): Promise<Connection | null>;
+    /**
+     * Execute a query on the DuckDB connection.
+     */
+    query(query: string): Promise<{
+        columns: ColumnInfo[];
+        data: TableData;
+    }>;
+    /**
+     * Close the DuckDB connection and cleanup resources.
+     */
+    close(): Promise<void>;
+}

package/src/duckdb-manager/duckdb-manager.js

@@ -0,0 +1,71 @@
+"use strict";
+Object.defineProperty(exports, "DuckDBManager", {
+    enumerable: true,
+    get: function() {
+        return DuckDBManager;
+    }
+});
+const _duckdbsingleton = require("../duckdb-singleton");
+const _duckdbtypeconvertor = require("../utils/duckdb-type-convertor");
+let DuckDBManager = class DuckDBManager {
+    /**
+   * Initialize the DuckDB instance
+   */ async initialize({ onInitialize }) {
+        this.db = _duckdbsingleton.DuckDBSingleton.getInstance();
+        await (onInitialize == null ? void 0 : onInitialize(this.db));
+    }
+    /**
+   * Get a DuckDB connection instance.
+   */ async getConnection() {
+        // Ensure database is initialized before returning the connection
+        await this.initPromise;
+        if (!this.connection) {
+            var _this_db;
+            var _this_db_connect;
+            this.connection = (_this_db_connect = (_this_db = this.db) == null ? void 0 : _this_db.connect()) != null ? _this_db_connect : null;
+        }
+        return this.connection;
+    }
+    /**
+   * Execute a query on the DuckDB connection.
+   */ async query(query) {
+        const connection = await this.getConnection();
+        return new Promise((resolve, reject)=>{
+            connection == null ? void 0 : connection.prepare(query, (err, statement)=>{
+                if (err) {
+                    reject(new Error(`Query preparation failed: ${err.message}`));
+                    return;
+                }
+                const columns = statement.columns();
+                statement.all((err, data)=>{
+                    if (err) {
+                        reject(new Error(`Query execution failed: ${err.message}`));
+                        return;
+                    }
+                    const result = (0, _duckdbtypeconvertor.convertTableDataToJSON)(data, columns);
+                    resolve({
+                        columns,
+                        data: result
+                    });
+                });
+            });
+        });
+    }
+    /**
+   * Close the DuckDB connection and cleanup resources.
+   */ async close() {
+        if (this.connection) {
+            this.connection.close();
+            this.connection = null;
+        }
+    }
+    constructor({ onInitialize }){
+        this.db = null;
+        this.connection = null;
+        this.initPromise = this.initialize({
+            onInitialize
+        });
+    }
+};
+
+//# sourceMappingURL=duckdb-manager.js.map

package/src/duckdb-manager/duckdb-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../meerkat-node/src/duckdb-manager/duckdb-manager.ts"],"sourcesContent":["import { ColumnInfo, Connection, Database, TableData } from 'duckdb';\n\nimport { DuckDBSingleton } from '../duckdb-singleton';\nimport { convertTableDataToJSON } from '../utils/duckdb-type-convertor';\n\nexport class DuckDBManager {\n  private db: Database | null = null;\n  private connection: Connection | null = null;\n\n  private initPromise: Promise<void>;\n\n  constructor({\n    onInitialize,\n  }: {\n    onInitialize?: (db: Database) => Promise<void>;\n  }) {\n    this.initPromise = this.initialize({ onInitialize });\n  }\n\n  /**\n   * Initialize the DuckDB instance\n   */\n  private async initialize({\n    onInitialize,\n  }: {\n    onInitialize?: (db: Database) => Promise<void>;\n  }) {\n    this.db = DuckDBSingleton.getInstance();\n\n    await onInitialize?.(this.db);\n  }\n\n  /**\n   * Get a DuckDB connection instance.\n   */\n  async getConnection() {\n    // Ensure database is initialized before returning the connection\n    await this.initPromise;\n\n    if (!this.connection) {\n      this.connection = this.db?.connect() ?? null;\n    }\n\n    return this.connection;\n  }\n\n  /**\n   * Execute a query on the DuckDB connection.\n   */\n  async query(\n    query: string\n  ): Promise<{ columns: ColumnInfo[]; data: TableData }> {\n    const connection = await this.getConnection();\n\n    return new Promise((resolve, reject) => {\n      connection?.prepare(query, (err, statement) => {\n        if (err) {\n          reject(new Error(`Query preparation failed: ${err.message}`));\n          return;\n        }\n\n        const columns = statement.columns();\n\n        statement.all((err, data) => {\n          if (err) {\n            reject(new Error(`Query execution failed: ${err.message}`));\n            return;\n          }\n\n          const result = convertTableDataToJSON(data, columns);\n\n          resolve({ columns, data: result });\n        });\n      });\n    });\n  }\n\n  /**\n   * Close the DuckDB connection and cleanup resources.\n   */\n  async close(): Promise<void> {\n    if (this.connection) {\n      this.connection.close();\n      this.connection = null;\n    }\n  }\n}\n"],"names":["DuckDBManager","initialize","onInitialize","db","DuckDBSingleton","getInstance","getConnection","initPromise","connection","connect","query","Promise","resolve","reject","prepare","err","statement","Error","message","columns","all","data","result","convertTableDataToJSON","close","constructor"],"mappings":";+BAKaA;;;eAAAA;;;iCAHmB;qCACO;AAEhC,IAAA,AAAMA,gBAAN,MAAMA;IAcX;;GAEC,GACD,MAAcC,WAAW,EACvBC,YAAY,EAGb,EAAE;QACD,IAAI,CAACC,EAAE,GAAGC,gCAAe,CAACC,WAAW;QAErC,OAAMH,gCAAAA,aAAe,IAAI,CAACC,EAAE;IAC9B;IAEA;;GAEC,GACD,MAAMG,gBAAgB;QACpB,iEAAiE;QACjE,MAAM,IAAI,CAACC,WAAW;QAEtB,IAAI,CAAC,IAAI,CAACC,UAAU,EAAE;gBACF;gBAAA;YAAlB,IAAI,CAACA,UAAU,GAAG,CAAA,oBAAA,WAAA,IAAI,CAACL,EAAE,qBAAP,SAASM,OAAO,cAAhB,mBAAsB;QAC1C;QAEA,OAAO,IAAI,CAACD,UAAU;IACxB;IAEA;;GAEC,GACD,MAAME,MACJA,KAAa,EACwC;QACrD,MAAMF,aAAa,MAAM,IAAI,CAACF,aAAa;QAE3C,OAAO,IAAIK,QAAQ,CAACC,SAASC;YAC3BL,8BAAAA,WAAYM,OAAO,CAACJ,OAAO,CAACK,KAAKC;gBAC/B,IAAID,KAAK;oBACPF,OAAO,IAAII,MAAM,CAAC,0BAA0B,EAAEF,IAAIG,OAAO,CAAC,CAAC;oBAC3D;gBACF;gBAEA,MAAMC,UAAUH,UAAUG,OAAO;gBAEjCH,UAAUI,GAAG,CAAC,CAACL,KAAKM;oBAClB,IAAIN,KAAK;wBACPF,OAAO,IAAII,MAAM,CAAC,wBAAwB,EAAEF,IAAIG,OAAO,CAAC,CAAC;wBACzD;oBACF;oBAEA,MAAMI,SAASC,IAAAA,2CAAsB,EAACF,MAAMF;oBAE5CP,QAAQ;wBAAEO;wBAASE,MAAMC;oBAAO;gBAClC;YACF;QACF;IACF;IAEA;;GAEC,GACD,MAAME,QAAuB;QAC3B,IAAI,IAAI,CAAChB,UAAU,EAAE;YACnB,IAAI,CAACA,UAAU,CAACgB,KAAK;YACrB,IAAI,CAAChB,UAAU,GAAG;QACpB;IACF;IA1EAiB,YAAY,EACVvB,YAAY,EAGb,CAAE;aATKC,KAAsB;aACtBK,aAAgC;QAStC,IAAI,CAACD,WAAW,GAAG,IAAI,CAACN,UAAU,CAAC;YAAEC;QAAa;IACpD;AAqEF"}

package/src/file-manager/file-manager.d.ts

@@ -0,0 +1,33 @@
+export declare class FileManager {
+    private readonly baseDir;
+    constructor(config: {
+        baseDir?: string;
+    });
+    getPath(tableName: string, fileName?: string): string;
+    private getHashedFileName;
+    /**
+     * Write a file buffer to the file system.
+     */
+    writeFileBuffer(file: {
+        tableName: string;
+        fileName: string;
+        buffer: Uint8Array;
+    }): Promise<void>;
+    /**
+     * Get the file paths for a table.
+     */
+    getTableFilePaths(tableName: string): Promise<string[]>;
+    /**
+     * Delete files from a table.
+     */
+    deleteTableFiles(tableName: string, files: string[]): Promise<void>;
+    /**
+     * Stream and register a file from a URL.
+     */
+    streamAndRegisterFile({ tableName, url, headers, fileName, }: {
+        tableName: string;
+        url: string;
+        headers: Record<string, string>;
+        fileName: string;
+    }): Promise<void>;
+}

package/src/file-manager/file-manager.js

@@ -0,0 +1,93 @@
+"use strict";
+Object.defineProperty(exports, "FileManager", {
+    enumerable: true,
+    get: function() {
+        return FileManager;
+    }
+});
+const _extends = require("@swc/helpers/_/_extends");
+const _axios = require("axios");
+const _fs = require("fs");
+const _path = require("path");
+const _hashstring = require("../utils/hash-string");
+let FileManager = class FileManager {
+    getPath(tableName, fileName) {
+        return fileName ? _path.join(this.baseDir, tableName, fileName) : _path.join(this.baseDir, tableName);
+    }
+    getHashedFileName(fileName) {
+        return (0, _hashstring.hashString)(fileName);
+    }
+    /**
+   * Write a file buffer to the file system.
+   */ async writeFileBuffer(file) {
+        // Hash the file name to avoid file name length issues
+        const hashedFileName = this.getHashedFileName(file.fileName);
+        const filePath = this.getPath(file.tableName, hashedFileName);
+        await _fs.promises.mkdir(_path.dirname(filePath), {
+            recursive: true
+        });
+        await _fs.promises.writeFile(filePath, file.buffer);
+    }
+    /**
+   * Get the file paths for a table.
+   */ async getTableFilePaths(tableName) {
+        try {
+            const files = await _fs.promises.readdir(this.getPath(tableName));
+            return files.map((file)=>this.getPath(tableName, file));
+        } catch (e) {
+            return [];
+        }
+    }
+    /**
+   * Delete files from a table.
+   */ async deleteTableFiles(tableName, files) {
+        await Promise.all(files.map(async (file)=>{
+            try {
+                await _fs.promises.unlink(this.getPath(tableName, this.getHashedFileName(file)));
+            } catch (err) {
+                console.error(err);
+            }
+        }));
+    }
+    /**
+   * Stream and register a file from a URL.
+   */ async streamAndRegisterFile({ tableName, url, headers, fileName }) {
+        try {
+            const response = await (0, _axios.default)({
+                headers: _extends._({}, headers),
+                method: 'get',
+                responseType: 'stream',
+                url
+            });
+            const hashedFileName = (0, _hashstring.hashString)(fileName);
+            const filePath = this.getPath(tableName, hashedFileName);
+            await _fs.promises.mkdir(_path.dirname(filePath), {
+                recursive: true
+            });
+            const writer = (0, _fs.createWriteStream)(filePath);
+            return new Promise((resolve, reject)=>{
+                response.data.pipe(writer);
+                writer.on('finish', ()=>{
+                    writer.close();
+                    resolve();
+                });
+                writer.on('error', (err)=>{
+                    writer.close();
+                    reject(err);
+                });
+            });
+        } catch (error) {
+            console.error('Error streaming file:', error);
+            throw error;
+        }
+    }
+    constructor(config){
+        var _config_baseDir;
+        this.baseDir = (_config_baseDir = config.baseDir) != null ? _config_baseDir : './data';
+        _fs.promises.mkdir(this.baseDir, {
+            recursive: true
+        }).catch(console.error);
+    }
+};
+
+//# sourceMappingURL=file-manager.js.map

package/src/file-manager/file-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../meerkat-node/src/file-manager/file-manager.ts"],"sourcesContent":["import axios from 'axios';\nimport { createWriteStream, promises as fs } from 'fs';\nimport * as path from 'path';\n\nimport { hashString } from '../utils/hash-string';\n\nexport class FileManager {\n  private readonly baseDir: string;\n\n  constructor(config: { baseDir?: string }) {\n    this.baseDir = config.baseDir ?? './data';\n    fs.mkdir(this.baseDir, { recursive: true }).catch(console.error);\n  }\n\n  public getPath(tableName: string, fileName?: string): string {\n    return fileName\n      ? path.join(this.baseDir, tableName, fileName)\n      : path.join(this.baseDir, tableName);\n  }\n\n  private getHashedFileName(fileName: string): string {\n    return hashString(fileName);\n  }\n\n  /**\n   * Write a file buffer to the file system.\n   */\n  async writeFileBuffer(file: {\n    tableName: string;\n    fileName: string;\n    buffer: Uint8Array;\n  }): Promise<void> {\n    // Hash the file name to avoid file name length issues\n    const hashedFileName = this.getHashedFileName(file.fileName);\n\n    const filePath = this.getPath(file.tableName, hashedFileName);\n\n    await fs.mkdir(path.dirname(filePath), { recursive: true });\n\n    await fs.writeFile(filePath, file.buffer);\n  }\n\n  /**\n   * Get the file paths for a table.\n   */\n  async getTableFilePaths(tableName: string): Promise<string[]> {\n    try {\n      const files = await fs.readdir(this.getPath(tableName));\n\n      return files.map((file) => this.getPath(tableName, file));\n    } catch {\n      return [];\n    }\n  }\n\n  /**\n   * Delete files from a table.\n   */\n  async deleteTableFiles(tableName: string, files: string[]): Promise<void> {\n    await Promise.all(\n      files.map(async (file) => {\n        try {\n          await fs.unlink(\n            this.getPath(tableName, this.getHashedFileName(file))\n          );\n        } catch (err) {\n          console.error(err);\n        }\n      })\n    );\n  }\n\n  /**\n   * Stream and register a file from a URL.\n   */\n  async streamAndRegisterFile({\n    tableName,\n    url,\n    headers,\n    fileName,\n  }: {\n    tableName: string;\n    url: string;\n    headers: Record<string, string>;\n    fileName: string;\n  }): Promise<void> {\n    try {\n      const response = await axios({\n        headers: {\n          ...headers,\n        },\n        method: 'get',\n        responseType: 'stream',\n        url,\n      });\n\n      const hashedFileName = hashString(fileName);\n\n      const filePath = this.getPath(tableName, hashedFileName);\n      await fs.mkdir(path.dirname(filePath), { recursive: true });\n\n      const writer = createWriteStream(filePath);\n\n      return new Promise((resolve, reject) => {\n        response.data.pipe(writer);\n        writer.on('finish', () => {\n          writer.close();\n          resolve();\n        });\n        writer.on('error', (err) => {\n          writer.close();\n          reject(err);\n        });\n      });\n    } catch (error) {\n      console.error('Error streaming file:', error);\n      throw error;\n    }\n  }\n}\n"],"names":["FileManager","getPath","tableName","fileName","path","join","baseDir","getHashedFileName","hashString","writeFileBuffer","file","hashedFileName","filePath","fs","mkdir","dirname","recursive","writeFile","buffer","getTableFilePaths","files","readdir","map","deleteTableFiles","Promise","all","unlink","err","console","error","streamAndRegisterFile","url","headers","response","axios","method","responseType","writer","createWriteStream","resolve","reject","data","pipe","on","close","constructor","config","catch"],"mappings":";+BAMaA;;;eAAAA;;;;uBANK;oBACgC;sBAC5B;4BAEK;AAEpB,IAAA,AAAMA,cAAN,MAAMA;IAQJC,QAAQC,SAAiB,EAAEC,QAAiB,EAAU;QAC3D,OAAOA,WACHC,MAAKC,IAAI,CAAC,IAAI,CAACC,OAAO,EAAEJ,WAAWC,YACnCC,MAAKC,IAAI,CAAC,IAAI,CAACC,OAAO,EAAEJ;IAC9B;IAEQK,kBAAkBJ,QAAgB,EAAU;QAClD,OAAOK,IAAAA,sBAAU,EAACL;IACpB;IAEA;;GAEC,GACD,MAAMM,gBAAgBC,IAIrB,EAAiB;QAChB,sDAAsD;QACtD,MAAMC,iBAAiB,IAAI,CAACJ,iBAAiB,CAACG,KAAKP,QAAQ;QAE3D,MAAMS,WAAW,IAAI,CAACX,OAAO,CAACS,KAAKR,SAAS,EAAES;QAE9C,MAAME,YAAE,CAACC,KAAK,CAACV,MAAKW,OAAO,CAACH,WAAW;YAAEI,WAAW;QAAK;QAEzD,MAAMH,YAAE,CAACI,SAAS,CAACL,UAAUF,KAAKQ,MAAM;IAC1C;IAEA;;GAEC,GACD,MAAMC,kBAAkBjB,SAAiB,EAAqB;QAC5D,IAAI;YACF,MAAMkB,QAAQ,MAAMP,YAAE,CAACQ,OAAO,CAAC,IAAI,CAACpB,OAAO,CAACC;YAE5C,OAAOkB,MAAME,GAAG,CAAC,CAACZ,OAAS,IAAI,CAACT,OAAO,CAACC,WAAWQ;QACrD,EAAE,UAAM;YACN,OAAO,EAAE;QACX;IACF;IAEA;;GAEC,GACD,MAAMa,iBAAiBrB,SAAiB,EAAEkB,KAAe,EAAiB;QACxE,MAAMI,QAAQC,GAAG,CACfL,MAAME,GAAG,CAAC,OAAOZ;YACf,IAAI;gBACF,MAAMG,YAAE,CAACa,MAAM,CACb,IAAI,CAACzB,OAAO,CAACC,WAAW,IAAI,CAACK,iBAAiB,CAACG;YAEnD,EAAE,OAAOiB,KAAK;gBACZC,QAAQC,KAAK,CAACF;YAChB;QACF;IAEJ;IAEA;;GAEC,GACD,MAAMG,sBAAsB,EAC1B5B,SAAS,EACT6B,GAAG,EACHC,OAAO,EACP7B,QAAQ,EAMT,EAAiB;QAChB,IAAI;YACF,MAAM8B,WAAW,MAAMC,IAAAA,cAAK,EAAC;gBAC3BF,SAAS,eACJA;gBAELG,QAAQ;gBACRC,cAAc;gBACdL;YACF;YAEA,MAAMpB,iBAAiBH,IAAAA,sBAAU,EAACL;YAElC,MAAMS,WAAW,IAAI,CAACX,OAAO,CAACC,WAAWS;YACzC,MAAME,YAAE,CAACC,KAAK,CAACV,MAAKW,OAAO,CAACH,WAAW;gBAAEI,WAAW;YAAK;YAEzD,MAAMqB,SAASC,IAAAA,qBAAiB,EAAC1B;YAEjC,OAAO,IAAIY,QAAQ,CAACe,SAASC;gBAC3BP,SAASQ,IAAI,CAACC,IAAI,CAACL;gBACnBA,OAAOM,EAAE,CAAC,UAAU;oBAClBN,OAAOO,KAAK;oBACZL;gBACF;gBACAF,OAAOM,EAAE,CAAC,SAAS,CAAChB;oBAClBU,OAAOO,KAAK;oBACZJ,OAAOb;gBACT;YACF;QACF,EAAE,OAAOE,OAAO;YACdD,QAAQC,KAAK,CAAC,yBAAyBA;YACvC,MAAMA;QACR;IACF;IA7GAgB,YAAYC,MAA4B,CAAE;YACzBA;QAAf,IAAI,CAACxC,OAAO,GAAGwC,CAAAA,kBAAAA,OAAOxC,OAAO,YAAdwC,kBAAkB;QACjCjC,YAAE,CAACC,KAAK,CAAC,IAAI,CAACR,OAAO,EAAE;YAAEU,WAAW;QAAK,GAAG+B,KAAK,CAACnB,QAAQC,KAAK;IACjE;AA2GF"}

package/src/index.d.ts

@@ -3,3 +3,5 @@ export * from './duckdb-singleton';
 export * from './node-sql-to-serialization';
 export { convertCubeStringToTableSchema };
 import { convertCubeStringToTableSchema } from '@devrev/meerkat-core';
+export * from './duckdb-manager/duckdb-manager';
+export * from './file-manager/file-manager';

package/src/index.js

@@ -10,5 +10,7 @@ _export_star._(require("./cube-to-sql/cube-to-sql"), exports);
 _export_star._(require("./duckdb-singleton"), exports);
 _export_star._(require("./node-sql-to-serialization"), exports);
 const _meerkatcore = require("@devrev/meerkat-core");
+_export_star._(require("./duckdb-manager/duckdb-manager"), exports);
+_export_star._(require("./file-manager/file-manager"), exports);
 
 //# sourceMappingURL=index.js.map

package/src/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../meerkat-node/src/index.ts"],"sourcesContent":["export * from './cube-to-sql/cube-to-sql';\nexport * from './duckdb-singleton';\nexport * from './node-sql-to-serialization';\nexport { convertCubeStringToTableSchema };\nimport { convertCubeStringToTableSchema } from '@devrev/meerkat-core';\n"],"names":["convertCubeStringToTableSchema"],"mappings":";+BAGSA;;;eAAAA,2CAA8B;;;;uBAHzB;uBACA;uBACA;6BAEiC"}
+{"version":3,"sources":["../../../meerkat-node/src/index.ts"],"sourcesContent":["export * from './cube-to-sql/cube-to-sql';\nexport * from './duckdb-singleton';\nexport * from './node-sql-to-serialization';\nexport { convertCubeStringToTableSchema };\nimport { convertCubeStringToTableSchema } from '@devrev/meerkat-core';\nexport * from './duckdb-manager/duckdb-manager';\nexport * from './file-manager/file-manager';\n"],"names":["convertCubeStringToTableSchema"],"mappings":";+BAGSA;;;eAAAA,2CAA8B;;;;uBAHzB;uBACA;uBACA;6BAEiC;uBACjC;uBACA"}

package/src/node-sql-to-serialization.js

@@ -12,7 +12,6 @@ const nodeSQLToSerialization = async (sql)=>{
     for(const key in queryOutput[0]){
         if (Object.prototype.hasOwnProperty.call(queryOutput[0], key)) {
             parsedOutput[key] = JSON.parse(queryOutput[0][key]);
-            break;
         }
     }
     return parsedOutput;

package/src/node-sql-to-serialization.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../meerkat-node/src/node-sql-to-serialization.ts"],"sourcesContent":["import { SelectStatement } from '@devrev/meerkat-core';\nimport { duckdbExec } from './duckdb-exec';\n\nexport interface ParsedSerialization {\n  [key: string]: {\n    error: boolean;\n    statements: SelectStatement[];\n  };\n}\n\nexport const nodeSQLToSerialization = async (\n  sql: string\n): Promise<ParsedSerialization> => {\n  const queryOutput = await duckdbExec<\n    {\n      [key: string]: string;\n    }[]\n  >(sql);\n\n  const parsedOutput: ParsedSerialization = {};\n\n  for (const key in queryOutput[0]) {\n    if (Object.prototype.hasOwnProperty.call(queryOutput[0], key)) {\n      parsedOutput[key] = JSON.parse(queryOutput[0][key]) as {\n        error: boolean;\n        statements: SelectStatement[];\n      };\n      break;\n    }\n  }\n  return parsedOutput;\n};\n"],"names":["nodeSQLToSerialization","sql","queryOutput","duckdbExec","parsedOutput","key","Object","prototype","hasOwnProperty","call","JSON","parse"],"mappings":";+BAUaA;;;eAAAA;;;4BATc;AASpB,MAAMA,yBAAyB,OACpCC;IAEA,MAAMC,cAAc,MAAMC,IAAAA,sBAAU,EAIlCF;IAEF,MAAMG,eAAoC,CAAC;IAE3C,IAAK,MAAMC,OAAOH,WAAW,CAAC,EAAE,CAAE;QAChC,IAAII,OAAOC,SAAS,CAACC,cAAc,CAACC,IAAI,CAACP,WAAW,CAAC,EAAE,EAAEG,MAAM;YAC7DD,YAAY,CAACC,IAAI,GAAGK,KAAKC,KAAK,CAACT,WAAW,CAAC,EAAE,CAACG,IAAI;YAIlD;QACF;IACF;IACA,OAAOD;AACT"}
+{"version":3,"sources":["../../../meerkat-node/src/node-sql-to-serialization.ts"],"sourcesContent":["import { SelectStatement } from '@devrev/meerkat-core';\nimport { duckdbExec } from './duckdb-exec';\n\nexport interface ParsedSerialization {\n  [key: string]: {\n    error: boolean;\n    statements: SelectStatement[];\n  };\n}\n\nexport const nodeSQLToSerialization = async (\n  sql: string\n): Promise<ParsedSerialization> => {\n  const queryOutput = await duckdbExec<\n    {\n      [key: string]: string;\n    }[]\n  >(sql);\n\n  const parsedOutput: ParsedSerialization = {};\n\n  for (const key in queryOutput[0]) {\n    if (Object.prototype.hasOwnProperty.call(queryOutput[0], key)) {\n      parsedOutput[key] = JSON.parse(queryOutput[0][key] as string) as {\n        error: boolean;\n        statements: SelectStatement[];\n      };\n    }\n  }\n  return parsedOutput;\n};\n"],"names":["nodeSQLToSerialization","sql","queryOutput","duckdbExec","parsedOutput","key","Object","prototype","hasOwnProperty","call","JSON","parse"],"mappings":";+BAUaA;;;eAAAA;;;4BATc;AASpB,MAAMA,yBAAyB,OACpCC;IAEA,MAAMC,cAAc,MAAMC,IAAAA,sBAAU,EAIlCF;IAEF,MAAMG,eAAoC,CAAC;IAE3C,IAAK,MAAMC,OAAOH,WAAW,CAAC,EAAE,CAAE;QAChC,IAAII,OAAOC,SAAS,CAACC,cAAc,CAACC,IAAI,CAACP,WAAW,CAAC,EAAE,EAAEG,MAAM;YAC7DD,YAAY,CAACC,IAAI,GAAGK,KAAKC,KAAK,CAACT,WAAW,CAAC,EAAE,CAACG,IAAI;QAIpD;IACF;IACA,OAAOD;AACT"}

package/src/utils/duckdb-type-convertor.d.ts

@@ -0,0 +1,3 @@
+import { ColumnInfo, TableData, TypeInfo } from 'duckdb';
+export declare const convertDuckDBValueToJS: (field: TypeInfo, value: unknown) => unknown;
+export declare const convertTableDataToJSON: (data: TableData, columns: ColumnInfo[]) => Record<string, unknown>[];

package/src/utils/duckdb-type-convertor.js

@@ -0,0 +1,61 @@
+"use strict";
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: all[name]
+    });
+}
+_export(exports, {
+    convertDuckDBValueToJS: function() {
+        return convertDuckDBValueToJS;
+    },
+    convertTableDataToJSON: function() {
+        return convertTableDataToJSON;
+    }
+});
+const _lodash = require("lodash");
+const convertDuckDBValueToJS = (field, value)=>{
+    if ((0, _lodash.isNil)(value)) return null;
+    switch(field.id){
+        case 'SQL_NULL':
+            return null;
+        case 'DATE':
+        case 'TIME':
+        case 'TIMESTAMP':
+            return new Date(value).toISOString();
+        case 'FLOAT':
+        case 'DOUBLE':
+            return value;
+        case 'INTEGER':
+        case 'TINYINT':
+        case 'SMALLINT':
+        case 'BIGINT':
+        case 'UTINYINT':
+        case 'USMALLINT':
+        case 'UINTEGER':
+        case 'UBIGINT':
+        case 'HUGEINT':
+        case 'UHUGEINT':
+            return parseInt(value.toString(), 10);
+        case 'DECIMAL':
+            return parseFloat(value.toString());
+        case 'LIST':
+            {
+                if (!value) return [];
+                const listValue = value;
+                return listValue.map((item)=>convertDuckDBValueToJS(field.child, item));
+            }
+        default:
+            return value;
+    }
+};
+const convertTableDataToJSON = (data, columns)=>{
+    return data.map((row)=>{
+        return columns.reduce((acc, column)=>{
+            acc[column.name] = convertDuckDBValueToJS(column.type, row[column.name]);
+            return acc;
+        }, {});
+    });
+};
+
+//# sourceMappingURL=duckdb-type-convertor.js.map

package/src/utils/duckdb-type-convertor.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../meerkat-node/src/utils/duckdb-type-convertor.ts"],"sourcesContent":["import { ColumnInfo, ListTypeInfo, TableData, TypeInfo } from 'duckdb';\nimport { isNil } from 'lodash';\n\nexport const convertDuckDBValueToJS = (\n  field: TypeInfo,\n  value: unknown\n): unknown => {\n  if (isNil(value)) return null;\n\n  switch (field.id) {\n    case 'SQL_NULL':\n      return null;\n    case 'DATE':\n    case 'TIME':\n    case 'TIMESTAMP':\n      return new Date(value as string).toISOString();\n    case 'FLOAT':\n    case 'DOUBLE':\n      return value;\n    case 'INTEGER':\n    case 'TINYINT':\n    case 'SMALLINT':\n    case 'BIGINT':\n    case 'UTINYINT':\n    case 'USMALLINT':\n    case 'UINTEGER':\n    case 'UBIGINT':\n    case 'HUGEINT':\n    case 'UHUGEINT':\n      return parseInt((value as object).toString(), 10);\n    case 'DECIMAL':\n      return parseFloat((value as object).toString());\n    case 'LIST': {\n      if (!value) return [];\n      const listValue = value as [];\n      return listValue.map((item) =>\n        convertDuckDBValueToJS((field as ListTypeInfo).child, item)\n      );\n    }\n    default:\n      return value;\n  }\n};\n\nexport const convertTableDataToJSON = (\n  data: TableData,\n  columns: ColumnInfo[]\n): Record<string, unknown>[] => {\n  return data.map((row: Record<string, unknown>) => {\n    return columns.reduce((acc, column) => {\n      acc[column.name] = convertDuckDBValueToJS(column.type, row[column.name]);\n      return acc;\n    }, {} as Record<string, unknown>);\n  });\n};\n"],"names":["convertDuckDBValueToJS","convertTableDataToJSON","field","value","isNil","id","Date","toISOString","parseInt","toString","parseFloat","listValue","map","item","child","data","columns","row","reduce","acc","column","name","type"],"mappings":";;;;;;;;IAGaA,sBAAsB;eAAtBA;;IAyCAC,sBAAsB;eAAtBA;;;wBA3CS;AAEf,MAAMD,yBAAyB,CACpCE,OACAC;IAEA,IAAIC,IAAAA,aAAK,EAACD,QAAQ,OAAO;IAEzB,OAAQD,MAAMG,EAAE;QACd,KAAK;YACH,OAAO;QACT,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAO,IAAIC,KAAKH,OAAiBI,WAAW;QAC9C,KAAK;QACL,KAAK;YACH,OAAOJ;QACT,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;QACL,KAAK;YACH,OAAOK,SAAS,AAACL,MAAiBM,QAAQ,IAAI;QAChD,KAAK;YACH,OAAOC,WAAW,AAACP,MAAiBM,QAAQ;QAC9C,KAAK;YAAQ;gBACX,IAAI,CAACN,OAAO,OAAO,EAAE;gBACrB,MAAMQ,YAAYR;gBAClB,OAAOQ,UAAUC,GAAG,CAAC,CAACC,OACpBb,uBAAuB,AAACE,MAAuBY,KAAK,EAAED;YAE1D;QACA;YACE,OAAOV;IACX;AACF;AAEO,MAAMF,yBAAyB,CACpCc,MACAC;IAEA,OAAOD,KAAKH,GAAG,CAAC,CAACK;QACf,OAAOD,QAAQE,MAAM,CAAC,CAACC,KAAKC;YAC1BD,GAAG,CAACC,OAAOC,IAAI,CAAC,GAAGrB,uBAAuBoB,OAAOE,IAAI,EAAEL,GAAG,CAACG,OAAOC,IAAI,CAAC;YACvE,OAAOF;QACT,GAAG,CAAC;IACN;AACF"}

package/src/utils/hash-string.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Hash a string using MD5.
+ */
+export declare const hashString: (text: string) => string;

package/src/utils/hash-string.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "hashString", {
+    enumerable: true,
+    get: function() {
+        return hashString;
+    }
+});
+const _crypto = require("crypto");
+const hashString = (text)=>{
+    return _crypto.createHash('md5').update(text).digest('hex');
+};
+
+//# sourceMappingURL=hash-string.js.map

package/src/utils/hash-string.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../meerkat-node/src/utils/hash-string.ts"],"sourcesContent":["import * as crypto from 'crypto';\n\n/**\n * Hash a string using MD5.\n */\nexport const hashString = (text: string): string => {\n  return crypto.createHash('md5').update(text).digest('hex');\n};\n"],"names":["hashString","text","crypto","createHash","update","digest"],"mappings":";+BAKaA;;;eAAAA;;;wBALW;AAKjB,MAAMA,aAAa,CAACC;IACzB,OAAOC,QAAOC,UAAU,CAAC,OAAOC,MAAM,CAACH,MAAMI,MAAM,CAAC;AACtD"}