@sqlrooms/duckdb 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DuckDbSlice.d.ts +110 -0
- package/dist/DuckDbSlice.d.ts.map +1 -0
- package/dist/DuckDbSlice.js +157 -0
- package/dist/DuckDbSlice.js.map +1 -0
- package/dist/connectors/DuckDbConnector.d.ts +40 -0
- package/dist/connectors/DuckDbConnector.d.ts.map +1 -0
- package/dist/connectors/DuckDbConnector.js +2 -0
- package/dist/connectors/DuckDbConnector.js.map +1 -0
- package/dist/connectors/WasmDuckDbConnector.d.ts +44 -0
- package/dist/connectors/WasmDuckDbConnector.d.ts.map +1 -0
- package/dist/connectors/WasmDuckDbConnector.js +210 -0
- package/dist/connectors/WasmDuckDbConnector.js.map +1 -0
- package/dist/connectors/load/create.d.ts +33 -0
- package/dist/connectors/load/create.d.ts.map +1 -0
- package/dist/connectors/load/create.js +33 -0
- package/dist/connectors/load/create.js.map +1 -0
- package/dist/connectors/load/load.d.ts +55 -0
- package/dist/connectors/load/load.d.ts.map +1 -0
- package/dist/connectors/load/load.js +147 -0
- package/dist/connectors/load/load.js.map +1 -0
- package/dist/connectors/load/sql-from.d.ts.map +1 -0
- package/dist/{sql-from.js → connectors/load/sql-from.js} +1 -1
- package/dist/connectors/load/sql-from.js.map +1 -0
- package/dist/duckdb-utils.d.ts +47 -0
- package/dist/duckdb-utils.d.ts.map +1 -0
- package/dist/duckdb-utils.js +65 -0
- package/dist/duckdb-utils.js.map +1 -0
- package/dist/exportToCsv.d.ts +3 -1
- package/dist/exportToCsv.d.ts.map +1 -1
- package/dist/exportToCsv.js +29 -24
- package/dist/exportToCsv.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -1
- package/dist/index.js.map +1 -1
- package/dist/useDuckDb.d.ts +2 -31
- package/dist/useDuckDb.d.ts.map +1 -1
- package/dist/useDuckDb.js +3 -229
- package/dist/useDuckDb.js.map +1 -1
- package/dist/useSql.d.ts.map +1 -1
- package/dist/useSql.js +4 -3
- package/dist/useSql.js.map +1 -1
- package/package.json +8 -4
- package/dist/__tests__/arrow-utils.test.d.ts +0 -2
- package/dist/__tests__/arrow-utils.test.d.ts.map +0 -1
- package/dist/__tests__/arrow-utils.test.js +0 -187
- package/dist/__tests__/arrow-utils.test.js.map +0 -1
- package/dist/__tests__/sql-from.test.d.ts +0 -2
- package/dist/__tests__/sql-from.test.d.ts.map +0 -1
- package/dist/__tests__/sql-from.test.js +0 -71
- package/dist/__tests__/sql-from.test.js.map +0 -1
- package/dist/duckdb.d.ts +0 -49
- package/dist/duckdb.d.ts.map +0 -1
- package/dist/duckdb.js +0 -82
- package/dist/duckdb.js.map +0 -1
- package/dist/sql-from.d.ts.map +0 -1
- package/dist/sql-from.js.map +0 -1
- /package/dist/{sql-from.d.ts → connectors/load/sql-from.d.ts} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"WasmDuckDbConnector.js","sourceRoot":"","sources":["../../src/connectors/WasmDuckDbConnector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,MAAM,qBAAqB,CAAC;AAC9C,OAAO,EAAC,kBAAkB,EAAoB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,EACL,wBAAwB,GAGzB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAC,IAAI,EAAE,WAAW,EAAE,WAAW,EAAC,MAAM,aAAa,CAAC;AAE3D,MAAM,OAAO,mBAAmB;IACtB,OAAO,CAAU;IACjB,MAAM,CAAS;IACf,mBAAmB,CAAS;IAC5B,EAAE,GAA8B,IAAI,CAAC;IACrC,IAAI,GAAwC,IAAI,CAAC;IACjD,MAAM,GAAkB,IAAI,CAAC;IAC7B,WAAW,GAAG,KAAK,CAAC;IACpB,YAAY,GAAyB,IAAI,CAAC;IAC1C,WAAW,CAAqB;IAExC,YAAY,EACV,OAAO,GAAG,KAAK,EACf,mBAAmB,GAAG,EAAE,EACxB,MAAM,GAAG,UAAU,EACnB,WAAW,MAMT,EAAE;QACJ,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,mBAAmB,GAAG,mBAAmB,CAAC;QAC/C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,KAAK,CAAC,UAAU;QACd,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,YAAY,CAAC;QAC3B,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC9C,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,CAAC,kBAAkB,EAAE,CAAC;YAC/C,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;YACzD,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC5D,CAAC;YACD,MAAM,SAAS,GAAG,GAAG,CAAC,eAAe,CACnC,IAAI,IAAI,CAAC,CAAC,kBAAkB,UAAU,CAAC,UAAU,KAAK,CAAC,EAAE;gBACvD,IAAI,EAAE,iBAAiB;aACxB,CAAC,CACH,CAAC;YAEF,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO;gBACzB,CAAC,CAAC,IAAI,MAAM,CAAC,aAAa,EAAE;gBAC5B,CAAC,CAAC;oBACE,eAAe;oBACf,GAAG,EAAE,GAAG,EAAE;wBACR,gBAAgB;oBAClB,CAAC;iBACF,CAAC;YAEN,MAAM,EAAE,GAAG,IAAI,CAAC,KAAM,SAAQ,MAAM,CAAC,WAAW;gBAC9C,OAAO,CAAC,KAAiB;oBACvB,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;oBACrB,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,KAAK,CAAC,CAAC;gBAC/C,CAAC;aACF,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;YAEnB,MAAM,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,UAAU,EAAE,UAAU,CAAC,aAAa,CAAC,CAAC;YACtE,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YAE/B,MAAM,EAAE,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,IAAI,CAAC,MAAM;gBACjB,KAAK,EAAE,IAAI,CAAC,WAAW;aACxB,CAAC,CAAC;YAEH,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,OAAO,EAAE,CAAC;YAEhC,mCAAmC;YACnC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC;YACjC,IAAI,CAAC,KAAK,GAAG,CAAC,KAAK,EAAE,CAAS,EAAE,EAAE;gBAChC,MAAM,KAAK,GAAG,IAAI,KAAK,EAAE,CAAC,KAAK,CAAC;gBAChC,IAAI,CAAC;oBACH,OAAO,MAAM,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAC3C,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,MAAM,IAAI,cAAc,CAAC,GAAG,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC,CAAsB,CAAC;YAExB,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;YAC7C,CAAC;YAED,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;YACrB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACd,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;YACnB,CAAC;YAED,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;gBACZ,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;gBAC1B,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC;YACjB,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;YACrB,CAAC;YAED,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CAAC,+BAA+B,EAAE,GAAG,CAAC,CAAC;YACpD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAa;QACvB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,QAAQ,CACZ,IAAmB,EACnB,SAAiB,EACjB,IAAsB;QAEtB,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE;YAC/D,IAAI,IAAI,IAAI,wBAAwB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3C,MAAM,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;YAC3D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,KAAK,CACd,IAAI,CAAC,IAAI,EAAE,MAAM,IAAI,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,CACxD,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CACb,IAA8B,EAC9B,SAAiB,EACjB,IAAwB;QAExB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,OAAO,GAAG,EAAC,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAC,CAAC;QACxD,IAAI,IAAI,YAAY,KAAK,CAAC,KAAK,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,KAAK,CAAC,WAAW,CACf,IAA+B,EAC/B,SAAiB,EACjB,IAA0B;QAE1B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAC5D,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAClC,IAAmB,EACnB,MAGkB;QAElB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,QAAgB,CAAC;QACrB,IAAI,IAAI,YAAY,IAAI,EAAE,CAAC;YACzB,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC;YACrB,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAC9B,QAAQ,EACR,IAAI,EACJ,kBAAkB,CAAC,kBAAkB,EACrC,IAAI,CACL,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,QAAQ,GAAG,IAAI,CAAC;QAClB,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC5C,CAAC;QACD,OAAO,IAAI,CAAC,EAAE,CAAC;IACjB,CAAC;IAED,aAAa;QACX,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;CACF;AAED,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC9B,KAAK,CAAU;IACf,KAAK,CAAqB;IAC1B,cAAc,CAAqB;IAC5C,YAAY,GAAY,EAAE,KAAa,EAAE,KAAyB;QAChE,KAAK,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,GAAG,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,SAAS,CAAC,CAAC;IACxD,CAAC;IACD,kBAAkB;QAChB,MAAM,EAAC,OAAO,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAC,GAAG,IAAI,CAAC;QACrD,OAAO,CACL,oBAAoB,OAAO,EAAE;YAC7B,sBAAsB,KAAK,4BAA4B,KAAK,MAAM,CACnE,CAAC;IACJ,CAAC;IACD,iBAAiB;QACf,MAAM,EAAC,OAAO,EAAC,GAAG,IAAI,CAAC;QACvB,OAAO,OAAO,CAAC;IACjB,CAAC;CACF","sourcesContent":["import * as duckdb from '@duckdb/duckdb-wasm';\nimport {DuckDBDataProtocol, DuckDBQueryConfig} from '@duckdb/duckdb-wasm';\nimport {\n isSpatialLoadFileOptions,\n LoadFileOptions,\n StandardLoadOptions,\n} from '@sqlrooms/project-config';\nimport * as arrow from 'apache-arrow';\nimport {DuckDbConnector} from './DuckDbConnector';\nimport {load, loadObjects, loadSpatial} from './load/load';\n\nexport class WasmDuckDbConnector implements DuckDbConnector {\n private logging: boolean;\n private dbPath: string;\n private initializationQuery: string;\n private db: duckdb.AsyncDuckDB | null = null;\n private conn: duckdb.AsyncDuckDBConnection | null = null;\n private worker: Worker | null = null;\n private initialized = false;\n private initializing: Promise<void> | null = null;\n private queryConfig?: DuckDBQueryConfig;\n\n constructor({\n logging = false,\n initializationQuery = '',\n dbPath = ':memory:',\n queryConfig,\n }: {\n dbPath?: string;\n queryConfig?: DuckDBQueryConfig;\n initializationQuery?: string;\n logging?: boolean;\n } = {}) {\n this.dbPath = dbPath;\n this.queryConfig = queryConfig;\n this.initializationQuery = initializationQuery;\n this.logging = logging;\n }\n\n async initialize(): Promise<void> {\n if (this.initialized) {\n return;\n }\n\n if (this.initializing) {\n return this.initializing;\n }\n\n if (!globalThis.Worker) {\n throw new Error('No Worker support in this environment');\n }\n\n this.initializing = this.initializeInternal();\n return this.initializing;\n }\n\n private async initializeInternal(): Promise<void> {\n try {\n const allBundles = duckdb.getJsDelivrBundles();\n const bestBundle = await duckdb.selectBundle(allBundles);\n if (!bestBundle.mainWorker) {\n throw new Error('No best bundle found for DuckDB worker');\n }\n const workerUrl = URL.createObjectURL(\n new Blob([`importScripts(\"${bestBundle.mainWorker}\");`], {\n type: 'text/javascript',\n }),\n );\n\n const worker = new window.Worker(workerUrl);\n const logger = this.logging\n ? new duckdb.ConsoleLogger()\n : {\n // Silently log\n log: () => {\n /* do nothing */\n },\n };\n\n const db = new (class extends duckdb.AsyncDuckDB {\n onError(event: ErrorEvent) {\n super.onError(event);\n console.error('DuckDB worker error:', event);\n }\n })(logger, worker);\n\n await db.instantiate(bestBundle.mainModule, bestBundle.pthreadWorker);\n URL.revokeObjectURL(workerUrl);\n\n await db.open({\n path: this.dbPath,\n query: this.queryConfig,\n });\n\n const conn = await db.connect();\n\n // Add error handling to conn.query\n const originalQuery = conn.query;\n conn.query = (async (q: string) => {\n const stack = new Error().stack;\n try {\n return await originalQuery.call(conn, q);\n } catch (err) {\n throw new DuckQueryError(err, q, stack);\n }\n }) as typeof conn.query;\n\n if (this.initializationQuery) {\n await conn.query(this.initializationQuery);\n }\n\n this.db = db;\n this.conn = conn;\n this.worker = worker;\n this.initialized = true;\n } catch (err) {\n this.initialized = false;\n this.initializing = null;\n throw err;\n }\n }\n\n async destroy(): Promise<void> {\n try {\n if (this.conn) {\n await this.conn.close();\n this.conn = null;\n }\n\n if (this.db) {\n await this.db.terminate();\n this.db = null;\n }\n\n if (this.worker) {\n this.worker.terminate();\n this.worker = null;\n }\n\n this.initialized = false;\n this.initializing = null;\n } catch (err) {\n console.error('Error during DuckDB shutdown:', err);\n throw err;\n }\n }\n\n private async ensureInitialized(): Promise<void> {\n if (!this.initialized) {\n await this.initialize();\n }\n }\n\n async query(query: string): Promise<arrow.Table> {\n await this.ensureInitialized();\n if (!this.conn) {\n throw new Error('DuckDB connection not initialized');\n }\n return await this.conn.query(query);\n }\n\n async loadFile(\n file: string | File,\n tableName: string,\n opts?: LoadFileOptions,\n ) {\n await this.withTempRegisteredFile(file, async (conn, fileName) => {\n if (opts && isSpatialLoadFileOptions(opts)) {\n await conn.query(loadSpatial(tableName, fileName, opts));\n } else {\n await conn.query(\n load(opts?.method ?? 'auto', tableName, fileName, opts),\n );\n }\n });\n }\n\n async loadArrow(\n file: arrow.Table | Uint8Array,\n tableName: string,\n opts?: {schema?: string},\n ) {\n await this.ensureInitialized();\n if (!this.conn) {\n throw new Error('DuckDB connection not initialized');\n }\n const options = {name: tableName, schema: opts?.schema};\n if (file instanceof arrow.Table) {\n await this.conn.insertArrowTable(file, options);\n } else {\n await this.conn.insertArrowFromIPCStream(file, options);\n }\n }\n\n async loadObjects(\n file: Record<string, unknown>[],\n tableName: string,\n opts?: StandardLoadOptions,\n ) {\n await this.ensureInitialized();\n if (!this.conn) {\n throw new Error('DuckDB connection not initialized');\n }\n await this.conn.query(loadObjects(tableName, file, opts));\n }\n\n private async withTempRegisteredFile(\n file: string | File,\n action: (\n conn: duckdb.AsyncDuckDBConnection,\n fileName: string,\n ) => Promise<void>,\n ) {\n await this.ensureInitialized();\n if (!this.conn || !this.db) {\n throw new Error('DuckDB connection not initialized');\n }\n let fileName: string;\n if (file instanceof File) {\n fileName = file.name;\n await this.db.registerFileHandle(\n fileName,\n file,\n DuckDBDataProtocol.BROWSER_FILEREADER,\n true,\n );\n } else {\n fileName = file;\n }\n try {\n await action(this.conn, fileName);\n } finally {\n await this.db.dropFile(fileName);\n }\n }\n\n getDb(): duckdb.AsyncDuckDB {\n if (!this.db) {\n throw new Error('DuckDB not initialized');\n }\n return this.db;\n }\n\n getConnection(): duckdb.AsyncDuckDBConnection {\n if (!this.conn) {\n throw new Error('DuckDB connection not initialized');\n }\n return this.conn;\n }\n}\n\nexport class DuckQueryError extends Error {\n readonly cause: unknown;\n readonly query: string | undefined;\n readonly queryCallStack: string | undefined;\n constructor(err: unknown, query: string, stack: string | undefined) {\n super(err instanceof Error ? err.message : `${err}`);\n this.cause = err;\n this.query = query;\n this.queryCallStack = stack;\n Object.setPrototypeOf(this, DuckQueryError.prototype);\n }\n getDetailedMessage() {\n const {message, query, queryCallStack: stack} = this;\n return (\n `DB query failed: ${message}` +\n `\\n\\nFull query:\\n\\n${query}\\n\\nQuery call stack:\\n\\n${stack}\\n\\n`\n );\n }\n getMessageForUser() {\n const {message} = this;\n return message;\n }\n}\n"]}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
interface TableOptions {
|
|
2
|
+
/** Whether to replace existing table/view if it exists */
|
|
3
|
+
replace?: boolean;
|
|
4
|
+
/** Whether to create a temporary table/view */
|
|
5
|
+
temp?: boolean;
|
|
6
|
+
/** Whether to create a view instead of a table */
|
|
7
|
+
view?: boolean;
|
|
8
|
+
}
|
|
9
|
+
interface SchemaOptions {
|
|
10
|
+
/** Whether to enforce strict schema creation (no IF NOT EXISTS) */
|
|
11
|
+
strict?: boolean;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Creates a SQL query string to create a new table or view in DuckDB
|
|
15
|
+
* @param name - Name of the table or view to create
|
|
16
|
+
* @param query - SQL query whose results will populate the table/view
|
|
17
|
+
* @param options - Options for table/view creation
|
|
18
|
+
* @param options.replace - Whether to replace existing table/view if it exists (default: false)
|
|
19
|
+
* @param options.temp - Whether to create a temporary table/view (default: false)
|
|
20
|
+
* @param options.view - Whether to create a view instead of a table (default: false)
|
|
21
|
+
* @returns SQL query string to create the table/view
|
|
22
|
+
*/
|
|
23
|
+
export declare function createTable(name: string, query: string, { replace, temp, view }?: TableOptions): string;
|
|
24
|
+
/**
|
|
25
|
+
* Creates a SQL query string to create a new schema in DuckDB
|
|
26
|
+
* @param name - Name of the schema to create
|
|
27
|
+
* @param options - Options for schema creation
|
|
28
|
+
* @param options.strict - Whether to enforce strict schema creation (no IF NOT EXISTS) (default: false)
|
|
29
|
+
* @returns SQL query string to create the schema
|
|
30
|
+
*/
|
|
31
|
+
export declare function createSchema(name: string, { strict }?: SchemaOptions): string;
|
|
32
|
+
export {};
|
|
33
|
+
//# sourceMappingURL=create.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create.d.ts","sourceRoot":"","sources":["../../../src/connectors/load/create.ts"],"names":[],"mappings":"AAGA,UAAU,YAAY;IACpB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,+CAA+C;IAC/C,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,kDAAkD;IAClD,IAAI,CAAC,EAAE,OAAO,CAAC;CAChB;AAED,UAAU,aAAa;IACrB,mEAAmE;IACnE,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,EAAC,OAAe,EAAE,IAAY,EAAE,IAAY,EAAC,GAAE,YAAiB,GAC/D,MAAM,CAWR;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAC1B,IAAI,EAAE,MAAM,EACZ,EAAC,MAAc,EAAC,GAAE,aAAkB,GACnC,MAAM,CAER"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
// Adapted from https://github.com/uwdata/mosaic/blob/main/packages/sql/src/load/
|
|
2
|
+
// BSD 3-Clause License Copyright (c) 2023, UW Interactive Data Lab
|
|
3
|
+
/**
|
|
4
|
+
* Creates a SQL query string to create a new table or view in DuckDB
|
|
5
|
+
* @param name - Name of the table or view to create
|
|
6
|
+
* @param query - SQL query whose results will populate the table/view
|
|
7
|
+
* @param options - Options for table/view creation
|
|
8
|
+
* @param options.replace - Whether to replace existing table/view if it exists (default: false)
|
|
9
|
+
* @param options.temp - Whether to create a temporary table/view (default: false)
|
|
10
|
+
* @param options.view - Whether to create a view instead of a table (default: false)
|
|
11
|
+
* @returns SQL query string to create the table/view
|
|
12
|
+
*/
|
|
13
|
+
export function createTable(name, query, { replace = false, temp = false, view = false } = {}) {
|
|
14
|
+
return ('CREATE' +
|
|
15
|
+
(replace ? ' OR REPLACE ' : ' ') +
|
|
16
|
+
(temp ? 'TEMP ' : '') +
|
|
17
|
+
(view ? 'VIEW' : 'TABLE') +
|
|
18
|
+
(replace ? ' ' : ' IF NOT EXISTS ') +
|
|
19
|
+
name +
|
|
20
|
+
' AS ' +
|
|
21
|
+
query);
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Creates a SQL query string to create a new schema in DuckDB
|
|
25
|
+
* @param name - Name of the schema to create
|
|
26
|
+
* @param options - Options for schema creation
|
|
27
|
+
* @param options.strict - Whether to enforce strict schema creation (no IF NOT EXISTS) (default: false)
|
|
28
|
+
* @returns SQL query string to create the schema
|
|
29
|
+
*/
|
|
30
|
+
export function createSchema(name, { strict = false } = {}) {
|
|
31
|
+
return 'CREATE SCHEMA ' + (strict ? '' : 'IF NOT EXISTS ') + name;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=create.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create.js","sourceRoot":"","sources":["../../../src/connectors/load/create.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,mEAAmE;AAgBnE;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,KAAa,EACb,EAAC,OAAO,GAAG,KAAK,EAAE,IAAI,GAAG,KAAK,EAAE,IAAI,GAAG,KAAK,KAAkB,EAAE;IAEhE,OAAO,CACL,QAAQ;QACR,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC;QAChC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACrB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QACzB,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,iBAAiB,CAAC;QACnC,IAAI;QACJ,MAAM;QACN,KAAK,CACN,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAC1B,IAAY,EACZ,EAAC,MAAM,GAAG,KAAK,KAAmB,EAAE;IAEpC,OAAO,gBAAgB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;AACpE,CAAC","sourcesContent":["// Adapted from https://github.com/uwdata/mosaic/blob/main/packages/sql/src/load/\n// BSD 3-Clause License Copyright (c) 2023, UW Interactive Data Lab\n\ninterface TableOptions {\n /** Whether to replace existing table/view if it exists */\n replace?: boolean;\n /** Whether to create a temporary table/view */\n temp?: boolean;\n /** Whether to create a view instead of a table */\n view?: boolean;\n}\n\ninterface SchemaOptions {\n /** Whether to enforce strict schema creation (no IF NOT EXISTS) */\n strict?: boolean;\n}\n\n/**\n * Creates a SQL query string to create a new table or view in DuckDB\n * @param name - Name of the table or view to create\n * @param query - SQL query whose results will populate the table/view\n * @param options - Options for table/view creation\n * @param options.replace - Whether to replace existing table/view if it exists (default: false)\n * @param options.temp - Whether to create a temporary table/view (default: false)\n * @param options.view - Whether to create a view instead of a table (default: false)\n * @returns SQL query string to create the table/view\n */\nexport function createTable(\n name: string,\n query: string,\n {replace = false, temp = false, view = false}: TableOptions = {},\n): string {\n return (\n 'CREATE' +\n (replace ? ' OR REPLACE ' : ' ') +\n (temp ? 'TEMP ' : '') +\n (view ? 'VIEW' : 'TABLE') +\n (replace ? ' ' : ' IF NOT EXISTS ') +\n name +\n ' AS ' +\n query\n );\n}\n\n/**\n * Creates a SQL query string to create a new schema in DuckDB\n * @param name - Name of the schema to create\n * @param options - Options for schema creation\n * @param options.strict - Whether to enforce strict schema creation (no IF NOT EXISTS) (default: false)\n * @returns SQL query string to create the schema\n */\nexport function createSchema(\n name: string,\n {strict = false}: SchemaOptions = {},\n): string {\n return 'CREATE SCHEMA ' + (strict ? '' : 'IF NOT EXISTS ') + name;\n}\n"]}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { LoadFile, SpatialLoadOptions, StandardLoadOptions } from '@sqlrooms/project-config';
|
|
2
|
+
/**
|
|
3
|
+
* Generic function to load data from a file into a DuckDB table
|
|
4
|
+
* @param method - The DuckDB read method to use (e.g., 'read_csv', 'read_json')
|
|
5
|
+
* @param tableName - Name of the table to create
|
|
6
|
+
* @param fileName - Path to the input file
|
|
7
|
+
* @param options - Load options including select, where, view, temp, replace and file-specific options
|
|
8
|
+
* @param defaults - Default options to merge with provided options
|
|
9
|
+
* @returns SQL query string to create the table
|
|
10
|
+
*/
|
|
11
|
+
export declare function load(method: LoadFile, tableName: string, fileName: string, options?: StandardLoadOptions, defaults?: Record<string, unknown>): string;
|
|
12
|
+
/**
|
|
13
|
+
* Load data from a CSV file into a DuckDB table
|
|
14
|
+
* @param tableName - Name of the table to create
|
|
15
|
+
* @param fileName - Path to the CSV file
|
|
16
|
+
* @param options - Load options
|
|
17
|
+
* @returns SQL query string to create the table
|
|
18
|
+
*/
|
|
19
|
+
export declare function loadCSV(tableName: string, fileName: string, options?: StandardLoadOptions): string;
|
|
20
|
+
/**
|
|
21
|
+
* Load data from a JSON file into a DuckDB table
|
|
22
|
+
* @param tableName - Name of the table to create
|
|
23
|
+
* @param fileName - Path to the JSON file
|
|
24
|
+
* @param options - Load options
|
|
25
|
+
* @returns SQL query string to create the table
|
|
26
|
+
*/
|
|
27
|
+
export declare function loadJSON(tableName: string, fileName: string, options?: StandardLoadOptions): string;
|
|
28
|
+
/**
|
|
29
|
+
* Load data from a Parquet file into a DuckDB table
|
|
30
|
+
* @param tableName - Name of the table to create
|
|
31
|
+
* @param fileName - Path to the Parquet file
|
|
32
|
+
* @param options - Load options
|
|
33
|
+
* @returns SQL query string to create the table
|
|
34
|
+
*/
|
|
35
|
+
export declare function loadParquet(tableName: string, fileName: string, options?: StandardLoadOptions): string;
|
|
36
|
+
/**
|
|
37
|
+
* Load geometry data within a spatial file format.
|
|
38
|
+
* This method requires that the DuckDB spatial extension is loaded.
|
|
39
|
+
* Supports GeoJSON, TopoJSON, and other common spatial formats.
|
|
40
|
+
* For TopoJSON, set the layer option to indicate the feature to extract.
|
|
41
|
+
* @param tableName - Name of the table to create
|
|
42
|
+
* @param fileName - Path to the spatial data file
|
|
43
|
+
* @param options - Load options including spatial-specific options
|
|
44
|
+
* @returns SQL query string to create the table
|
|
45
|
+
*/
|
|
46
|
+
export declare function loadSpatial(tableName: string, fileName: string, options?: SpatialLoadOptions): string;
|
|
47
|
+
/**
|
|
48
|
+
* Load JavaScript objects directly into a DuckDB table
|
|
49
|
+
* @param tableName - Name of the table to create
|
|
50
|
+
* @param data - Array of objects to load
|
|
51
|
+
* @param options - Load options
|
|
52
|
+
* @returns SQL query string to create the table
|
|
53
|
+
*/
|
|
54
|
+
export declare function loadObjects(tableName: string, data: Record<string, unknown>[], options?: StandardLoadOptions): string;
|
|
55
|
+
//# sourceMappingURL=load.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load.d.ts","sourceRoot":"","sources":["../../../src/connectors/load/load.ts"],"names":[],"mappings":"AAGA,OAAO,EACL,QAAQ,EACR,kBAAkB,EAClB,mBAAmB,EACpB,MAAM,0BAA0B,CAAC;AAIlC;;;;;;;;GAQG;AACH,wBAAgB,IAAI,CAClB,MAAM,EAAE,QAAQ,EAChB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,mBAAwB,EACjC,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GACrC,MAAM,CAcR;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CACrB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,mBAAmB,GAC5B,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CACtB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,mBAAmB,GAC5B,MAAM,CAKR;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CACzB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,mBAAmB,GAC5B,MAAM,CAER;AAED;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CACzB,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,kBAAuB,GAC/B,MAAM,CAsBR;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CACzB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,OAAO,GAAE,mBAAwB,GAChC,MAAM,CAQR"}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
// Adapted from https://github.com/uwdata/mosaic/blob/main/packages/sql/src/load/
|
|
2
|
+
// BSD 3-Clause License Copyright (c) 2023, UW Interactive Data Lab
|
|
3
|
+
import { createTable } from './create';
|
|
4
|
+
import { sqlFrom } from './sql-from';
|
|
5
|
+
/**
|
|
6
|
+
* Generic function to load data from a file into a DuckDB table
|
|
7
|
+
* @param method - The DuckDB read method to use (e.g., 'read_csv', 'read_json')
|
|
8
|
+
* @param tableName - Name of the table to create
|
|
9
|
+
* @param fileName - Path to the input file
|
|
10
|
+
* @param options - Load options including select, where, view, temp, replace and file-specific options
|
|
11
|
+
* @param defaults - Default options to merge with provided options
|
|
12
|
+
* @returns SQL query string to create the table
|
|
13
|
+
*/
|
|
14
|
+
export function load(method, tableName, fileName, options = {}, defaults = {}) {
|
|
15
|
+
const { schema, select = ['*'], where, view, temp, replace, ...file } = options;
|
|
16
|
+
const params = parameters({ ...defaults, ...file });
|
|
17
|
+
const read = method === 'auto'
|
|
18
|
+
? `'${fileName}'${params ? ', ' + params : ''}`
|
|
19
|
+
: `${method}('${fileName}'${params ? ', ' + params : ''})`;
|
|
20
|
+
const filter = where ? ` WHERE ${where}` : '';
|
|
21
|
+
const query = `SELECT ${select.join(', ')} FROM ${read}${filter}`;
|
|
22
|
+
return createTable(schema ? `${schema}.${tableName}` : tableName, query, {
|
|
23
|
+
view,
|
|
24
|
+
temp,
|
|
25
|
+
replace,
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Load data from a CSV file into a DuckDB table
|
|
30
|
+
* @param tableName - Name of the table to create
|
|
31
|
+
* @param fileName - Path to the CSV file
|
|
32
|
+
* @param options - Load options
|
|
33
|
+
* @returns SQL query string to create the table
|
|
34
|
+
*/
|
|
35
|
+
export function loadCSV(tableName, fileName, options) {
|
|
36
|
+
return load('read_csv', tableName, fileName, options, {
|
|
37
|
+
auto_detect: true,
|
|
38
|
+
sample_size: -1,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* Load data from a JSON file into a DuckDB table
|
|
43
|
+
* @param tableName - Name of the table to create
|
|
44
|
+
* @param fileName - Path to the JSON file
|
|
45
|
+
* @param options - Load options
|
|
46
|
+
* @returns SQL query string to create the table
|
|
47
|
+
*/
|
|
48
|
+
export function loadJSON(tableName, fileName, options) {
|
|
49
|
+
return load('read_json', tableName, fileName, options, {
|
|
50
|
+
auto_detect: true,
|
|
51
|
+
format: 'auto',
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Load data from a Parquet file into a DuckDB table
|
|
56
|
+
* @param tableName - Name of the table to create
|
|
57
|
+
* @param fileName - Path to the Parquet file
|
|
58
|
+
* @param options - Load options
|
|
59
|
+
* @returns SQL query string to create the table
|
|
60
|
+
*/
|
|
61
|
+
export function loadParquet(tableName, fileName, options) {
|
|
62
|
+
return load('read_parquet', tableName, fileName, options);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Load geometry data within a spatial file format.
|
|
66
|
+
* This method requires that the DuckDB spatial extension is loaded.
|
|
67
|
+
* Supports GeoJSON, TopoJSON, and other common spatial formats.
|
|
68
|
+
* For TopoJSON, set the layer option to indicate the feature to extract.
|
|
69
|
+
* @param tableName - Name of the table to create
|
|
70
|
+
* @param fileName - Path to the spatial data file
|
|
71
|
+
* @param options - Load options including spatial-specific options
|
|
72
|
+
* @returns SQL query string to create the table
|
|
73
|
+
*/
|
|
74
|
+
export function loadSpatial(tableName, fileName, options = {}) {
|
|
75
|
+
// nested options map to the open_options argument of st_read
|
|
76
|
+
const { schema, options: opt, ...rest } = options;
|
|
77
|
+
if (opt) {
|
|
78
|
+
// TODO: check correct syntax for open_options
|
|
79
|
+
const open = Array.isArray(opt)
|
|
80
|
+
? opt.join(', ')
|
|
81
|
+
: typeof opt === 'string'
|
|
82
|
+
? opt
|
|
83
|
+
: Object.entries(opt)
|
|
84
|
+
.map(([key, value]) => `${key}=${value}`)
|
|
85
|
+
.join(', ');
|
|
86
|
+
Object.assign(rest, { open_options: open.toUpperCase() });
|
|
87
|
+
}
|
|
88
|
+
// TODO: handle box_2d for spatial_filter_box option
|
|
89
|
+
// TODO: handle wkb_blob for spatial_filter option
|
|
90
|
+
return load('st_read', schema ? `${schema}.${tableName}` : tableName, fileName, rest);
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Load JavaScript objects directly into a DuckDB table
|
|
94
|
+
* @param tableName - Name of the table to create
|
|
95
|
+
* @param data - Array of objects to load
|
|
96
|
+
* @param options - Load options
|
|
97
|
+
* @returns SQL query string to create the table
|
|
98
|
+
*/
|
|
99
|
+
export function loadObjects(tableName, data, options = {}) {
|
|
100
|
+
const { schema, select = ['*'], ...opt } = options;
|
|
101
|
+
const values = sqlFrom(data);
|
|
102
|
+
const query = select.length === 1 && select[0] === '*'
|
|
103
|
+
? values
|
|
104
|
+
: `SELECT ${select} FROM ${values}`;
|
|
105
|
+
return createTable(schema ? `${schema}.${tableName}` : tableName, query, opt);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Convert options object to DuckDB parameter string
|
|
109
|
+
* @param options - Object containing parameter key-value pairs
|
|
110
|
+
* @returns Formatted parameter string
|
|
111
|
+
*/
|
|
112
|
+
function parameters(options) {
|
|
113
|
+
return Object.entries(options)
|
|
114
|
+
.map(([key, value]) => `${key}=${toDuckDBValue(value)}`)
|
|
115
|
+
.join(', ');
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Convert JavaScript value to DuckDB literal string representation
|
|
119
|
+
* @param value - Value to convert
|
|
120
|
+
* @returns DuckDB literal string
|
|
121
|
+
*/
|
|
122
|
+
function toDuckDBValue(value) {
|
|
123
|
+
switch (typeof value) {
|
|
124
|
+
case 'boolean':
|
|
125
|
+
return String(value);
|
|
126
|
+
case 'string':
|
|
127
|
+
return `'${value}'`;
|
|
128
|
+
case 'undefined':
|
|
129
|
+
case 'object':
|
|
130
|
+
if (value == null) {
|
|
131
|
+
return 'NULL';
|
|
132
|
+
}
|
|
133
|
+
else if (Array.isArray(value)) {
|
|
134
|
+
return '[' + value.map((v) => toDuckDBValue(v)).join(', ') + ']';
|
|
135
|
+
}
|
|
136
|
+
else {
|
|
137
|
+
return ('{' +
|
|
138
|
+
Object.entries(value)
|
|
139
|
+
.map(([k, v]) => `'${k}': ${toDuckDBValue(v)}`)
|
|
140
|
+
.join(', ') +
|
|
141
|
+
'}');
|
|
142
|
+
}
|
|
143
|
+
default:
|
|
144
|
+
return String(value);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
//# sourceMappingURL=load.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"load.js","sourceRoot":"","sources":["../../../src/connectors/load/load.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,mEAAmE;AAOnE,OAAO,EAAC,WAAW,EAAC,MAAM,UAAU,CAAC;AACrC,OAAO,EAAC,OAAO,EAAC,MAAM,YAAY,CAAC;AAEnC;;;;;;;;GAQG;AACH,MAAM,UAAU,IAAI,CAClB,MAAgB,EAChB,SAAiB,EACjB,QAAgB,EAChB,UAA+B,EAAE,EACjC,WAAoC,EAAE;IAEtC,MAAM,EAAC,MAAM,EAAE,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,EAAC,GAAG,OAAO,CAAC;IAC9E,MAAM,MAAM,GAAG,UAAU,CAAC,EAAC,GAAG,QAAQ,EAAE,GAAG,IAAI,EAAC,CAAC,CAAC;IAClD,MAAM,IAAI,GACR,MAAM,KAAK,MAAM;QACf,CAAC,CAAC,IAAI,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QAC/C,CAAC,CAAC,GAAG,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;IAC/D,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC9C,MAAM,KAAK,GAAG,UAAU,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,GAAG,MAAM,EAAE,CAAC;IAClE,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE;QACvE,IAAI;QACJ,IAAI;QACJ,OAAO;KACR,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,OAAO,CACrB,SAAiB,EACjB,QAAgB,EAChB,OAA6B;IAE7B,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE;QACpD,WAAW,EAAE,IAAI;QACjB,WAAW,EAAE,CAAC,CAAC;KAChB,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CACtB,SAAiB,EACjB,QAAgB,EAChB,OAA6B;IAE7B,OAAO,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,EAAE;QACrD,WAAW,EAAE,IAAI;QACjB,MAAM,EAAE,MAAM;KACf,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CACzB,SAAiB,EACjB,QAAgB,EAChB,OAA6B;IAE7B,OAAO,IAAI,CAAC,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;AAC5D,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CACzB,SAAiB,EACjB,QAAgB,EAChB,UAA8B,EAAE;IAEhC,6DAA6D;IAC7D,MAAM,EAAC,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,IAAI,EAAC,GAAG,OAAO,CAAC;IAChD,IAAI,GAAG,EAAE,CAAC;QACR,8CAA8C;QAC9C,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;YAC7B,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,CAAC,CAAC,OAAO,GAAG,KAAK,QAAQ;gBACvB,CAAC,CAAC,GAAG;gBACL,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC;qBAChB,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;qBACxC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,EAAC,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE,EAAC,CAAC,CAAC;IAC1D,CAAC;IACD,oDAAoD;IACpD,kDAAkD;IAClD,OAAO,IAAI,CACT,SAAS,EACT,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,EAC7C,QAAQ,EACR,IAAI,CACL,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CACzB,SAAiB,EACjB,IAA+B,EAC/B,UAA+B,EAAE;IAEjC,MAAM,EAAC,MAAM,EAAE,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,EAAC,GAAG,OAAO,CAAC;IACjD,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,KAAK,GACT,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,GAAG;QACtC,CAAC,CAAC,MAAM;QACR,CAAC,CAAC,UAAU,MAAM,SAAS,MAAM,EAAE,CAAC;IACxC,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;AAChF,CAAC;AAED;;;;GAIG;AACH,SAAS,UAAU,CAAC,OAAgC;IAClD,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;SACvD,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,KAAc;IACnC,QAAQ,OAAO,KAAK,EAAE,CAAC;QACrB,KAAK,SAAS;YACZ,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;QACvB,KAAK,QAAQ;YACX,OAAO,IAAI,KAAK,GAAG,CAAC;QACtB,KAAK,WAAW,CAAC;QACjB,KAAK,QAAQ;YACX,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,OAAO,MAAM,CAAC;YAChB,CAAC;iBAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBAChC,OAAO,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;YACnE,CAAC;iBAAM,CAAC;gBACN,OAAO,CACL,GAAG;oBACH,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC;yBAClB,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;yBAC9C,IAAI,CAAC,IAAI,CAAC;oBACb,GAAG,CACJ,CAAC;YACJ,CAAC;QACH;YACE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;AACH,CAAC","sourcesContent":["// Adapted from https://github.com/uwdata/mosaic/blob/main/packages/sql/src/load/\n// BSD 3-Clause License Copyright (c) 2023, UW Interactive Data Lab\n\nimport {\n LoadFile,\n SpatialLoadOptions,\n StandardLoadOptions,\n} from '@sqlrooms/project-config';\nimport {createTable} from './create';\nimport {sqlFrom} from './sql-from';\n\n/**\n * Generic function to load data from a file into a DuckDB table\n * @param method - The DuckDB read method to use (e.g., 'read_csv', 'read_json')\n * @param tableName - Name of the table to create\n * @param fileName - Path to the input file\n * @param options - Load options including select, where, view, temp, replace and file-specific options\n * @param defaults - Default options to merge with provided options\n * @returns SQL query string to create the table\n */\nexport function load(\n method: LoadFile,\n tableName: string,\n fileName: string,\n options: StandardLoadOptions = {},\n defaults: Record<string, unknown> = {},\n): string {\n const {schema, select = ['*'], where, view, temp, replace, ...file} = options;\n const params = parameters({...defaults, ...file});\n const read =\n method === 'auto'\n ? `'${fileName}'${params ? ', ' + params : ''}`\n : `${method}('${fileName}'${params ? ', ' + params : ''})`;\n const filter = where ? ` WHERE ${where}` : '';\n const query = `SELECT ${select.join(', ')} FROM ${read}${filter}`;\n return createTable(schema ? `${schema}.${tableName}` : tableName, query, {\n view,\n temp,\n replace,\n });\n}\n\n/**\n * Load data from a CSV file into a DuckDB table\n * @param tableName - Name of the table to create\n * @param fileName - Path to the CSV file\n * @param options - Load options\n * @returns SQL query string to create the table\n */\nexport function loadCSV(\n tableName: string,\n fileName: string,\n options?: StandardLoadOptions,\n): string {\n return load('read_csv', tableName, fileName, options, {\n auto_detect: true,\n sample_size: -1,\n });\n}\n\n/**\n * Load data from a JSON file into a DuckDB table\n * @param tableName - Name of the table to create\n * @param fileName - Path to the JSON file\n * @param options - Load options\n * @returns SQL query string to create the table\n */\nexport function loadJSON(\n tableName: string,\n fileName: string,\n options?: StandardLoadOptions,\n): string {\n return load('read_json', tableName, fileName, options, {\n auto_detect: true,\n format: 'auto',\n });\n}\n\n/**\n * Load data from a Parquet file into a DuckDB table\n * @param tableName - Name of the table to create\n * @param fileName - Path to the Parquet file\n * @param options - Load options\n * @returns SQL query string to create the table\n */\nexport function loadParquet(\n tableName: string,\n fileName: string,\n options?: StandardLoadOptions,\n): string {\n return load('read_parquet', tableName, fileName, options);\n}\n\n/**\n * Load geometry data within a spatial file format.\n * This method requires that the DuckDB spatial extension is loaded.\n * Supports GeoJSON, TopoJSON, and other common spatial formats.\n * For TopoJSON, set the layer option to indicate the feature to extract.\n * @param tableName - Name of the table to create\n * @param fileName - Path to the spatial data file\n * @param options - Load options including spatial-specific options\n * @returns SQL query string to create the table\n */\nexport function loadSpatial(\n tableName: string,\n fileName: string,\n options: SpatialLoadOptions = {},\n): string {\n // nested options map to the open_options argument of st_read\n const {schema, options: opt, ...rest} = options;\n if (opt) {\n // TODO: check correct syntax for open_options\n const open = Array.isArray(opt)\n ? opt.join(', ')\n : typeof opt === 'string'\n ? opt\n : Object.entries(opt)\n .map(([key, value]) => `${key}=${value}`)\n .join(', ');\n Object.assign(rest, {open_options: open.toUpperCase()});\n }\n // TODO: handle box_2d for spatial_filter_box option\n // TODO: handle wkb_blob for spatial_filter option\n return load(\n 'st_read',\n schema ? `${schema}.${tableName}` : tableName,\n fileName,\n rest,\n );\n}\n\n/**\n * Load JavaScript objects directly into a DuckDB table\n * @param tableName - Name of the table to create\n * @param data - Array of objects to load\n * @param options - Load options\n * @returns SQL query string to create the table\n */\nexport function loadObjects(\n tableName: string,\n data: Record<string, unknown>[],\n options: StandardLoadOptions = {},\n): string {\n const {schema, select = ['*'], ...opt} = options;\n const values = sqlFrom(data);\n const query =\n select.length === 1 && select[0] === '*'\n ? values\n : `SELECT ${select} FROM ${values}`;\n return createTable(schema ? `${schema}.${tableName}` : tableName, query, opt);\n}\n\n/**\n * Convert options object to DuckDB parameter string\n * @param options - Object containing parameter key-value pairs\n * @returns Formatted parameter string\n */\nfunction parameters(options: Record<string, unknown>): string {\n return Object.entries(options)\n .map(([key, value]) => `${key}=${toDuckDBValue(value)}`)\n .join(', ');\n}\n\n/**\n * Convert JavaScript value to DuckDB literal string representation\n * @param value - Value to convert\n * @returns DuckDB literal string\n */\nfunction toDuckDBValue(value: unknown): string {\n switch (typeof value) {\n case 'boolean':\n return String(value);\n case 'string':\n return `'${value}'`;\n case 'undefined':\n case 'object':\n if (value == null) {\n return 'NULL';\n } else if (Array.isArray(value)) {\n return '[' + value.map((v) => toDuckDBValue(v)).join(', ') + ']';\n } else {\n return (\n '{' +\n Object.entries(value)\n .map(([k, v]) => `'${k}': ${toDuckDBValue(v)}`)\n .join(', ') +\n '}'\n );\n }\n default:\n return String(value);\n }\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sql-from.d.ts","sourceRoot":"","sources":["../../../src/connectors/load/sql-from.ts"],"names":[],"mappings":"AAGA;;;;;;;GAOG;AACH,wBAAgB,OAAO,CACrB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,EAC/B,EACE,OAAsC,GACvC,GAAE;IAAC,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAM,UAsBtD;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,UA2B1C"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Adapted from https://github.com/uwdata/mosaic/blob/main/packages/sql/src/load/
|
|
1
|
+
// Adapted from https://github.com/uwdata/mosaic/blob/main/packages/sql/src/load/
|
|
2
2
|
// BSD 3-Clause License Copyright (c) 2023, UW Interactive Data Lab
|
|
3
3
|
/**
|
|
4
4
|
* Create a SQL query that embeds the given data for loading.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sql-from.js","sourceRoot":"","sources":["../../../src/connectors/load/sql-from.ts"],"names":[],"mappings":"AAAA,iFAAiF;AACjF,mEAAmE;AAEnE;;;;;;;GAOG;AACH,MAAM,UAAU,OAAO,CACrB,IAA+B,EAC/B,EACE,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MACW,EAAE;IAErD,IAAI,IAAI,GAAa,EAAE,CAAC;IACxB,IAAI,SAAiC,CAAC;IACtC,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC3B,IAAI,GAAG,OAAO,CAAC;QACf,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1D,CAAC;SAAM,CAAC;QACN,SAAS,GAAG,OAAO,CAAC;QACpB,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9B,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,IAAI,GAAG,EAAE,CAAC;IAChB,KAAK,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAClB,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,SAAS,CAAC,CAAC,CAAC,GAAG,CACxD,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AAClC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,QAAQ,OAAO,KAAK,EAAE,CAAC;QACrB,KAAK,QAAQ;YACX,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACtD,KAAK,QAAQ;YACX,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;QACzC,KAAK,SAAS;YACZ,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;QAClC;YACE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,OAAO,MAAM,CAAC;YAChB,CAAC;iBAAM,IAAI,KAAK,YAAY,IAAI,EAAE,CAAC;gBACjC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC;gBAClB,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;oBAAE,OAAO,MAAM,CAAC;gBACpC,MAAM,CAAC,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC;gBACjC,MAAM,CAAC,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;gBAC9B,MAAM,CAAC,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC;gBAC7B,OAAO,EAAE,KAAK,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;oBAC7B,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW;oBACzC,CAAC,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC,YAAY;YACrC,CAAC;iBAAM,IAAI,KAAK,YAAY,MAAM,EAAE,CAAC;gBACnC,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,oCAAoC;gBACpC,OAAO,GAAG,KAAK,EAAE,CAAC;YACpB,CAAC;IACL,CAAC;AACH,CAAC","sourcesContent":["// Adapted from https://github.com/uwdata/mosaic/blob/main/packages/sql/src/load/\n// BSD 3-Clause License Copyright (c) 2023, UW Interactive Data Lab\n\n/**\n * Create a SQL query that embeds the given data for loading.\n * @param {*} data The dataset as an array of objects.\n * @param {object} [options] Loading options.\n * @param {string[]|Record<string,string>} [options.columns] The columns to include.\n * If not specified, the keys of the first data object are used.\n * @returns {string} SQL query string to load data.\n */\nexport function sqlFrom(\n data: Record<string, unknown>[],\n {\n columns = Object.keys(data?.[0] || {}),\n }: {columns?: string[] | Record<string, string>} = {},\n) {\n let keys: string[] = [];\n let columnMap: Record<string, string>;\n if (Array.isArray(columns)) {\n keys = columns;\n columnMap = keys.reduce((m, k) => ({...m, [k]: k}), {});\n } else {\n columnMap = columns;\n keys = Object.keys(columns);\n }\n if (!keys.length) {\n throw new Error('Can not create table from empty column set.');\n }\n const subq = [];\n for (const datum of data) {\n const sel = keys.map(\n (k) => `${literalToSQL(datum[k])} AS \"${columnMap[k]}\"`,\n );\n subq.push(`(SELECT ${sel.join(', ')})`);\n }\n return subq.join(' UNION ALL ');\n}\n\n/**\n * Convert a value to a SQL literal.\n * @param {*} value The value to convert.\n * @returns {string} The SQL literal.\n */\nexport function literalToSQL(value: unknown) {\n switch (typeof value) {\n case 'number':\n return Number.isFinite(value) ? `${value}` : 'NULL';\n case 'string':\n return `'${value.replace(`'`, `''`)}'`;\n case 'boolean':\n return value ? 'TRUE' : 'FALSE';\n default:\n if (value == null) {\n return 'NULL';\n } else if (value instanceof Date) {\n const ts = +value;\n if (Number.isNaN(ts)) return 'NULL';\n const y = value.getUTCFullYear();\n const m = value.getUTCMonth();\n const d = value.getUTCDate();\n return ts === Date.UTC(y, m, d)\n ? `DATE '${y}-${m + 1}-${d}'` // utc date\n : `epoch_ms(${ts})`; // timestamp\n } else if (value instanceof RegExp) {\n return `'${value.source}'`;\n } else {\n // otherwise rely on string coercion\n return `${value}`;\n }\n }\n}\n"]}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import * as arrow from 'apache-arrow';
|
|
2
|
+
/**
|
|
3
|
+
* Escapes a value for use in DuckDB SQL queries by wrapping it in single quotes
|
|
4
|
+
* and escaping any existing single quotes by doubling them.
|
|
5
|
+
*
|
|
6
|
+
* @param val - The value to escape. Will be converted to string if not already a string.
|
|
7
|
+
* @returns The escaped string value wrapped in single quotes.
|
|
8
|
+
* @example
|
|
9
|
+
* escapeVal("John's data") // Returns "'John''s data'"
|
|
10
|
+
*/
|
|
11
|
+
export declare const escapeVal: (val: unknown) => string;
|
|
12
|
+
/**
|
|
13
|
+
* Escapes an identifier (like table or column names) for use in DuckDB SQL queries
|
|
14
|
+
* by wrapping it in double quotes and escaping any existing double quotes by doubling them.
|
|
15
|
+
* If the identifier is already properly quoted, returns it as is.
|
|
16
|
+
*
|
|
17
|
+
* @param id - The identifier string to escape
|
|
18
|
+
* @returns The escaped identifier wrapped in double quotes
|
|
19
|
+
* @example
|
|
20
|
+
* escapeId("my_table") // Returns '"my_table"'
|
|
21
|
+
* escapeId("my""table") // Returns '"my""""table"'
|
|
22
|
+
*/
|
|
23
|
+
export declare const escapeId: (id: string) => string;
|
|
24
|
+
/**
|
|
25
|
+
* Checks if a DuckDB type string represents a numeric type.
|
|
26
|
+
* Includes INTEGER, DECIMAL, FLOAT, REAL, and DOUBLE types.
|
|
27
|
+
*
|
|
28
|
+
* @param type - The DuckDB type string to check
|
|
29
|
+
* @returns True if the type is numeric, false otherwise
|
|
30
|
+
* @example
|
|
31
|
+
* isNumericDuckType('INTEGER') // Returns true
|
|
32
|
+
* isNumericDuckType('VARCHAR') // Returns false
|
|
33
|
+
*/
|
|
34
|
+
export declare const isNumericDuckType: (type: string) => boolean;
|
|
35
|
+
/**
|
|
36
|
+
* Extracts a numeric value from an Arrow Table at the specified column and row index.
|
|
37
|
+
* Handles both column name and index-based access. Converts BigInt values to numbers.
|
|
38
|
+
*
|
|
39
|
+
* @param res - The Arrow Table containing the data
|
|
40
|
+
* @param column - The column name or index (0-based) to read from. Defaults to first column (0)
|
|
41
|
+
* @param index - The row index (0-based) to read from. Defaults to first row (0)
|
|
42
|
+
* @returns The numeric value at the specified position, or NaN if the value is null/undefined
|
|
43
|
+
* @example
|
|
44
|
+
* const value = getColValAsNumber(table, "amount", 0)
|
|
45
|
+
*/
|
|
46
|
+
export declare function getColValAsNumber(res: arrow.Table, column?: string | number, index?: number): number;
|
|
47
|
+
//# sourceMappingURL=duckdb-utils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"duckdb-utils.d.ts","sourceRoot":"","sources":["../src/duckdb-utils.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC;;;;;;;;GAQG;AACH,eAAO,MAAM,SAAS,GAAI,KAAK,OAAO,WAErC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,QAAQ,GAAI,IAAI,MAAM,WAMlC,CAAC;AAEF;;;;;;;;;GASG;AACH,eAAO,MAAM,iBAAiB,GAAI,MAAM,MAAM,YAKjB,CAAC;AAE9B;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,GAAG,EAAE,KAAK,CAAC,KAAK,EAChB,MAAM,GAAE,MAAM,GAAG,MAAU,EAC3B,KAAK,SAAI,GACR,MAAM,CASR"}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Escapes a value for use in DuckDB SQL queries by wrapping it in single quotes
|
|
3
|
+
* and escaping any existing single quotes by doubling them.
|
|
4
|
+
*
|
|
5
|
+
* @param val - The value to escape. Will be converted to string if not already a string.
|
|
6
|
+
* @returns The escaped string value wrapped in single quotes.
|
|
7
|
+
* @example
|
|
8
|
+
* escapeVal("John's data") // Returns "'John''s data'"
|
|
9
|
+
*/
|
|
10
|
+
export const escapeVal = (val) => {
|
|
11
|
+
return `'${String(val).replace(/'/g, "''")}'`;
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Escapes an identifier (like table or column names) for use in DuckDB SQL queries
|
|
15
|
+
* by wrapping it in double quotes and escaping any existing double quotes by doubling them.
|
|
16
|
+
* If the identifier is already properly quoted, returns it as is.
|
|
17
|
+
*
|
|
18
|
+
* @param id - The identifier string to escape
|
|
19
|
+
* @returns The escaped identifier wrapped in double quotes
|
|
20
|
+
* @example
|
|
21
|
+
* escapeId("my_table") // Returns '"my_table"'
|
|
22
|
+
* escapeId("my""table") // Returns '"my""""table"'
|
|
23
|
+
*/
|
|
24
|
+
export const escapeId = (id) => {
|
|
25
|
+
const str = String(id);
|
|
26
|
+
if (str.startsWith('"') && str.endsWith('"')) {
|
|
27
|
+
return str;
|
|
28
|
+
}
|
|
29
|
+
return `"${str.replace(/"/g, '""')}"`;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* Checks if a DuckDB type string represents a numeric type.
|
|
33
|
+
* Includes INTEGER, DECIMAL, FLOAT, REAL, and DOUBLE types.
|
|
34
|
+
*
|
|
35
|
+
* @param type - The DuckDB type string to check
|
|
36
|
+
* @returns True if the type is numeric, false otherwise
|
|
37
|
+
* @example
|
|
38
|
+
* isNumericDuckType('INTEGER') // Returns true
|
|
39
|
+
* isNumericDuckType('VARCHAR') // Returns false
|
|
40
|
+
*/
|
|
41
|
+
export const isNumericDuckType = (type) => type.indexOf('INT') >= 0 ||
|
|
42
|
+
type.indexOf('DECIMAL') >= 0 ||
|
|
43
|
+
type.indexOf('FLOAT') >= 0 ||
|
|
44
|
+
type.indexOf('REAL') >= 0 ||
|
|
45
|
+
type.indexOf('DOUBLE') >= 0;
|
|
46
|
+
/**
|
|
47
|
+
* Extracts a numeric value from an Arrow Table at the specified column and row index.
|
|
48
|
+
* Handles both column name and index-based access. Converts BigInt values to numbers.
|
|
49
|
+
*
|
|
50
|
+
* @param res - The Arrow Table containing the data
|
|
51
|
+
* @param column - The column name or index (0-based) to read from. Defaults to first column (0)
|
|
52
|
+
* @param index - The row index (0-based) to read from. Defaults to first row (0)
|
|
53
|
+
* @returns The numeric value at the specified position, or NaN if the value is null/undefined
|
|
54
|
+
* @example
|
|
55
|
+
* const value = getColValAsNumber(table, "amount", 0)
|
|
56
|
+
*/
|
|
57
|
+
export function getColValAsNumber(res, column = 0, index = 0) {
|
|
58
|
+
const v = (typeof column === 'number' ? res.getChildAt(column) : res.getChild(column))?.get(index);
|
|
59
|
+
if (v === undefined || v === null) {
|
|
60
|
+
return NaN;
|
|
61
|
+
}
|
|
62
|
+
// if it's an array (can be returned by duckdb as bigint)
|
|
63
|
+
return Number(v[0] ?? v);
|
|
64
|
+
}
|
|
65
|
+
//# sourceMappingURL=duckdb-utils.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"duckdb-utils.js","sourceRoot":"","sources":["../src/duckdb-utils.ts"],"names":[],"mappings":"AAEA;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAY,EAAE,EAAE;IACxC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;AAChD,CAAC,CAAC;AAEF;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,EAAU,EAAE,EAAE;IACrC,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;IACvB,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7C,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;AACxC,CAAC,CAAC;AAEF;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAE,EAAE,CAChD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IACxB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC;IAC5B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;IAC1B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;IACzB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AAE9B;;;;;;;;;;GAUG;AACH,MAAM,UAAU,iBAAiB,CAC/B,GAAgB,EAChB,SAA0B,CAAC,EAC3B,KAAK,GAAG,CAAC;IAET,MAAM,CAAC,GAAG,CACR,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAC3E,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IACd,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QAClC,OAAO,GAAG,CAAC;IACb,CAAC;IACD,yDAAyD;IACzD,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AAC3B,CAAC","sourcesContent":["import * as arrow from 'apache-arrow';\n\n/**\n * Escapes a value for use in DuckDB SQL queries by wrapping it in single quotes\n * and escaping any existing single quotes by doubling them.\n *\n * @param val - The value to escape. Will be converted to string if not already a string.\n * @returns The escaped string value wrapped in single quotes.\n * @example\n * escapeVal(\"John's data\") // Returns \"'John''s data'\"\n */\nexport const escapeVal = (val: unknown) => {\n return `'${String(val).replace(/'/g, \"''\")}'`;\n};\n\n/**\n * Escapes an identifier (like table or column names) for use in DuckDB SQL queries\n * by wrapping it in double quotes and escaping any existing double quotes by doubling them.\n * If the identifier is already properly quoted, returns it as is.\n *\n * @param id - The identifier string to escape\n * @returns The escaped identifier wrapped in double quotes\n * @example\n * escapeId(\"my_table\") // Returns '\"my_table\"'\n * escapeId(\"my\"\"table\") // Returns '\"my\"\"\"\"table\"'\n */\nexport const escapeId = (id: string) => {\n const str = String(id);\n if (str.startsWith('\"') && str.endsWith('\"')) {\n return str;\n }\n return `\"${str.replace(/\"/g, '\"\"')}\"`;\n};\n\n/**\n * Checks if a DuckDB type string represents a numeric type.\n * Includes INTEGER, DECIMAL, FLOAT, REAL, and DOUBLE types.\n *\n * @param type - The DuckDB type string to check\n * @returns True if the type is numeric, false otherwise\n * @example\n * isNumericDuckType('INTEGER') // Returns true\n * isNumericDuckType('VARCHAR') // Returns false\n */\nexport const isNumericDuckType = (type: string) =>\n type.indexOf('INT') >= 0 ||\n type.indexOf('DECIMAL') >= 0 ||\n type.indexOf('FLOAT') >= 0 ||\n type.indexOf('REAL') >= 0 ||\n type.indexOf('DOUBLE') >= 0;\n\n/**\n * Extracts a numeric value from an Arrow Table at the specified column and row index.\n * Handles both column name and index-based access. Converts BigInt values to numbers.\n *\n * @param res - The Arrow Table containing the data\n * @param column - The column name or index (0-based) to read from. Defaults to first column (0)\n * @param index - The row index (0-based) to read from. Defaults to first row (0)\n * @returns The numeric value at the specified position, or NaN if the value is null/undefined\n * @example\n * const value = getColValAsNumber(table, \"amount\", 0)\n */\nexport function getColValAsNumber(\n res: arrow.Table,\n column: string | number = 0,\n index = 0,\n): number {\n const v = (\n typeof column === 'number' ? res.getChildAt(column) : res.getChild(column)\n )?.get(index);\n if (v === undefined || v === null) {\n return NaN;\n }\n // if it's an array (can be returned by duckdb as bigint)\n return Number(v[0] ?? v);\n}\n"]}
|
package/dist/exportToCsv.d.ts
CHANGED
|
@@ -1,2 +1,4 @@
|
|
|
1
|
-
export declare function
|
|
1
|
+
export declare function useExportToCsv(): {
|
|
2
|
+
exportToCsv: (query: string, fileName: string, pageSize?: number) => Promise<void>;
|
|
3
|
+
};
|
|
2
4
|
//# sourceMappingURL=exportToCsv.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exportToCsv.d.ts","sourceRoot":"","sources":["../src/exportToCsv.ts"],"names":[],"mappings":"AAGA,
|
|
1
|
+
{"version":3,"file":"exportToCsv.d.ts","sourceRoot":"","sources":["../src/exportToCsv.ts"],"names":[],"mappings":"AAGA,wBAAgB,cAAc;yBAGC,MAAM,YAAY,MAAM;EAgCtD"}
|
package/dist/exportToCsv.js
CHANGED
|
@@ -1,27 +1,32 @@
|
|
|
1
|
-
import {
|
|
2
|
-
export
|
|
3
|
-
const
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
1
|
+
import { useStoreWithDuckDb } from './DuckDbSlice';
|
|
2
|
+
export function useExportToCsv() {
|
|
3
|
+
const getConnector = useStoreWithDuckDb((state) => state.db.getConnector);
|
|
4
|
+
return {
|
|
5
|
+
exportToCsv: async (query, fileName, pageSize = 100000) => {
|
|
6
|
+
const dbConnector = await getConnector();
|
|
7
|
+
let offset = 0;
|
|
8
|
+
const blobs = [];
|
|
9
|
+
let headersAdded = false;
|
|
10
|
+
while (true) {
|
|
11
|
+
const currentQuery = `(
|
|
12
|
+
${query}
|
|
13
|
+
) LIMIT ${pageSize} OFFSET ${offset}`;
|
|
14
|
+
const results = await dbConnector.query(currentQuery);
|
|
15
|
+
// Check if we received any results; if not, we are done.
|
|
16
|
+
if (results.numRows === 0) {
|
|
17
|
+
break;
|
|
18
|
+
}
|
|
19
|
+
const csvChunk = convertToCsv(results, !headersAdded);
|
|
20
|
+
blobs.push(new Blob([csvChunk], { type: 'text/csv' }));
|
|
21
|
+
// Ensure that headers are not added in subsequent iterations
|
|
22
|
+
headersAdded = true;
|
|
23
|
+
// Increment offset to fetch the next chunk
|
|
24
|
+
offset += pageSize;
|
|
25
|
+
}
|
|
26
|
+
const fullCsvBlob = new Blob(blobs, { type: 'text/csv' });
|
|
27
|
+
downloadBlob(fullCsvBlob, fileName);
|
|
28
|
+
},
|
|
29
|
+
};
|
|
25
30
|
}
|
|
26
31
|
function convertToCsv(arrowTable, includeHeaders) {
|
|
27
32
|
// return includeHeaders
|
package/dist/exportToCsv.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exportToCsv.js","sourceRoot":"","sources":["../src/exportToCsv.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,
|
|
1
|
+
{"version":3,"file":"exportToCsv.js","sourceRoot":"","sources":["../src/exportToCsv.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,kBAAkB,EAAC,MAAM,eAAe,CAAC;AAEjD,MAAM,UAAU,cAAc;IAC5B,MAAM,YAAY,GAAG,kBAAkB,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;IAC1E,OAAO;QACL,WAAW,EAAE,KAAK,EAAE,KAAa,EAAE,QAAgB,EAAE,QAAQ,GAAG,MAAM,EAAE,EAAE;YACxE,MAAM,WAAW,GAAG,MAAM,YAAY,EAAE,CAAC;YAEzC,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,MAAM,KAAK,GAAW,EAAE,CAAC;YACzB,IAAI,YAAY,GAAG,KAAK,CAAC;YAEzB,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,YAAY,GAAG;YACjB,KAAK;kBACC,QAAQ,WAAW,MAAM,EAAE,CAAC;gBACtC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBAEtD,yDAAyD;gBACzD,IAAI,OAAO,CAAC,OAAO,KAAK,CAAC,EAAE,CAAC;oBAC1B,MAAM;gBACR,CAAC;gBAED,MAAM,QAAQ,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,YAAY,CAAC,CAAC;gBACtD,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAC,IAAI,EAAE,UAAU,EAAC,CAAC,CAAC,CAAC;gBAErD,6DAA6D;gBAC7D,YAAY,GAAG,IAAI,CAAC;gBAEpB,2CAA2C;gBAC3C,MAAM,IAAI,QAAQ,CAAC;YACrB,CAAC;YAED,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,KAAK,EAAE,EAAC,IAAI,EAAE,UAAU,EAAC,CAAC,CAAC;YACxD,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;QACtC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CACnB,UAAuB,EACvB,cAAuB;IAEvB,wBAAwB;IACxB,sCAAsC;IACtC,2CAA2C;IAE3C,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACxE,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CACtC,CAAC,GAAG,EAAE,UAAU,EAAE,EAAE;QAClB,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,GAAG;YAAE,GAAG,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC;QAC/B,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAAkC,CACnC,CAAC;IAEF,aAAa;IACb,IAAI,UAAU,GAAG,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAEtE,gBAAgB;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,WAAW;aACvB,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;YAClB,MAAM,SAAS,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;YAEpD,qEAAqE;YACrE,IAAI,SAAS,IAAI,IAAI;gBAAE,OAAO,EAAE,CAAC;YAEjC,+BAA+B;YAC/B,IAAI,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;YAErC,yEAAyE;YACzE,IACE,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAC1B,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC;gBAC1B,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EAC3B,CAAC;gBACD,YAAY,GAAG,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,GAAG,CAAC;YAC9D,CAAC;YAED,OAAO,YAAY,CAAC;QACtB,CAAC,CAAC;aACD,IAAI,CAAC,GAAG,CAAC,CAAC;QAEb,UAAU,IAAI,MAAM,GAAG,MAAM,CAAC;IAChC,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,YAAY,CAAC,IAAU,EAAE,QAAgB;IAChD,MAAM,GAAG,GAAG,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC;IACb,CAAC,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACtB,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC,CAAC,KAAK,EAAE,CAAC;IACV,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;AAC/B,CAAC","sourcesContent":["import * as arrow from 'apache-arrow';\nimport {useStoreWithDuckDb} from './DuckDbSlice';\n\nexport function useExportToCsv() {\n const getConnector = useStoreWithDuckDb((state) => state.db.getConnector);\n return {\n exportToCsv: async (query: string, fileName: string, pageSize = 100000) => {\n const dbConnector = await getConnector();\n\n let offset = 0;\n const blobs: Blob[] = [];\n let headersAdded = false;\n\n while (true) {\n const currentQuery = `(\n ${query}\n ) LIMIT ${pageSize} OFFSET ${offset}`;\n const results = await dbConnector.query(currentQuery);\n\n // Check if we received any results; if not, we are done.\n if (results.numRows === 0) {\n break;\n }\n\n const csvChunk = convertToCsv(results, !headersAdded);\n blobs.push(new Blob([csvChunk], {type: 'text/csv'}));\n\n // Ensure that headers are not added in subsequent iterations\n headersAdded = true;\n\n // Increment offset to fetch the next chunk\n offset += pageSize;\n }\n\n const fullCsvBlob = new Blob(blobs, {type: 'text/csv'});\n downloadBlob(fullCsvBlob, fileName);\n },\n };\n}\n\nfunction convertToCsv(\n arrowTable: arrow.Table,\n includeHeaders: boolean,\n): string {\n // return includeHeaders\n // ? csvFormat(arrowTable.toArray())\n // : csvFormatBody(arrowTable.toArray());\n\n const columnNames = arrowTable.schema.fields.map((field) => field.name);\n const columnsByName = columnNames.reduce(\n (acc, columnName) => {\n const col = arrowTable.getChild(columnName);\n if (col) acc[columnName] = col;\n return acc;\n },\n {} as Record<string, arrow.Vector>,\n );\n\n // Add header\n let csvContent = includeHeaders ? columnNames.join(',') + '\\r\\n' : '';\n\n // Add data rows\n for (let i = 0; i < arrowTable.numRows; i++) {\n const csvRow = columnNames\n .map((columnName) => {\n const cellValue = columnsByName[columnName]?.get(i);\n\n // If the cell value is null or undefined, set it to an empty string.\n if (cellValue == null) return '';\n\n // Convert cell value to string\n let cellValueStr = String(cellValue);\n\n // Escape double quotes and wrap cell value in double quotes if necessary\n if (\n cellValueStr.includes('\"') ||\n cellValueStr.includes(',') ||\n cellValueStr.includes('\\n')\n ) {\n cellValueStr = '\"' + cellValueStr.replace(/\"/g, '\"\"') + '\"';\n }\n\n return cellValueStr;\n })\n .join(',');\n\n csvContent += csvRow + '\\r\\n';\n }\n\n return csvContent;\n}\n\nfunction downloadBlob(blob: Blob, filename: string) {\n const url = URL.createObjectURL(blob);\n const a = document.createElement('a');\n a.href = url;\n a.download = filename;\n document.body.appendChild(a);\n a.click();\n URL.revokeObjectURL(url);\n document.body.removeChild(a);\n}\n"]}
|
package/dist/index.d.ts
CHANGED
|
@@ -2,10 +2,15 @@
|
|
|
2
2
|
* {@include ../README.md}
|
|
3
3
|
* @packageDocumentation
|
|
4
4
|
*/
|
|
5
|
-
export * from './duckdb';
|
|
6
5
|
export * from './types';
|
|
7
6
|
export * from './useDuckDb';
|
|
8
7
|
export * from './exportToCsv';
|
|
9
8
|
export * from './arrow-utils';
|
|
10
9
|
export * from './useSql';
|
|
10
|
+
export { createDuckDbSlice, createDefaultDuckDbConfig, DuckDbSliceConfig, type DuckDbSliceState, } from './DuckDbSlice';
|
|
11
|
+
export * from './connectors/DuckDbConnector';
|
|
12
|
+
export * from './connectors/WasmDuckDbConnector';
|
|
13
|
+
export * from './connectors/load/load';
|
|
14
|
+
export * from './duckdb-utils';
|
|
15
|
+
export { LoadFileOptions, SpatialLoadFileOptions, isSpatialLoadFileOptions, } from '@sqlrooms/project-config';
|
|
11
16
|
//# sourceMappingURL=index.d.ts.map
|