@yolean/evidence-universal-sql 0.0.0-stage → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +459 -0
- package/LICENSE.md +9 -0
- package/package.json +44 -3
- package/src/build-parquet.d.ts +20 -0
- package/src/build-parquet.js +251 -0
- package/src/build-parquet.spec.js +174 -0
- package/src/cache-duckdb.d.ts +38 -0
- package/src/cache-duckdb.js +195 -0
- package/src/calculateScore.d.ts +25 -0
- package/src/calculateScore.js +147 -0
- package/src/client-duckdb/both.d.ts +6 -0
- package/src/client-duckdb/both.js +104 -0
- package/src/client-duckdb/browser.d.ts +41 -0
- package/src/client-duckdb/browser.js +167 -0
- package/src/client-duckdb/node.d.ts +43 -0
- package/src/client-duckdb/node.js +166 -0
- package/src/index.d.ts +3 -0
- package/src/index.js +3 -0
- package/README.md +0 -3
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/* // generate with DESCRIBE SELECT column_type FROM test_all_types();
|
|
2
|
+
type DuckDBColumnType =
|
|
3
|
+
| `STRUCT(${string})`
|
|
4
|
+
| `${string}[]`
|
|
5
|
+
| 'BIGINT'
|
|
6
|
+
| 'BIT'
|
|
7
|
+
| 'BOOLEAN'
|
|
8
|
+
| 'BLOB'
|
|
9
|
+
| 'DATE'
|
|
10
|
+
| 'DOUBLE'
|
|
11
|
+
| `DECIMAL(${number},${number})`
|
|
12
|
+
| 'HUGEINT'
|
|
13
|
+
| 'INTEGER'
|
|
14
|
+
| 'INTERVAL'
|
|
15
|
+
| 'FLOAT'
|
|
16
|
+
| 'SMALLINT'
|
|
17
|
+
| 'TIME'
|
|
18
|
+
| 'TIMESTAMP'
|
|
19
|
+
| 'TIMESTAMP_S'
|
|
20
|
+
| 'TIMESTAMP_MS'
|
|
21
|
+
| 'TIMESTAMP_NS'
|
|
22
|
+
| 'TIME WITH TIME ZONE'
|
|
23
|
+
| 'TIMESTAMP WITH TIME ZONE'
|
|
24
|
+
| 'TINYINT'
|
|
25
|
+
| 'UBIGINT'
|
|
26
|
+
| 'UINTEGER'
|
|
27
|
+
| 'USMALLINT'
|
|
28
|
+
| 'UTINYINT'
|
|
29
|
+
| 'UUID'
|
|
30
|
+
| 'VARCHAR';
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @param {string} column_type
|
|
35
|
+
* @returns {column_type is `STRUCT(${string})` | `${string}[]`}
|
|
36
|
+
*/
|
|
37
|
+
function isObjectType(column_type) {
|
|
38
|
+
return column_type.startsWith('STRUCT') || column_type.endsWith('[]');
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* @param {string} column_type
|
|
43
|
+
* @returns {column_type is `DECIMAL(${number},${number})`}
|
|
44
|
+
*/
|
|
45
|
+
function isDecimalType(column_type) {
|
|
46
|
+
return column_type.startsWith('DECIMAL');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
*
|
|
51
|
+
* @param {string} column_type
|
|
52
|
+
* @returns {number}
|
|
53
|
+
*/
|
|
54
|
+
function columnTypeToScore(column_type) {
|
|
55
|
+
switch (column_type) {
|
|
56
|
+
case 'number':
|
|
57
|
+
return 12; // 8 bytes in a double + 4 bytes heap overhead?
|
|
58
|
+
case 'boolean':
|
|
59
|
+
return 4; // booleans are stored as 4 byte integers
|
|
60
|
+
case 'string':
|
|
61
|
+
return 2 * 15; // assume 15 character string
|
|
62
|
+
case 'date':
|
|
63
|
+
return 48; // dates use 48 bytes
|
|
64
|
+
default:
|
|
65
|
+
return 0;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
*
|
|
71
|
+
* @param {string} column_type
|
|
72
|
+
* @returns {'number' | 'boolean' | 'string' | 'date'}
|
|
73
|
+
*/
|
|
74
|
+
export function duckdbTypeToEvidenceType(column_type) {
|
|
75
|
+
// objects aren't handled well
|
|
76
|
+
if (isObjectType(column_type)) return 'string';
|
|
77
|
+
if (isDecimalType(column_type)) return 'number';
|
|
78
|
+
|
|
79
|
+
switch (column_type) {
|
|
80
|
+
case 'BOOLEAN':
|
|
81
|
+
return 'boolean';
|
|
82
|
+
case 'BIGINT':
|
|
83
|
+
case 'DOUBLE':
|
|
84
|
+
case 'FLOAT':
|
|
85
|
+
case 'INTEGER':
|
|
86
|
+
case 'SMALLINT':
|
|
87
|
+
case 'TINYINT':
|
|
88
|
+
case 'UBIGINT':
|
|
89
|
+
case 'UINTEGER':
|
|
90
|
+
case 'USMALLINT':
|
|
91
|
+
case 'UTINYINT':
|
|
92
|
+
case 'HUGEINT':
|
|
93
|
+
return 'number';
|
|
94
|
+
case 'UUID':
|
|
95
|
+
case 'VARCHAR':
|
|
96
|
+
return 'string';
|
|
97
|
+
case 'DATE':
|
|
98
|
+
case 'TIMESTAMP':
|
|
99
|
+
case 'TIMESTAMP_S':
|
|
100
|
+
case 'TIMESTAMP_MS':
|
|
101
|
+
case 'TIMESTAMP_NS':
|
|
102
|
+
case 'TIMESTAMP WITH TIME ZONE':
|
|
103
|
+
return 'date';
|
|
104
|
+
|
|
105
|
+
// the badlands
|
|
106
|
+
// we should probably convert these in the client library too
|
|
107
|
+
case 'INTERVAL': // return 'Uint32Array';
|
|
108
|
+
case 'TIME':
|
|
109
|
+
case 'TIME WITH TIME ZONE': // return 'bigint';
|
|
110
|
+
case 'BLOB':
|
|
111
|
+
case 'JSON':
|
|
112
|
+
case 'BIT': // return 'Uint8Array';
|
|
113
|
+
return 'string';
|
|
114
|
+
default:
|
|
115
|
+
// column_type should be `never`
|
|
116
|
+
console.error(`Column type ${column_type} is not supported`);
|
|
117
|
+
return 'string';
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* @param {{ name: string; evidenceType: string }[]} columns
|
|
123
|
+
* @returns {number}
|
|
124
|
+
*/
|
|
125
|
+
export function evidenceColumnsToScore(columns) {
|
|
126
|
+
let score = columns.length * 4; // some overhead for each column
|
|
127
|
+
|
|
128
|
+
for (const { type } of columns) {
|
|
129
|
+
score += columnTypeToScore(type);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return score;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* @param {{ name: string; type: string }[]} columns
|
|
137
|
+
* @returns {number}
|
|
138
|
+
*/
|
|
139
|
+
export function columnsToScore(columns) {
|
|
140
|
+
let score = columns.length * 4; // some overhead for each column
|
|
141
|
+
|
|
142
|
+
for (const { type } of columns) {
|
|
143
|
+
score += columnTypeToScore(duckdbTypeToEvidenceType(type));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
return score;
|
|
147
|
+
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { Type } from 'apache-arrow';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Converts an Apache Arrow type to an Evidence type.
|
|
5
|
+
*
|
|
6
|
+
* @param {import("apache-arrow").Type} type
|
|
7
|
+
*/
|
|
8
|
+
function apacheToEvidenceType(type) {
|
|
9
|
+
switch (
|
|
10
|
+
type.typeId // maybe just replace with `typeof`
|
|
11
|
+
) {
|
|
12
|
+
case Type.Date:
|
|
13
|
+
return 'date';
|
|
14
|
+
case Type.Float:
|
|
15
|
+
case Type.Int:
|
|
16
|
+
return 'number';
|
|
17
|
+
case Type.Bool:
|
|
18
|
+
return 'boolean';
|
|
19
|
+
case Type.Dictionary:
|
|
20
|
+
default:
|
|
21
|
+
return 'string';
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Converts an Apache Arrow table to a Javascript array.
|
|
27
|
+
* @param {import("apache-arrow").Table} table
|
|
28
|
+
* @returns {any[]}
|
|
29
|
+
*/
|
|
30
|
+
export function arrowTableToJSON(table) {
|
|
31
|
+
if (table == null) return [];
|
|
32
|
+
const arr = table.toArray();
|
|
33
|
+
|
|
34
|
+
Object.defineProperty(arr, '_evidenceColumnTypes', {
|
|
35
|
+
enumerable: false,
|
|
36
|
+
value: table.schema.fields.map((field) => ({
|
|
37
|
+
name: field.name,
|
|
38
|
+
evidenceType: apacheToEvidenceType(field.type),
|
|
39
|
+
typeFidelity: 'precise'
|
|
40
|
+
}))
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
const date_cols = table.schema.fields.filter((field) => field.type.typeId === Type.Date);
|
|
44
|
+
const list_cols = table.schema.fields.filter((field) => field.type.typeId === Type.List);
|
|
45
|
+
|
|
46
|
+
for (const row of arr) {
|
|
47
|
+
for (const col of date_cols) {
|
|
48
|
+
row[col.name] = new Date(row[col.name]);
|
|
49
|
+
}
|
|
50
|
+
for (const col of list_cols) {
|
|
51
|
+
row[col.name] = arrowVectorToJSON(row[col.name]);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return arr;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Converts an Apache Arrow vector to a Javascript array.
|
|
60
|
+
* @param {import("apache-arrow").Vector} vector
|
|
61
|
+
* @returns {any[]}
|
|
62
|
+
*/
|
|
63
|
+
function arrowVectorToJSON(vector) {
|
|
64
|
+
if (vector == null) return [];
|
|
65
|
+
const arr = vector.toArray();
|
|
66
|
+
|
|
67
|
+
const date_cols = vector.type?.children?.filter((field) => field.type.typeId === Type.Date) ?? [];
|
|
68
|
+
const list_cols = vector.type?.children?.filter((field) => field.type.typeId === Type.List) ?? [];
|
|
69
|
+
|
|
70
|
+
for (const row of arr) {
|
|
71
|
+
for (const col of date_cols) {
|
|
72
|
+
row[col.name] = new Date(row[col.name]);
|
|
73
|
+
}
|
|
74
|
+
for (const col of list_cols) {
|
|
75
|
+
row[col.name] = arrowVectorToJSON(row[col.name]);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return arr;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Creates a new Promise object and returns it along with its resolve and reject functions.
|
|
84
|
+
*
|
|
85
|
+
* @return {{resolve: CallableFunction, reject: CallableFunction, promise: Promise<void>}} An object containing the resolve and reject functions, as well as the Promise object.
|
|
86
|
+
*/
|
|
87
|
+
export function getPromise() {
|
|
88
|
+
let resolve, reject;
|
|
89
|
+
let promise = new Promise((res, rej) => {
|
|
90
|
+
resolve = res;
|
|
91
|
+
reject = rej;
|
|
92
|
+
});
|
|
93
|
+
return { resolve, reject, promise };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function withTimeout(p) {
|
|
97
|
+
return Promise.race([
|
|
98
|
+
p,
|
|
99
|
+
new Promise((_, rej) =>
|
|
100
|
+
// If the database isn't initialized after 5 seconds, throw an error
|
|
101
|
+
setTimeout(() => rej(new Error('Timeout while initializing database')), 5000)
|
|
102
|
+
)
|
|
103
|
+
]);
|
|
104
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export { tableFromIPC } from 'apache-arrow';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Initializes the database.
|
|
5
|
+
*
|
|
6
|
+
* @returns {Promise<void>}
|
|
7
|
+
*/
|
|
8
|
+
export function initDB(): Promise<void>;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Updates the duckdb search path to include only the list of included schemas
|
|
12
|
+
* @param {string[]} schemas
|
|
13
|
+
* @returns {Promise<void>}
|
|
14
|
+
*/
|
|
15
|
+
export function updateSearchPath(schemas): Promise<void>;
|
|
16
|
+
/**
|
|
17
|
+
* Queries the database with the given SQL statement.
|
|
18
|
+
*
|
|
19
|
+
* @param {string} sql
|
|
20
|
+
* @returns {Promise<Record<string, unknown>[]>}
|
|
21
|
+
*/
|
|
22
|
+
export function query(sql: string): Promise<Record<string, unknown>[]>;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Adds a new view to the database, pointing to the provided parquet URLs.
|
|
26
|
+
*
|
|
27
|
+
* @param {Record<string, string[]>} urls
|
|
28
|
+
* @param {{ append?: boolean, addBasePath?: (path: string) => string }} [opts]
|
|
29
|
+
* @returns {Promise<void>}
|
|
30
|
+
*/
|
|
31
|
+
export function setParquetURLs(
|
|
32
|
+
urls: Record<string, string[]>,
|
|
33
|
+
opts?: { append?: boolean; addBasePath?: (path: string) => string }
|
|
34
|
+
): Promise<void>;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Converts an Apache Arrow table to a Javascript array.
|
|
38
|
+
* @param {import("apache-arrow").Table} table
|
|
39
|
+
* @returns {any[]}
|
|
40
|
+
*/
|
|
41
|
+
export function arrowTableToJSON(table: import('apache-arrow').Table): any[];
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { arrowTableToJSON, getPromise, withTimeout } from './both.js';
|
|
2
|
+
import {
|
|
3
|
+
AsyncDuckDB,
|
|
4
|
+
ConsoleLogger,
|
|
5
|
+
DuckDBDataProtocol,
|
|
6
|
+
getPlatformFeatures,
|
|
7
|
+
VoidLogger
|
|
8
|
+
} from '@duckdb/duckdb-wasm';
|
|
9
|
+
|
|
10
|
+
export { tableFromIPC } from 'apache-arrow';
|
|
11
|
+
|
|
12
|
+
/** @type {import("@duckdb/duckdb-wasm").AsyncDuckDB} */
|
|
13
|
+
let db;
|
|
14
|
+
|
|
15
|
+
/** @type {import("@duckdb/duckdb-wasm").AsyncDuckDBConnection} */
|
|
16
|
+
let connection;
|
|
17
|
+
|
|
18
|
+
const { resolve: resolveInit, reject: rejectInit, promise: initPromise } = getPromise();
|
|
19
|
+
const { resolve: resolveTables, reject: rejectTables, promise: tablesPromise } = getPromise();
|
|
20
|
+
let initializing = false;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Initializes the database.
|
|
24
|
+
*
|
|
25
|
+
* @returns {Promise<void>}
|
|
26
|
+
*/
|
|
27
|
+
export async function initDB() {
|
|
28
|
+
// If the database is already available, don't do anything
|
|
29
|
+
if (db) return;
|
|
30
|
+
|
|
31
|
+
// If the database is already initializing, don't try to do it twice
|
|
32
|
+
// Instead, let the call wait for the initPromise
|
|
33
|
+
if (initializing) return withTimeout(initPromise);
|
|
34
|
+
// This call is the first (to execute), don't let anybody else try
|
|
35
|
+
// to initialize the database
|
|
36
|
+
initializing = true;
|
|
37
|
+
try {
|
|
38
|
+
const useEh = await getPlatformFeatures().then((x) => x.wasmExceptions);
|
|
39
|
+
|
|
40
|
+
const DUCKDB_CONFIG = useEh
|
|
41
|
+
? {
|
|
42
|
+
mainModule: (await import('@duckdb/duckdb-wasm/dist/duckdb-eh.wasm?url')).default,
|
|
43
|
+
mainWorker: (await import('@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js?worker'))
|
|
44
|
+
.default
|
|
45
|
+
}
|
|
46
|
+
: {
|
|
47
|
+
mainModule: (await import('@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm?url')).default,
|
|
48
|
+
mainWorker: (await import('@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js?worker'))
|
|
49
|
+
.default
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const logger = import.meta.env.VITE_EVIDENCE_DEBUG ? new ConsoleLogger() : new VoidLogger();
|
|
53
|
+
const worker = new DUCKDB_CONFIG.mainWorker();
|
|
54
|
+
|
|
55
|
+
// use an intermediate variable to prevent db from being a not-ready database
|
|
56
|
+
const _db = new AsyncDuckDB(logger, worker);
|
|
57
|
+
window[Symbol.for('EVIDENCE_QUERY_ENGINE')] = _db;
|
|
58
|
+
|
|
59
|
+
await _db.instantiate(DUCKDB_CONFIG.mainModule);
|
|
60
|
+
db = _db;
|
|
61
|
+
|
|
62
|
+
await db.open({
|
|
63
|
+
query: {
|
|
64
|
+
castBigIntToDouble: true,
|
|
65
|
+
castTimestampToDate: true,
|
|
66
|
+
castDecimalToDouble: true,
|
|
67
|
+
castDurationToTime64: true
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
connection = await db.connect();
|
|
71
|
+
|
|
72
|
+
// https://duckdb.org/2024/09/09/announcing-duckdb-110.html#breaking-sql-changes
|
|
73
|
+
await connection.query('SET ieee_floating_point_ops = false;');
|
|
74
|
+
// https://duckdb.org/2024/02/13/announcing-duckdb-0100.html#breaking-sql-changes
|
|
75
|
+
await connection.query('SET old_implicit_casting = true;');
|
|
76
|
+
|
|
77
|
+
resolveInit();
|
|
78
|
+
} catch (e) {
|
|
79
|
+
rejectInit(e);
|
|
80
|
+
throw e;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Updates the duckdb search path to include only the list of included schemas
|
|
86
|
+
* @param {string[]} schemas
|
|
87
|
+
* @returns {Promise<void>}
|
|
88
|
+
*/
|
|
89
|
+
export async function updateSearchPath(schemas) {
|
|
90
|
+
if (!db) await initDB();
|
|
91
|
+
|
|
92
|
+
await connection.query(`PRAGMA search_path='${schemas.join(',')}'`);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* @param {string} targetGlob
|
|
97
|
+
*/
|
|
98
|
+
export async function emptyDbFs(targetGlob) {
|
|
99
|
+
await db.flushFiles();
|
|
100
|
+
for (const f of await db.globFiles(targetGlob)) {
|
|
101
|
+
await db.dropFile(f.fileName);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Adds a new view to the database, pointing to the provided parquet URL.
|
|
107
|
+
* @param {Record<string, string[]>} urls
|
|
108
|
+
* @param {{ append?: boolean, addBasePath?: (path: string) => string }} [opts]
|
|
109
|
+
* @returns {Promise<void>}
|
|
110
|
+
*/
|
|
111
|
+
export async function setParquetURLs(urls, { append, addBasePath = (x) => x } = {}) {
|
|
112
|
+
if (!db) await initDB();
|
|
113
|
+
if (!append) await emptyDbFs('*');
|
|
114
|
+
if (import.meta.env.VITE_EVIDENCE_DEBUG) console.debug('Updating Parquet URLs');
|
|
115
|
+
|
|
116
|
+
try {
|
|
117
|
+
for (const source in urls) {
|
|
118
|
+
await connection.query(`CREATE SCHEMA IF NOT EXISTS "${source}";`);
|
|
119
|
+
for (const url of urls[source]) {
|
|
120
|
+
const table = url.split(/[\\/]/).at(-1).slice(0, -'.parquet'.length);
|
|
121
|
+
const file_name = `${source}_${table}.parquet`;
|
|
122
|
+
let path = url;
|
|
123
|
+
|
|
124
|
+
if (!url.startsWith('http') && !url.startsWith('/')) {
|
|
125
|
+
// URL Needs to be absolute
|
|
126
|
+
path = `/${url}`;
|
|
127
|
+
}
|
|
128
|
+
// Sveltekit doesn't like referencing the static dir expilcitly
|
|
129
|
+
if (path.startsWith('/static')) path = path.substring(7);
|
|
130
|
+
|
|
131
|
+
if (append) {
|
|
132
|
+
await emptyDbFs(file_name);
|
|
133
|
+
await emptyDbFs(url);
|
|
134
|
+
}
|
|
135
|
+
await db.registerFileURL(file_name, addBasePath(path), DuckDBDataProtocol.HTTP, false);
|
|
136
|
+
await connection.query(
|
|
137
|
+
`CREATE OR REPLACE VIEW "${source}"."${table}" AS (SELECT * FROM read_parquet('${file_name}'));`
|
|
138
|
+
);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
resolveTables();
|
|
142
|
+
} catch (e) {
|
|
143
|
+
rejectTables(e);
|
|
144
|
+
console.error(`Error encountered while updating Parquet URLs`, e);
|
|
145
|
+
throw e;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Queries the database with the given SQL statement.
|
|
151
|
+
*
|
|
152
|
+
* @param {string} sql
|
|
153
|
+
* @returns {Promise<Record<string, unknown>[]>}
|
|
154
|
+
*/
|
|
155
|
+
export async function query(sql) {
|
|
156
|
+
// After this point, the database has been initialized
|
|
157
|
+
if (!db) await initDB();
|
|
158
|
+
// We need to wait for tables to be available
|
|
159
|
+
await withTimeout(tablesPromise);
|
|
160
|
+
|
|
161
|
+
// Now we can safely execute our query
|
|
162
|
+
const res = await connection.query(sql).then(arrowTableToJSON);
|
|
163
|
+
|
|
164
|
+
return res;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
export { arrowTableToJSON };
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export { tableFromIPC } from 'apache-arrow';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Initializes the database.
|
|
5
|
+
*
|
|
6
|
+
* @returns {Promise<void>}
|
|
7
|
+
*/
|
|
8
|
+
export function initDB(): Promise<void>;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Updates the duckdb search path to include only the list of included schemas
|
|
12
|
+
* @param {string[]} schemas
|
|
13
|
+
* @returns {Promise<void>}
|
|
14
|
+
*/
|
|
15
|
+
export function updateSearchPath(schemas): Promise<void>;
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Queries the database with the given SQL statement.
|
|
19
|
+
*
|
|
20
|
+
* @param {string} sql
|
|
21
|
+
* @param {{ route_hash: string, query_name: string, prerendering: boolean }} [cache_options]
|
|
22
|
+
* @returns {Record<string, unknown[]>}
|
|
23
|
+
*/
|
|
24
|
+
export function query(
|
|
25
|
+
sql: string,
|
|
26
|
+
cache_options?: { route_hash: string; query_name: string; prerendering: boolean }
|
|
27
|
+
): Record<string, unknown>[];
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Adds a new view to the database, pointing to the provided parquet URLs.
|
|
31
|
+
*
|
|
32
|
+
* @param {Record<string, string[]>} urls
|
|
33
|
+
* @param {boolean} [append]
|
|
34
|
+
* @returns {void}
|
|
35
|
+
*/
|
|
36
|
+
export function setParquetURLs(urls: Record<string, string[]>, append?: boolean): void;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Converts an Apache Arrow table to a Javascript array.
|
|
40
|
+
* @param {import("apache-arrow").Table} table
|
|
41
|
+
* @returns {any[]}
|
|
42
|
+
*/
|
|
43
|
+
export function arrowTableToJSON(table: import('apache-arrow').Table): any[];
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { arrowTableToJSON, getPromise } from './both.js';
|
|
2
|
+
import {
|
|
3
|
+
ConsoleLogger,
|
|
4
|
+
createDuckDB,
|
|
5
|
+
DuckDBDataProtocol,
|
|
6
|
+
NODE_RUNTIME,
|
|
7
|
+
VoidLogger
|
|
8
|
+
} from '@duckdb/duckdb-wasm/dist/duckdb-node-blocking';
|
|
9
|
+
import { createRequire } from 'module';
|
|
10
|
+
import path, { dirname, resolve } from 'path';
|
|
11
|
+
import { cache_for_hash, get_arrow_if_sql_already_run } from '../cache-duckdb.js';
|
|
12
|
+
import { withTimeout } from './both.js';
|
|
13
|
+
|
|
14
|
+
const require = createRequire(import.meta.url);
|
|
15
|
+
const DUCKDB_DIST = dirname(require.resolve('@duckdb/duckdb-wasm'));
|
|
16
|
+
|
|
17
|
+
export { tableFromIPC } from 'apache-arrow';
|
|
18
|
+
|
|
19
|
+
/** @type {import("@duckdb/duckdb-wasm/dist/types/src/bindings/bindings_node_base").DuckDBNodeBindings} */
|
|
20
|
+
let db;
|
|
21
|
+
|
|
22
|
+
/** @type {import("@duckdb/duckdb-wasm/dist/types/src/bindings/connection").DuckDBConnection} */
|
|
23
|
+
let connection;
|
|
24
|
+
|
|
25
|
+
const { resolve: resolveInit, reject: rejectInit, promise: initPromise } = getPromise();
|
|
26
|
+
let initializing = false;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Initializes the database.
|
|
30
|
+
*
|
|
31
|
+
* @returns {Promise<void>}
|
|
32
|
+
*/
|
|
33
|
+
export async function initDB() {
|
|
34
|
+
// If the database is already available, don't do anything
|
|
35
|
+
if (db) return;
|
|
36
|
+
|
|
37
|
+
// If the database is already initializing, don't try to do it twice
|
|
38
|
+
// Instead, let the call wait for the initPromise
|
|
39
|
+
if (initializing) return withTimeout(initPromise);
|
|
40
|
+
|
|
41
|
+
// This call is the first (to execute), don't let anybody else try
|
|
42
|
+
// to initialize the database
|
|
43
|
+
initializing = true;
|
|
44
|
+
|
|
45
|
+
try {
|
|
46
|
+
const DUCKDB_BUNDLES = {
|
|
47
|
+
mvp: {
|
|
48
|
+
mainModule: resolve(DUCKDB_DIST, './duckdb-mvp.wasm'),
|
|
49
|
+
mainWorker: resolve(DUCKDB_DIST, './duckdb-node-mvp.worker.cjs')
|
|
50
|
+
},
|
|
51
|
+
eh: {
|
|
52
|
+
mainModule: resolve(DUCKDB_DIST, './duckdb-eh.wasm'),
|
|
53
|
+
mainWorker: resolve(DUCKDB_DIST, './duckdb-node-eh.worker.cjs')
|
|
54
|
+
}
|
|
55
|
+
};
|
|
56
|
+
const logger = process.env.VITE_EVIDENCE_DEBUG ? new ConsoleLogger() : new VoidLogger();
|
|
57
|
+
|
|
58
|
+
// and synchronous database
|
|
59
|
+
db = await createDuckDB(DUCKDB_BUNDLES, logger, NODE_RUNTIME);
|
|
60
|
+
await db.instantiate();
|
|
61
|
+
db.open({
|
|
62
|
+
query: {
|
|
63
|
+
castBigIntToDouble: true,
|
|
64
|
+
castTimestampToDate: true,
|
|
65
|
+
castDecimalToDouble: true,
|
|
66
|
+
castDurationToTime64: true
|
|
67
|
+
}
|
|
68
|
+
});
|
|
69
|
+
connection = db.connect();
|
|
70
|
+
|
|
71
|
+
// https://duckdb.org/2024/09/09/announcing-duckdb-110.html#breaking-sql-changes
|
|
72
|
+
await connection.query('SET ieee_floating_point_ops = false;');
|
|
73
|
+
// https://duckdb.org/2024/02/13/announcing-duckdb-0100.html#breaking-sql-changes
|
|
74
|
+
await connection.query('SET old_implicit_casting = true;');
|
|
75
|
+
|
|
76
|
+
resolveInit();
|
|
77
|
+
} catch (e) {
|
|
78
|
+
rejectInit(e);
|
|
79
|
+
throw e;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Updates the duckdb search path to include only the list of included schemas
|
|
85
|
+
* @param {string[]} schemas
|
|
86
|
+
* @returns {void}
|
|
87
|
+
*/
|
|
88
|
+
export function updateSearchPath(schemas) {
|
|
89
|
+
connection.query(`PRAGMA search_path='${schemas.join(',')}'`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* @param {string} targetGlob
|
|
94
|
+
*/
|
|
95
|
+
export async function emptyDbFs(targetGlob) {
|
|
96
|
+
await db.flushFiles();
|
|
97
|
+
for (const f of db.globFiles(targetGlob)) {
|
|
98
|
+
await db.dropFile(f.fileName);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Adds a new view to the database, pointing to the provided parquet URLs.
|
|
104
|
+
*
|
|
105
|
+
* @param {Record<string, string[]>} urls
|
|
106
|
+
* @param {boolean} [append]
|
|
107
|
+
* @returns {void}
|
|
108
|
+
*/
|
|
109
|
+
export async function setParquetURLs(urls, append = false) {
|
|
110
|
+
if (!append) await emptyDbFs('*');
|
|
111
|
+
|
|
112
|
+
const pathDelimiterRegex = /[\\/]/;
|
|
113
|
+
|
|
114
|
+
if (process.env.VITE_EVIDENCE_DEBUG) console.log(`Updating Parquet URLs`);
|
|
115
|
+
for (const source in urls) {
|
|
116
|
+
connection.query(`CREATE SCHEMA IF NOT EXISTS "${source}";`);
|
|
117
|
+
for (const url of urls[source]) {
|
|
118
|
+
const table = url.split(pathDelimiterRegex).at(-1).slice(0, -'.parquet'.length);
|
|
119
|
+
const file_name = `${source}_${table}.parquet`;
|
|
120
|
+
if (append) {
|
|
121
|
+
await emptyDbFs(file_name);
|
|
122
|
+
await emptyDbFs(url);
|
|
123
|
+
}
|
|
124
|
+
db.registerFileURL(
|
|
125
|
+
file_name,
|
|
126
|
+
url.split(pathDelimiterRegex).join(path.sep),
|
|
127
|
+
DuckDBDataProtocol.NODE_FS,
|
|
128
|
+
false
|
|
129
|
+
);
|
|
130
|
+
connection.query(
|
|
131
|
+
`CREATE OR REPLACE VIEW "${source}"."${table}" AS (SELECT * FROM read_parquet('${file_name}'));`
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Queries the database with the given SQL statement.
|
|
139
|
+
*
|
|
140
|
+
* @param {string} sql
|
|
141
|
+
* @param {Parameters<typeof cache_for_hash>[2]} [cache_options]
|
|
142
|
+
* @returns {Record<string, unknown>[]}
|
|
143
|
+
*/
|
|
144
|
+
export function query(sql, cache_options) {
|
|
145
|
+
let result;
|
|
146
|
+
|
|
147
|
+
// only cache during build, because
|
|
148
|
+
// parquet can/will eventually change during dev
|
|
149
|
+
if (cache_options?.prerendering) {
|
|
150
|
+
result = get_arrow_if_sql_already_run(sql, cache_options.scope);
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// TODO: This just fails, where is the process going?
|
|
154
|
+
// if cache missed, fallback to querying
|
|
155
|
+
if (!result) {
|
|
156
|
+
result = connection.query(sql);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (cache_options) {
|
|
160
|
+
cache_for_hash(sql, result, cache_options);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
return arrowTableToJSON(result);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export { arrowTableToJSON };
|
package/src/index.d.ts
ADDED
package/src/index.js
ADDED
package/README.md
DELETED