@yolean/evidence-universal-sql 0.0.0-stage → 3.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +451 -0
- package/LICENSE.md +9 -0
- package/package.json +44 -3
- package/src/build-parquet.d.ts +20 -0
- package/src/build-parquet.js +251 -0
- package/src/build-parquet.spec.js +174 -0
- package/src/cache-duckdb.d.ts +32 -0
- package/src/cache-duckdb.js +183 -0
- package/src/calculateScore.d.ts +25 -0
- package/src/calculateScore.js +147 -0
- package/src/client-duckdb/both.d.ts +6 -0
- package/src/client-duckdb/both.js +104 -0
- package/src/client-duckdb/browser.d.ts +41 -0
- package/src/client-duckdb/browser.js +167 -0
- package/src/client-duckdb/node.d.ts +43 -0
- package/src/client-duckdb/node.js +166 -0
- package/src/index.d.ts +3 -0
- package/src/index.js +3 -0
- package/README.md +0 -3
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
import {
|
|
2
|
+
tableFromArrays,
|
|
3
|
+
tableToIPC,
|
|
4
|
+
vectorFromArray,
|
|
5
|
+
Float64,
|
|
6
|
+
Utf8,
|
|
7
|
+
Bool,
|
|
8
|
+
TimestampMillisecond
|
|
9
|
+
} from 'apache-arrow';
|
|
10
|
+
import {
|
|
11
|
+
Compression,
|
|
12
|
+
writeParquet,
|
|
13
|
+
WriterPropertiesBuilder,
|
|
14
|
+
Table
|
|
15
|
+
} from 'parquet-wasm/node/arrow1.js';
|
|
16
|
+
import fs from 'fs/promises';
|
|
17
|
+
import path from 'path';
|
|
18
|
+
// using node-async.js makes CLI command hang - why??
|
|
19
|
+
import { emptyDbFs, initDB, query } from './client-duckdb/node.js';
|
|
20
|
+
import chunk from 'lodash.chunk';
|
|
21
|
+
import { columnsToScore } from './calculateScore.js';
|
|
22
|
+
import chalk from 'chalk';
|
|
23
|
+
import { log } from '@evidence-dev/sdk/logger';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* @param {{name: string, evidenceType: string}} column
|
|
27
|
+
* @param {any[]} rawValues
|
|
28
|
+
* @returns {import("apache-arrow").Vector}
|
|
29
|
+
*/
|
|
30
|
+
function convertArrayToVector(column, rawValues) {
|
|
31
|
+
switch (column.evidenceType) {
|
|
32
|
+
case 'number':
|
|
33
|
+
return vectorFromArray(rawValues, new Float64());
|
|
34
|
+
case 'string':
|
|
35
|
+
return vectorFromArray(rawValues, new Utf8());
|
|
36
|
+
case 'date':
|
|
37
|
+
if (!rawValues.some((v) => v !== null)) {
|
|
38
|
+
// All null date columns error out, so we have to do this
|
|
39
|
+
// https://github.com/evidence-dev/evidence/issues/2897
|
|
40
|
+
// separate reason than the bool version
|
|
41
|
+
console.warn(
|
|
42
|
+
chalk.yellow(
|
|
43
|
+
`\nWarning: Column "${column.name}" (type Date) contains only null values so it has been cast to Float64`
|
|
44
|
+
)
|
|
45
|
+
);
|
|
46
|
+
return vectorFromArray(rawValues, new Float64());
|
|
47
|
+
}
|
|
48
|
+
// TODO: What gives with timezones
|
|
49
|
+
return vectorFromArray(rawValues, new TimestampMillisecond());
|
|
50
|
+
case 'boolean':
|
|
51
|
+
if (!rawValues.some((v) => v !== null)) {
|
|
52
|
+
// All null bool columns error out, so we have to do this
|
|
53
|
+
// https://github.com/evidence-dev/evidence/issues/1504
|
|
54
|
+
console.warn(
|
|
55
|
+
chalk.yellow(
|
|
56
|
+
`\nWarning: Column "${column.name}" (type Bool) contains only null values so it has been cast to Float64`
|
|
57
|
+
)
|
|
58
|
+
);
|
|
59
|
+
return vectorFromArray(rawValues, new Float64());
|
|
60
|
+
}
|
|
61
|
+
return vectorFromArray(rawValues, new Bool());
|
|
62
|
+
default:
|
|
63
|
+
throw new Error(
|
|
64
|
+
'Unrecognized EvidenceType: ' +
|
|
65
|
+
column.evidenceType +
|
|
66
|
+
'\n This is likely an error in a datasource connector.'
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* @template T
|
|
73
|
+
* @param {{name: string, evidenceType: string}[]} columns
|
|
74
|
+
* @param {Generator<T[] | Promise<T[]> | T[] | Promise<T[]>} data
|
|
75
|
+
* @param {string} tmpDir
|
|
76
|
+
* @param {string} outDir
|
|
77
|
+
* @param {string} outputFilename
|
|
78
|
+
* @param {number} [batchSize]
|
|
79
|
+
* @returns {Promise<number>} Number of rows
|
|
80
|
+
*/
|
|
81
|
+
export async function buildMultipartParquet(
|
|
82
|
+
columns,
|
|
83
|
+
data,
|
|
84
|
+
tmpDir,
|
|
85
|
+
outDir,
|
|
86
|
+
outputFilename,
|
|
87
|
+
batchSize = 1000000
|
|
88
|
+
) {
|
|
89
|
+
log.debug(`Building parquet file ${outputFilename}`);
|
|
90
|
+
let { meta: fn_meta, done: fn_done } = log.measure('buildMultipartParquet');
|
|
91
|
+
fn_meta('output filename', outputFilename);
|
|
92
|
+
|
|
93
|
+
let batchNum = 0;
|
|
94
|
+
const outputSubpath = outputFilename.split('.parquet')[0];
|
|
95
|
+
let tmpFilenames = [];
|
|
96
|
+
let rowCount = 0;
|
|
97
|
+
|
|
98
|
+
// Unique id for this build run to avoid reusing the same temp filenames across
|
|
99
|
+
// consecutive builds in the same session. DuckDB (wasm) may keep handles or
|
|
100
|
+
// mappings to filenames; creating unique filenames per build avoids races or
|
|
101
|
+
// protocol errors when files are replaced.
|
|
102
|
+
const buildId = `${Date.now()}.${Math.random().toString(36).slice(2, 8)}`;
|
|
103
|
+
|
|
104
|
+
const flush = async (results) => {
|
|
105
|
+
log.debug(`Flushing batch ${batchNum} with ${results.length} rows`);
|
|
106
|
+
let { meta, done } = log.measure('flush');
|
|
107
|
+
meta('batch number', batchNum);
|
|
108
|
+
|
|
109
|
+
// Convert JS Objects -> Arrow
|
|
110
|
+
const vectorized = Object.fromEntries(
|
|
111
|
+
columns.map((c) => [
|
|
112
|
+
c.name,
|
|
113
|
+
convertArrayToVector(
|
|
114
|
+
c,
|
|
115
|
+
results.map((i) => i[c.name] ?? null)
|
|
116
|
+
)
|
|
117
|
+
])
|
|
118
|
+
);
|
|
119
|
+
const table = tableFromArrays(vectorized);
|
|
120
|
+
for (const field of table.schema.fields) {
|
|
121
|
+
field.nullable = true;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Converts the arrow table to a buffer
|
|
125
|
+
const IPC = tableToIPC(table, 'stream');
|
|
126
|
+
|
|
127
|
+
const writerProperties = new WriterPropertiesBuilder().setCompression(Compression.ZSTD).build();
|
|
128
|
+
// Converts the arrow buffer to a parquet buffer
|
|
129
|
+
// This can be slow; it includes the compression step
|
|
130
|
+
|
|
131
|
+
const parquetBuffer = writeParquet(Table.fromIPCStream(IPC), writerProperties);
|
|
132
|
+
|
|
133
|
+
const finalTempFilename = path.join(tmpDir, `${outputSubpath}.${buildId}.${batchNum}.parquet`);
|
|
134
|
+
// Write to a unique temporary file first, then atomically rename into place.
|
|
135
|
+
// This avoids a race where DuckDB may attempt to read a file while it's being written
|
|
136
|
+
// (e.g., during rapid dev refreshes). Using an atomic rename ensures readers only
|
|
137
|
+
// see the completed file.
|
|
138
|
+
const uniqueSuffix = `.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
139
|
+
const writeTempFilename = finalTempFilename + uniqueSuffix;
|
|
140
|
+
|
|
141
|
+
await fs.mkdir(path.dirname(finalTempFilename), { recursive: true });
|
|
142
|
+
await fs.writeFile(writeTempFilename, parquetBuffer);
|
|
143
|
+
// Atomically move into final location
|
|
144
|
+
await fs.rename(writeTempFilename, finalTempFilename);
|
|
145
|
+
|
|
146
|
+
tmpFilenames.push(finalTempFilename);
|
|
147
|
+
rowCount += results.length;
|
|
148
|
+
|
|
149
|
+
done();
|
|
150
|
+
log.debug(`Flushed batch ${batchNum} with ${results.length} rows`);
|
|
151
|
+
|
|
152
|
+
batchNum++;
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
if (typeof data === 'function') data = data();
|
|
156
|
+
if (data instanceof Promise) data = await data;
|
|
157
|
+
// Data is an array, but not a nested one
|
|
158
|
+
// We expect a set of result sets;
|
|
159
|
+
if (Array.isArray(data) && !Array.isArray(data[0])) data = [data];
|
|
160
|
+
if (Array.isArray(data)) {
|
|
161
|
+
const arrays = data;
|
|
162
|
+
data = (function* () {
|
|
163
|
+
for (const results of arrays) {
|
|
164
|
+
for (const batch of chunk(results, batchSize)) {
|
|
165
|
+
yield batch;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
})();
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
log.debug('Reading rows from a generator object');
|
|
172
|
+
let { meta, done } = log.measure('buildMultipartParquet');
|
|
173
|
+
meta('batch number', batchNum);
|
|
174
|
+
|
|
175
|
+
const currentBatch = [];
|
|
176
|
+
for await (const results of data) {
|
|
177
|
+
for (const result of results) currentBatch.push(result);
|
|
178
|
+
|
|
179
|
+
if (currentBatch.length >= batchSize) {
|
|
180
|
+
done();
|
|
181
|
+
log.debug(`Flushing batch ${batchNum} with ${currentBatch.length} rows`);
|
|
182
|
+
await flush(currentBatch);
|
|
183
|
+
currentBatch.length = 0;
|
|
184
|
+
({ meta, done } = log.measure('buildMultipartParquet'));
|
|
185
|
+
meta('batch number', batchNum);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
done();
|
|
190
|
+
log.debug(`Flushing batch ${batchNum} with ${currentBatch.length} rows`);
|
|
191
|
+
|
|
192
|
+
if (currentBatch.length) await flush(currentBatch);
|
|
193
|
+
|
|
194
|
+
if (!tmpFilenames.length) return 0;
|
|
195
|
+
|
|
196
|
+
await initDB();
|
|
197
|
+
|
|
198
|
+
const outputFilepath = path.join(outDir, outputFilename);
|
|
199
|
+
|
|
200
|
+
const parquetFiles = tmpFilenames.map((filename) => `'${filename.replaceAll('\\', '/')}'`);
|
|
201
|
+
|
|
202
|
+
const select = `SELECT * FROM read_parquet([${parquetFiles.join(',')}])`;
|
|
203
|
+
const copy = `COPY (${select}) TO '${outputFilepath}' (FORMAT 'PARQUET', CODEC 'ZSTD', USE_TMP_FILE false);`;
|
|
204
|
+
|
|
205
|
+
await fs.mkdir(path.dirname(outputFilepath), { recursive: true });
|
|
206
|
+
await query(copy);
|
|
207
|
+
|
|
208
|
+
await fs.chmod(outputFilepath, 0o644);
|
|
209
|
+
|
|
210
|
+
const { size } = await fs.stat(outputFilepath);
|
|
211
|
+
if (size > 100 * 1024 * 1024) {
|
|
212
|
+
console.warn(
|
|
213
|
+
chalk.yellow(` Estimated disk size is ${Intl.NumberFormat().format(size / (1024 * 1024))}mb.`)
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
const score =
|
|
218
|
+
rowCount *
|
|
219
|
+
columnsToScore(
|
|
220
|
+
columns.map(({ name, evidenceType }) => ({
|
|
221
|
+
name,
|
|
222
|
+
type:
|
|
223
|
+
evidenceType === 'number'
|
|
224
|
+
? 'DOUBLE'
|
|
225
|
+
: evidenceType === 'boolean'
|
|
226
|
+
? 'BOOLEAN'
|
|
227
|
+
: evidenceType === 'date'
|
|
228
|
+
? 'TIMESTAMP'
|
|
229
|
+
: 'VARCHAR'
|
|
230
|
+
}))
|
|
231
|
+
);
|
|
232
|
+
|
|
233
|
+
if (score > 100 * 1024 * 1024) {
|
|
234
|
+
console.warn(
|
|
235
|
+
chalk.yellow(
|
|
236
|
+
` WARNING: Estimated output size is ${Intl.NumberFormat().format(
|
|
237
|
+
score / (1024 * 1024)
|
|
238
|
+
)}mb uncompressed. This may cause client-side performance issues.`
|
|
239
|
+
)
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
for (const tmpFile of tmpFilenames) {
|
|
244
|
+
await fs.rm(tmpFile, { force: true });
|
|
245
|
+
}
|
|
246
|
+
await emptyDbFs('*');
|
|
247
|
+
|
|
248
|
+
fn_done();
|
|
249
|
+
|
|
250
|
+
return rowCount;
|
|
251
|
+
}
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
|
2
|
+
import { buildMultipartParquet } from './build-parquet';
|
|
3
|
+
|
|
4
|
+
import mockfs from 'mock-fs';
|
|
5
|
+
import fs from 'fs/promises';
|
|
6
|
+
import { initDB } from './client-duckdb/node';
|
|
7
|
+
import path from 'path';
|
|
8
|
+
|
|
9
|
+
function adaptFilePath(filepath) {
|
|
10
|
+
return path.join(...filepath.split('/'));
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// Spying on writeFile breaks things; but we can listen for the cleanup.
|
|
14
|
+
vi.spyOn(fs, 'rm');
|
|
15
|
+
|
|
16
|
+
describe('buildMultipartParquet', () => {
|
|
17
|
+
beforeEach(async () => {
|
|
18
|
+
vi.resetAllMocks();
|
|
19
|
+
await initDB();
|
|
20
|
+
mockfs({
|
|
21
|
+
sources: {},
|
|
22
|
+
static: {
|
|
23
|
+
data: {}
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('should be defined', () => expect(buildMultipartParquet).toBeDefined());
|
|
29
|
+
it('should handle a generator that returns no results correctly', async () => {
|
|
30
|
+
const mockCols = [
|
|
31
|
+
{
|
|
32
|
+
name: 'x',
|
|
33
|
+
evidenceType: 'string'
|
|
34
|
+
}
|
|
35
|
+
];
|
|
36
|
+
|
|
37
|
+
function* emptyGenerator() {
|
|
38
|
+
yield [];
|
|
39
|
+
return [];
|
|
40
|
+
}
|
|
41
|
+
const r = await buildMultipartParquet(
|
|
42
|
+
mockCols,
|
|
43
|
+
emptyGenerator(),
|
|
44
|
+
adaptFilePath('./.evidence/template'),
|
|
45
|
+
adaptFilePath(''),
|
|
46
|
+
'out.parquet'
|
|
47
|
+
);
|
|
48
|
+
expect(r).toBe(0);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
it('should write one temporary file when given fewer than one batch of rows', async () => {
|
|
52
|
+
const mockCols = [
|
|
53
|
+
{
|
|
54
|
+
name: 'x',
|
|
55
|
+
evidenceType: 'string'
|
|
56
|
+
}
|
|
57
|
+
];
|
|
58
|
+
function* gen() {
|
|
59
|
+
yield [{ x: 'hello' }];
|
|
60
|
+
yield [{ x: 'world' }];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const r = await buildMultipartParquet(
|
|
64
|
+
mockCols,
|
|
65
|
+
gen(),
|
|
66
|
+
adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet'),
|
|
67
|
+
adaptFilePath('.evidence/template/static/data'),
|
|
68
|
+
'out.parquet'
|
|
69
|
+
);
|
|
70
|
+
expect(r).toBe(2);
|
|
71
|
+
const stat = await fs.stat('.evidence/template/static/data/out.parquet');
|
|
72
|
+
expect(stat.isFile()).toBeTruthy();
|
|
73
|
+
// Make sure it contains data
|
|
74
|
+
expect(stat.size).toBeGreaterThan(0);
|
|
75
|
+
expect(fs.rm).toHaveBeenCalledOnce();
|
|
76
|
+
// verify the rm call targeted the intermediate directory and a tempfile
|
|
77
|
+
const calledPath = fs.rm.mock.calls[0][0];
|
|
78
|
+
expect(path.dirname(calledPath)).toBe(
|
|
79
|
+
adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet')
|
|
80
|
+
);
|
|
81
|
+
// basename should look like out.<timestamp>.<rand>.0.parquet
|
|
82
|
+
expect(path.basename(calledPath)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it('should write two temporary files when given enough rows for two batches', async () => {
|
|
86
|
+
const mockCols = [
|
|
87
|
+
{
|
|
88
|
+
name: 'x',
|
|
89
|
+
evidenceType: 'string'
|
|
90
|
+
}
|
|
91
|
+
];
|
|
92
|
+
function* gen() {
|
|
93
|
+
yield [{ x: 'hello' }];
|
|
94
|
+
yield [{ x: 'world' }];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const r = await buildMultipartParquet(
|
|
98
|
+
mockCols,
|
|
99
|
+
gen(),
|
|
100
|
+
adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet'),
|
|
101
|
+
adaptFilePath('.evidence/template/static/data'),
|
|
102
|
+
'out.parquet',
|
|
103
|
+
1
|
|
104
|
+
);
|
|
105
|
+
expect(r).toBe(2);
|
|
106
|
+
const stat = await fs.stat('.evidence/template/static/data/out.parquet');
|
|
107
|
+
expect(stat.isFile()).toBeTruthy();
|
|
108
|
+
// Make sure it contains data
|
|
109
|
+
expect(stat.size).toBeGreaterThan(0);
|
|
110
|
+
expect(fs.rm).toHaveBeenCalledTimes(2);
|
|
111
|
+
const calledPath1 = fs.rm.mock.calls[0][0];
|
|
112
|
+
const calledPath2 = fs.rm.mock.calls[1][0];
|
|
113
|
+
expect(path.dirname(calledPath1)).toBe(
|
|
114
|
+
adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet')
|
|
115
|
+
);
|
|
116
|
+
expect(path.dirname(calledPath2)).toBe(
|
|
117
|
+
adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet')
|
|
118
|
+
);
|
|
119
|
+
expect(path.basename(calledPath1)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/);
|
|
120
|
+
expect(path.basename(calledPath2)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.1\.parquet$/);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('should accept an array as the data argument', async () => {
|
|
124
|
+
const mockCols = [{ name: 'x', evidenceType: 'string' }];
|
|
125
|
+
|
|
126
|
+
const r = await buildMultipartParquet(
|
|
127
|
+
mockCols,
|
|
128
|
+
[{ x: 'hello' }, { x: 'hello' }],
|
|
129
|
+
adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet'),
|
|
130
|
+
adaptFilePath('.evidence/template/static/data'),
|
|
131
|
+
'out.parquet'
|
|
132
|
+
);
|
|
133
|
+
expect(r).toBe(2);
|
|
134
|
+
const stat = await fs.stat('.evidence/template/static/data/out.parquet');
|
|
135
|
+
expect(stat.isFile()).toBeTruthy();
|
|
136
|
+
// Make sure it contains data
|
|
137
|
+
expect(stat.size).toBeGreaterThan(0);
|
|
138
|
+
expect(fs.rm).toHaveBeenCalledOnce();
|
|
139
|
+
const calledPathA = fs.rm.mock.calls[0][0];
|
|
140
|
+
expect(path.dirname(calledPathA)).toBe(
|
|
141
|
+
adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet')
|
|
142
|
+
);
|
|
143
|
+
expect(path.basename(calledPathA)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
it('should handle a very large number of batches', async () => {
|
|
147
|
+
const mockCols = [{ name: 'x', evidenceType: 'string' }];
|
|
148
|
+
|
|
149
|
+
const VERY_LARGE_NUMBER = 800;
|
|
150
|
+
|
|
151
|
+
function* gen() {
|
|
152
|
+
for (let i = 0; i < VERY_LARGE_NUMBER; i++) yield [{ x: i }];
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const r = await buildMultipartParquet(
|
|
156
|
+
mockCols,
|
|
157
|
+
gen(),
|
|
158
|
+
adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet'),
|
|
159
|
+
adaptFilePath('.evidence/template/static/data'),
|
|
160
|
+
'out.parquet',
|
|
161
|
+
1
|
|
162
|
+
);
|
|
163
|
+
expect(r).toBe(VERY_LARGE_NUMBER);
|
|
164
|
+
const stat = await fs.stat('.evidence/template/static/data/out.parquet');
|
|
165
|
+
expect(stat.isFile()).toBeTruthy();
|
|
166
|
+
// Make sure it contains data
|
|
167
|
+
expect(stat.size).toBeGreaterThan(0);
|
|
168
|
+
expect(fs.rm).toHaveBeenCalledTimes(VERY_LARGE_NUMBER);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
// TODO: Test how it handles invalid filepath
|
|
172
|
+
// TODO: Test how it handles missing column data
|
|
173
|
+
// TODO: Test how it handles data that includes more keys than are in columns
|
|
174
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Caches DuckDB queries that have been
|
|
3
|
+
* @param {string} sql_string SQL string executed
|
|
4
|
+
* @param {import("apache-arrow").Table} data result of the query
|
|
5
|
+
* @param {{ route_hash: string, additional_hash: string, query_name: string, prerendering: boolean }} cache_options
|
|
6
|
+
* @returns {void}
|
|
7
|
+
*/
|
|
8
|
+
export function cache_for_hash(
|
|
9
|
+
sql_string: string,
|
|
10
|
+
data: import('apache-arrow').Table,
|
|
11
|
+
cache_options: { route_hash: string; query_name: string; prerendering: boolean }
|
|
12
|
+
): void;
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Gets the cached arrow response for a given route hash, additional hash, and query name
|
|
16
|
+
* @param {string} route_hash
|
|
17
|
+
* @param {string} additional_hash
|
|
18
|
+
* @param {string} query_name
|
|
19
|
+
* @returns {Buffer}
|
|
20
|
+
*/
|
|
21
|
+
export function get_cache_for_hash(
|
|
22
|
+
route_hash: string,
|
|
23
|
+
additional_hash: string,
|
|
24
|
+
query_name: string
|
|
25
|
+
): Buffer;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Gets all the queries that run on a given page
|
|
29
|
+
* @param {string} route_hash
|
|
30
|
+
* @returns {string}
|
|
31
|
+
*/
|
|
32
|
+
export function get_all_page_queries(route_hash: string): string;
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
|
|
3
|
+
import { createRequire } from 'module';
|
|
4
|
+
import { dirname } from 'path';
|
|
5
|
+
|
|
6
|
+
/*
|
|
7
|
+
apache-arrow is a cjs module, so we need to require it
|
|
8
|
+
apache-arrow also uses a lot of `instanceof`, which is very fragile in javascript
|
|
9
|
+
By requiring apache-arrow relative to duckdb-wasm, we can be sure that we are sharing
|
|
10
|
+
the same references with duckdb, thus making instanceof work again.
|
|
11
|
+
*/
|
|
12
|
+
const require1 = createRequire(import.meta.url);
|
|
13
|
+
const require = createRequire(require1.resolve('@duckdb/duckdb-wasm'));
|
|
14
|
+
|
|
15
|
+
const { tableToIPC, tableFromIPC } = require('apache-arrow');
|
|
16
|
+
// blocking duckdb-wasm uses cjs and need to have same Table declaration for instanceof
|
|
17
|
+
|
|
18
|
+
/*
|
|
19
|
+
Caching strategy:
|
|
20
|
+
- SQL string and the corresponding output from that is fed to `cache_for_hash`
|
|
21
|
+
- Additional route and query information is also given
|
|
22
|
+
- If the SQL has not been cached yet, it is written to
|
|
23
|
+
1. the cache folder, and, if we're building,
|
|
24
|
+
2. the location sveltekit writes prerendered API routes (`prerender_path`)
|
|
25
|
+
|
|
26
|
+
- The SQL string and hash of the resulting arrow table are then written to a route-specific _queries.json
|
|
27
|
+
which will be used when we're refreshing routes in the future (will likely change by then)
|
|
28
|
+
|
|
29
|
+
- A subset of _queries.json (just query name -> result hash object) is written to a route-specific
|
|
30
|
+
`all-queries.json` which is used by `+layout.js` to load the prerendered data
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
/** @type {typeof writeFileSync} */
|
|
34
|
+
const writeToPossiblyNonexistentFile = (path, data) => {
|
|
35
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
36
|
+
writeFileSync(path, data);
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
function getCacheFolder(route_hash, additional_hash) {
|
|
40
|
+
if (additional_hash) return `./.evidence-queries/cache/${route_hash}/${additional_hash}`;
|
|
41
|
+
else if (route_hash) return `./.evidence-queries/cache/${route_hash}`;
|
|
42
|
+
else return `./.evidence-queries/cache`;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
*
|
|
47
|
+
* @param {Uint8Array | string} buffer
|
|
48
|
+
* @returns {string}
|
|
49
|
+
*/
|
|
50
|
+
function hash(buffer) {
|
|
51
|
+
return createHash('md5').update(buffer).digest('hex');
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Points SQL strings to a hash of their output
|
|
56
|
+
* @type {Map<string, string>}
|
|
57
|
+
*/
|
|
58
|
+
const sql_string_cache = new Map();
|
|
59
|
+
|
|
60
|
+
// while prerendering static sites, sveltekit finds endpoints based on what's used in load functions with
|
|
61
|
+
// the provided fetch - since the data isn't ready yet, instead we write the data to the place where sveltekit
|
|
62
|
+
// would have written it
|
|
63
|
+
const prerender_path = `.svelte-kit/output/prerendered/dependencies`;
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Caches DuckDB queries that have been executed during SSR
|
|
67
|
+
* @param {string} sql_string SQL string executed
|
|
68
|
+
* @param {import("apache-arrow").Table} data result of the query
|
|
69
|
+
* @param {{ route_hash: string, additional_hash: string, query_name: string, prerendering: boolean }} cache_options
|
|
70
|
+
* @returns {void}
|
|
71
|
+
*/
|
|
72
|
+
export function cache_for_hash(
|
|
73
|
+
sql_string,
|
|
74
|
+
data,
|
|
75
|
+
{ route_hash, additional_hash, query_name, prerendering }
|
|
76
|
+
) {
|
|
77
|
+
// if the it's in the cache then we shouldn't bother
|
|
78
|
+
let ipc_table_hash = sql_string_cache.get(sql_string);
|
|
79
|
+
|
|
80
|
+
// this can only be false during `prerendering`
|
|
81
|
+
if (!sql_string_cache.has(sql_string)) {
|
|
82
|
+
const ipc_table = tableToIPC(data);
|
|
83
|
+
ipc_table_hash = hash(ipc_table);
|
|
84
|
+
|
|
85
|
+
// write the data to cache
|
|
86
|
+
// query caches are grouped by the hash of the ipc_table
|
|
87
|
+
// this prevents duplicate queries (ie parameterized page queries) from exploding built site size
|
|
88
|
+
const base_cache_path = getCacheFolder();
|
|
89
|
+
writeToPossiblyNonexistentFile(`${base_cache_path}/${ipc_table_hash}.arrow`, ipc_table);
|
|
90
|
+
if (prerendering) {
|
|
91
|
+
// save the result of the query
|
|
92
|
+
writeToPossiblyNonexistentFile(
|
|
93
|
+
`${prerender_path}/api/prerendered_queries/${ipc_table_hash}.arrow`,
|
|
94
|
+
ipc_table
|
|
95
|
+
);
|
|
96
|
+
sql_string_cache.set(sql_string, ipc_table_hash);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
// keeps a cache of the sql queries for each route
|
|
101
|
+
// for later refreshing without fully rebuilding the site
|
|
102
|
+
const double_cache_path = getCacheFolder(route_hash, additional_hash);
|
|
103
|
+
const sql_path = `${double_cache_path}/_queries.json`;
|
|
104
|
+
if (!existsSync(sql_path)) {
|
|
105
|
+
writeToPossiblyNonexistentFile(sql_path, '{}');
|
|
106
|
+
}
|
|
107
|
+
const sql_cache = JSON.parse(readFileSync(sql_path, 'utf-8'));
|
|
108
|
+
sql_cache[query_name] = { sql_string, query_hash: ipc_table_hash };
|
|
109
|
+
writeFileSync(sql_path, JSON.stringify(sql_cache));
|
|
110
|
+
|
|
111
|
+
// keep track of query hashes so `+layout.js` knows what to fetch
|
|
112
|
+
const sql_cache_with_hashed_query_strings = Object.fromEntries(
|
|
113
|
+
Object.entries(sql_cache).map((entry) => [entry[0], entry[1].query_hash])
|
|
114
|
+
);
|
|
115
|
+
/*
|
|
116
|
+
uses double cache path because parameterized pages could theoretically have
|
|
117
|
+
distinct queries run in them, such as:
|
|
118
|
+
{#if params.country === 'USA'}
|
|
119
|
+
<Dropdown />
|
|
120
|
+
{:else}
|
|
121
|
+
<Slider />
|
|
122
|
+
{/if}
|
|
123
|
+
*/
|
|
124
|
+
writeToPossiblyNonexistentFile(
|
|
125
|
+
`${double_cache_path}/all-queries.json`,
|
|
126
|
+
JSON.stringify(sql_cache_with_hashed_query_strings)
|
|
127
|
+
);
|
|
128
|
+
if (prerendering) {
|
|
129
|
+
// keep track of the query names for the page
|
|
130
|
+
writeToPossiblyNonexistentFile(
|
|
131
|
+
`${prerender_path}/api/${route_hash}/${additional_hash}/all-queries.json`,
|
|
132
|
+
JSON.stringify(sql_cache_with_hashed_query_strings)
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Gets the cached arrow response for a given route hash and query name
|
|
139
|
+
* @param {string} query_hash
|
|
140
|
+
* @returns {Buffer}
|
|
141
|
+
*/
|
|
142
|
+
export function get_cache_for_hash(query_hash) {
|
|
143
|
+
const cache_path = getCacheFolder();
|
|
144
|
+
return readFileSync(`${cache_path}/${query_hash}.arrow`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Gets all the queries that run on a given page
|
|
149
|
+
* @param {string} route_hash
|
|
150
|
+
* @param {string} additional_hash
|
|
151
|
+
* @returns {string}
|
|
152
|
+
*/
|
|
153
|
+
export function get_all_page_queries(route_hash, additional_hash) {
|
|
154
|
+
try {
|
|
155
|
+
const cache_path = getCacheFolder(route_hash, additional_hash);
|
|
156
|
+
|
|
157
|
+
// Check if the file exists before attempting to read it
|
|
158
|
+
const file_path = `${cache_path}/all-queries.json`;
|
|
159
|
+
if (!existsSync(file_path)) {
|
|
160
|
+
return '{}'; // Return an empty object string as a fallback
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// Read and return the file contents
|
|
164
|
+
return readFileSync(file_path, 'utf8');
|
|
165
|
+
} catch (error) {
|
|
166
|
+
// Log the error and return a fallback
|
|
167
|
+
console.error(`Error reading all-queries.json:`, error);
|
|
168
|
+
return '{}'; // Return an empty object string as a fallback
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/**
|
|
173
|
+
* If a string has already been queried during the lifetime of the program
|
|
174
|
+
* fetch it from cache instead of re-executing it
|
|
175
|
+
*
|
|
176
|
+
* Only runs during build
|
|
177
|
+
* @param {string} sql
|
|
178
|
+
* @returns {import("apache-arrow").Table | null}
|
|
179
|
+
*/
|
|
180
|
+
export function get_arrow_if_sql_already_run(sql) {
|
|
181
|
+
if (!sql_string_cache.has(sql)) return null;
|
|
182
|
+
return tableFromIPC(get_cache_for_hash(sql_string_cache.get(sql)));
|
|
183
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @param {{ name: string; type: string }[]} columns
|
|
3
|
+
* @returns {number}
|
|
4
|
+
*/
|
|
5
|
+
export function columnsToScore(
|
|
6
|
+
columns: {
|
|
7
|
+
name: string;
|
|
8
|
+
type: string;
|
|
9
|
+
}[]
|
|
10
|
+
): number;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* @param {{ name: string; evidenceType: string }[]} columns
|
|
14
|
+
* @returns {number}
|
|
15
|
+
*/
|
|
16
|
+
export function evidenceColumnsToScore(columns: { name: string; evidenceType: string }[]): number;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
*
|
|
20
|
+
* @param {string} column_type
|
|
21
|
+
* @returns {'number' | 'boolean' | 'string' | 'date'}
|
|
22
|
+
*/
|
|
23
|
+
export function duckdbTypeToEvidenceType(
|
|
24
|
+
column_type: string
|
|
25
|
+
): 'number' | 'boolean' | 'string' | 'date';
|