@yolean/evidence-universal-sql 0.0.0-stage → 3.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,251 @@
1
+ import {
2
+ tableFromArrays,
3
+ tableToIPC,
4
+ vectorFromArray,
5
+ Float64,
6
+ Utf8,
7
+ Bool,
8
+ TimestampMillisecond
9
+ } from 'apache-arrow';
10
+ import {
11
+ Compression,
12
+ writeParquet,
13
+ WriterPropertiesBuilder,
14
+ Table
15
+ } from 'parquet-wasm/node/arrow1.js';
16
+ import fs from 'fs/promises';
17
+ import path from 'path';
18
+ // using node-async.js makes CLI command hang - why??
19
+ import { emptyDbFs, initDB, query } from './client-duckdb/node.js';
20
+ import chunk from 'lodash.chunk';
21
+ import { columnsToScore } from './calculateScore.js';
22
+ import chalk from 'chalk';
23
+ import { log } from '@evidence-dev/sdk/logger';
24
+
25
+ /**
26
+ * @param {{name: string, evidenceType: string}} column
27
+ * @param {any[]} rawValues
28
+ * @returns {import("apache-arrow").Vector}
29
+ */
30
+ function convertArrayToVector(column, rawValues) {
31
+ switch (column.evidenceType) {
32
+ case 'number':
33
+ return vectorFromArray(rawValues, new Float64());
34
+ case 'string':
35
+ return vectorFromArray(rawValues, new Utf8());
36
+ case 'date':
37
+ if (!rawValues.some((v) => v !== null)) {
38
+ // All null date columns error out, so we have to do this
39
+ // https://github.com/evidence-dev/evidence/issues/2897
40
+ // separate reason than the bool version
41
+ console.warn(
42
+ chalk.yellow(
43
+ `\nWarning: Column "${column.name}" (type Date) contains only null values so it has been cast to Float64`
44
+ )
45
+ );
46
+ return vectorFromArray(rawValues, new Float64());
47
+ }
48
+ // TODO: What gives with timezones
49
+ return vectorFromArray(rawValues, new TimestampMillisecond());
50
+ case 'boolean':
51
+ if (!rawValues.some((v) => v !== null)) {
52
+ // All null bool columns error out, so we have to do this
53
+ // https://github.com/evidence-dev/evidence/issues/1504
54
+ console.warn(
55
+ chalk.yellow(
56
+ `\nWarning: Column "${column.name}" (type Bool) contains only null values so it has been cast to Float64`
57
+ )
58
+ );
59
+ return vectorFromArray(rawValues, new Float64());
60
+ }
61
+ return vectorFromArray(rawValues, new Bool());
62
+ default:
63
+ throw new Error(
64
+ 'Unrecognized EvidenceType: ' +
65
+ column.evidenceType +
66
+ '\n This is likely an error in a datasource connector.'
67
+ );
68
+ }
69
+ }
70
+
71
+ /**
72
+ * @template T
73
+ * @param {{name: string, evidenceType: string}[]} columns
74
+ * @param {Generator<T[] | Promise<T[]> | T[] | Promise<T[]>} data
75
+ * @param {string} tmpDir
76
+ * @param {string} outDir
77
+ * @param {string} outputFilename
78
+ * @param {number} [batchSize]
79
+ * @returns {Promise<number>} Number of rows
80
+ */
81
+ export async function buildMultipartParquet(
82
+ columns,
83
+ data,
84
+ tmpDir,
85
+ outDir,
86
+ outputFilename,
87
+ batchSize = 1000000
88
+ ) {
89
+ log.debug(`Building parquet file ${outputFilename}`);
90
+ let { meta: fn_meta, done: fn_done } = log.measure('buildMultipartParquet');
91
+ fn_meta('output filename', outputFilename);
92
+
93
+ let batchNum = 0;
94
+ const outputSubpath = outputFilename.split('.parquet')[0];
95
+ let tmpFilenames = [];
96
+ let rowCount = 0;
97
+
98
+ // Unique id for this build run to avoid reusing the same temp filenames across
99
+ // consecutive builds in the same session. DuckDB (wasm) may keep handles or
100
+ // mappings to filenames; creating unique filenames per build avoids races or
101
+ // protocol errors when files are replaced.
102
+ const buildId = `${Date.now()}.${Math.random().toString(36).slice(2, 8)}`;
103
+
104
+ const flush = async (results) => {
105
+ log.debug(`Flushing batch ${batchNum} with ${results.length} rows`);
106
+ let { meta, done } = log.measure('flush');
107
+ meta('batch number', batchNum);
108
+
109
+ // Convert JS Objects -> Arrow
110
+ const vectorized = Object.fromEntries(
111
+ columns.map((c) => [
112
+ c.name,
113
+ convertArrayToVector(
114
+ c,
115
+ results.map((i) => i[c.name] ?? null)
116
+ )
117
+ ])
118
+ );
119
+ const table = tableFromArrays(vectorized);
120
+ for (const field of table.schema.fields) {
121
+ field.nullable = true;
122
+ }
123
+
124
+ // Converts the arrow table to a buffer
125
+ const IPC = tableToIPC(table, 'stream');
126
+
127
+ const writerProperties = new WriterPropertiesBuilder().setCompression(Compression.ZSTD).build();
128
+ // Converts the arrow buffer to a parquet buffer
129
+ // This can be slow; it includes the compression step
130
+
131
+ const parquetBuffer = writeParquet(Table.fromIPCStream(IPC), writerProperties);
132
+
133
+ const finalTempFilename = path.join(tmpDir, `${outputSubpath}.${buildId}.${batchNum}.parquet`);
134
+ // Write to a unique temporary file first, then atomically rename into place.
135
+ // This avoids a race where DuckDB may attempt to read a file while it's being written
136
+ // (e.g., during rapid dev refreshes). Using an atomic rename ensures readers only
137
+ // see the completed file.
138
+ const uniqueSuffix = `.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`;
139
+ const writeTempFilename = finalTempFilename + uniqueSuffix;
140
+
141
+ await fs.mkdir(path.dirname(finalTempFilename), { recursive: true });
142
+ await fs.writeFile(writeTempFilename, parquetBuffer);
143
+ // Atomically move into final location
144
+ await fs.rename(writeTempFilename, finalTempFilename);
145
+
146
+ tmpFilenames.push(finalTempFilename);
147
+ rowCount += results.length;
148
+
149
+ done();
150
+ log.debug(`Flushed batch ${batchNum} with ${results.length} rows`);
151
+
152
+ batchNum++;
153
+ };
154
+
155
+ if (typeof data === 'function') data = data();
156
+ if (data instanceof Promise) data = await data;
157
+ // Data is an array, but not a nested one
158
+ // We expect a set of result sets;
159
+ if (Array.isArray(data) && !Array.isArray(data[0])) data = [data];
160
+ if (Array.isArray(data)) {
161
+ const arrays = data;
162
+ data = (function* () {
163
+ for (const results of arrays) {
164
+ for (const batch of chunk(results, batchSize)) {
165
+ yield batch;
166
+ }
167
+ }
168
+ })();
169
+ }
170
+
171
+ log.debug('Reading rows from a generator object');
172
+ let { meta, done } = log.measure('buildMultipartParquet');
173
+ meta('batch number', batchNum);
174
+
175
+ const currentBatch = [];
176
+ for await (const results of data) {
177
+ for (const result of results) currentBatch.push(result);
178
+
179
+ if (currentBatch.length >= batchSize) {
180
+ done();
181
+ log.debug(`Flushing batch ${batchNum} with ${currentBatch.length} rows`);
182
+ await flush(currentBatch);
183
+ currentBatch.length = 0;
184
+ ({ meta, done } = log.measure('buildMultipartParquet'));
185
+ meta('batch number', batchNum);
186
+ }
187
+ }
188
+
189
+ done();
190
+ log.debug(`Flushing batch ${batchNum} with ${currentBatch.length} rows`);
191
+
192
+ if (currentBatch.length) await flush(currentBatch);
193
+
194
+ if (!tmpFilenames.length) return 0;
195
+
196
+ await initDB();
197
+
198
+ const outputFilepath = path.join(outDir, outputFilename);
199
+
200
+ const parquetFiles = tmpFilenames.map((filename) => `'${filename.replaceAll('\\', '/')}'`);
201
+
202
+ const select = `SELECT * FROM read_parquet([${parquetFiles.join(',')}])`;
203
+ const copy = `COPY (${select}) TO '${outputFilepath}' (FORMAT 'PARQUET', CODEC 'ZSTD', USE_TMP_FILE false);`;
204
+
205
+ await fs.mkdir(path.dirname(outputFilepath), { recursive: true });
206
+ await query(copy);
207
+
208
+ await fs.chmod(outputFilepath, 0o644);
209
+
210
+ const { size } = await fs.stat(outputFilepath);
211
+ if (size > 100 * 1024 * 1024) {
212
+ console.warn(
213
+ chalk.yellow(` Estimated disk size is ${Intl.NumberFormat().format(size / (1024 * 1024))}mb.`)
214
+ );
215
+ }
216
+
217
+ const score =
218
+ rowCount *
219
+ columnsToScore(
220
+ columns.map(({ name, evidenceType }) => ({
221
+ name,
222
+ type:
223
+ evidenceType === 'number'
224
+ ? 'DOUBLE'
225
+ : evidenceType === 'boolean'
226
+ ? 'BOOLEAN'
227
+ : evidenceType === 'date'
228
+ ? 'TIMESTAMP'
229
+ : 'VARCHAR'
230
+ }))
231
+ );
232
+
233
+ if (score > 100 * 1024 * 1024) {
234
+ console.warn(
235
+ chalk.yellow(
236
+ ` WARNING: Estimated output size is ${Intl.NumberFormat().format(
237
+ score / (1024 * 1024)
238
+ )}mb uncompressed. This may cause client-side performance issues.`
239
+ )
240
+ );
241
+ }
242
+
243
+ for (const tmpFile of tmpFilenames) {
244
+ await fs.rm(tmpFile, { force: true });
245
+ }
246
+ await emptyDbFs('*');
247
+
248
+ fn_done();
249
+
250
+ return rowCount;
251
+ }
@@ -0,0 +1,174 @@
1
+ import { describe, it, expect, beforeEach, vi } from 'vitest';
2
+ import { buildMultipartParquet } from './build-parquet';
3
+
4
+ import mockfs from 'mock-fs';
5
+ import fs from 'fs/promises';
6
+ import { initDB } from './client-duckdb/node';
7
+ import path from 'path';
8
+
9
+ function adaptFilePath(filepath) {
10
+ return path.join(...filepath.split('/'));
11
+ }
12
+
13
+ // Spying on writeFile breaks things; but we can listen for the cleanup.
14
+ vi.spyOn(fs, 'rm');
15
+
16
+ describe('buildMultipartParquet', () => {
17
+ beforeEach(async () => {
18
+ vi.resetAllMocks();
19
+ await initDB();
20
+ mockfs({
21
+ sources: {},
22
+ static: {
23
+ data: {}
24
+ }
25
+ });
26
+ });
27
+
28
+ it('should be defined', () => expect(buildMultipartParquet).toBeDefined());
29
+ it('should handle a generator that returns no results correctly', async () => {
30
+ const mockCols = [
31
+ {
32
+ name: 'x',
33
+ evidenceType: 'string'
34
+ }
35
+ ];
36
+
37
+ function* emptyGenerator() {
38
+ yield [];
39
+ return [];
40
+ }
41
+ const r = await buildMultipartParquet(
42
+ mockCols,
43
+ emptyGenerator(),
44
+ adaptFilePath('./.evidence/template'),
45
+ adaptFilePath(''),
46
+ 'out.parquet'
47
+ );
48
+ expect(r).toBe(0);
49
+ });
50
+
51
+ it('should write one temporary file when given fewer than one batch of rows', async () => {
52
+ const mockCols = [
53
+ {
54
+ name: 'x',
55
+ evidenceType: 'string'
56
+ }
57
+ ];
58
+ function* gen() {
59
+ yield [{ x: 'hello' }];
60
+ yield [{ x: 'world' }];
61
+ }
62
+
63
+ const r = await buildMultipartParquet(
64
+ mockCols,
65
+ gen(),
66
+ adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet'),
67
+ adaptFilePath('.evidence/template/static/data'),
68
+ 'out.parquet'
69
+ );
70
+ expect(r).toBe(2);
71
+ const stat = await fs.stat('.evidence/template/static/data/out.parquet');
72
+ expect(stat.isFile()).toBeTruthy();
73
+ // Make sure it contains data
74
+ expect(stat.size).toBeGreaterThan(0);
75
+ expect(fs.rm).toHaveBeenCalledOnce();
76
+ // verify the rm call targeted the intermediate directory and a tempfile
77
+ const calledPath = fs.rm.mock.calls[0][0];
78
+ expect(path.dirname(calledPath)).toBe(
79
+ adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet')
80
+ );
81
+ // basename should look like out.<timestamp>.<rand>.0.parquet
82
+ expect(path.basename(calledPath)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/);
83
+ });
84
+
85
+ it('should write two temporary files when given enough rows for two batches', async () => {
86
+ const mockCols = [
87
+ {
88
+ name: 'x',
89
+ evidenceType: 'string'
90
+ }
91
+ ];
92
+ function* gen() {
93
+ yield [{ x: 'hello' }];
94
+ yield [{ x: 'world' }];
95
+ }
96
+
97
+ const r = await buildMultipartParquet(
98
+ mockCols,
99
+ gen(),
100
+ adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet'),
101
+ adaptFilePath('.evidence/template/static/data'),
102
+ 'out.parquet',
103
+ 1
104
+ );
105
+ expect(r).toBe(2);
106
+ const stat = await fs.stat('.evidence/template/static/data/out.parquet');
107
+ expect(stat.isFile()).toBeTruthy();
108
+ // Make sure it contains data
109
+ expect(stat.size).toBeGreaterThan(0);
110
+ expect(fs.rm).toHaveBeenCalledTimes(2);
111
+ const calledPath1 = fs.rm.mock.calls[0][0];
112
+ const calledPath2 = fs.rm.mock.calls[1][0];
113
+ expect(path.dirname(calledPath1)).toBe(
114
+ adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet')
115
+ );
116
+ expect(path.dirname(calledPath2)).toBe(
117
+ adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet')
118
+ );
119
+ expect(path.basename(calledPath1)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/);
120
+ expect(path.basename(calledPath2)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.1\.parquet$/);
121
+ });
122
+
123
+ it('should accept an array as the data argument', async () => {
124
+ const mockCols = [{ name: 'x', evidenceType: 'string' }];
125
+
126
+ const r = await buildMultipartParquet(
127
+ mockCols,
128
+ [{ x: 'hello' }, { x: 'hello' }],
129
+ adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet'),
130
+ adaptFilePath('.evidence/template/static/data'),
131
+ 'out.parquet'
132
+ );
133
+ expect(r).toBe(2);
134
+ const stat = await fs.stat('.evidence/template/static/data/out.parquet');
135
+ expect(stat.isFile()).toBeTruthy();
136
+ // Make sure it contains data
137
+ expect(stat.size).toBeGreaterThan(0);
138
+ expect(fs.rm).toHaveBeenCalledOnce();
139
+ const calledPathA = fs.rm.mock.calls[0][0];
140
+ expect(path.dirname(calledPathA)).toBe(
141
+ adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet')
142
+ );
143
+ expect(path.basename(calledPathA)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/);
144
+ });
145
+
146
+ it('should handle a very large number of batches', async () => {
147
+ const mockCols = [{ name: 'x', evidenceType: 'string' }];
148
+
149
+ const VERY_LARGE_NUMBER = 800;
150
+
151
+ function* gen() {
152
+ for (let i = 0; i < VERY_LARGE_NUMBER; i++) yield [{ x: i }];
153
+ }
154
+
155
+ const r = await buildMultipartParquet(
156
+ mockCols,
157
+ gen(),
158
+ adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet'),
159
+ adaptFilePath('.evidence/template/static/data'),
160
+ 'out.parquet',
161
+ 1
162
+ );
163
+ expect(r).toBe(VERY_LARGE_NUMBER);
164
+ const stat = await fs.stat('.evidence/template/static/data/out.parquet');
165
+ expect(stat.isFile()).toBeTruthy();
166
+ // Make sure it contains data
167
+ expect(stat.size).toBeGreaterThan(0);
168
+ expect(fs.rm).toHaveBeenCalledTimes(VERY_LARGE_NUMBER);
169
+ });
170
+
171
+ // TODO: Test how it handles invalid filepath
172
+ // TODO: Test how it handles missing column data
173
+ // TODO: Test how it handles data that includes more keys than are in columns
174
+ });
@@ -0,0 +1,32 @@
1
+ /**
2
+ * Caches DuckDB queries that have been
3
+ * @param {string} sql_string SQL string executed
4
+ * @param {import("apache-arrow").Table} data result of the query
5
+ * @param {{ route_hash: string, additional_hash: string, query_name: string, prerendering: boolean }} cache_options
6
+ * @returns {void}
7
+ */
8
+ export function cache_for_hash(
9
+ sql_string: string,
10
+ data: import('apache-arrow').Table,
11
+ cache_options: { route_hash: string; query_name: string; prerendering: boolean }
12
+ ): void;
13
+
14
+ /**
15
+ * Gets the cached arrow response for a given route hash, additional hash, and query name
16
+ * @param {string} route_hash
17
+ * @param {string} additional_hash
18
+ * @param {string} query_name
19
+ * @returns {Buffer}
20
+ */
21
+ export function get_cache_for_hash(
22
+ route_hash: string,
23
+ additional_hash: string,
24
+ query_name: string
25
+ ): Buffer;
26
+
27
+ /**
28
+ * Gets all the queries that run on a given page
29
+ * @param {string} route_hash
30
+ * @returns {string}
31
+ */
32
+ export function get_all_page_queries(route_hash: string): string;
@@ -0,0 +1,183 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { writeFileSync, readFileSync, existsSync, mkdirSync } from 'fs';
3
+ import { createRequire } from 'module';
4
+ import { dirname } from 'path';
5
+
6
+ /*
7
+ apache-arrow is a cjs module, so we need to require it
8
+ apache-arrow also uses a lot of `instanceof`, which is very fragile in javascript
9
+ By requiring apache-arrow relative to duckdb-wasm, we can be sure that we are sharing
10
+ the same references with duckdb, thus making instanceof work again.
11
+ */
12
+ const require1 = createRequire(import.meta.url);
13
+ const require = createRequire(require1.resolve('@duckdb/duckdb-wasm'));
14
+
15
+ const { tableToIPC, tableFromIPC } = require('apache-arrow');
16
+ // blocking duckdb-wasm uses cjs and need to have same Table declaration for instanceof
17
+
18
+ /*
19
+ Caching strategy:
20
+ - SQL string and the corresponding output from that is fed to `cache_for_hash`
21
+ - Additional route and query information is also given
22
+ - If the SQL has not been cached yet, it is written to
23
+ 1. the cache folder, and, if we're building,
24
+ 2. the location sveltekit writes prerendered API routes (`prerender_path`)
25
+
26
+ - The SQL string and hash of the resulting arrow table are then written to a route-specific _queries.json
27
+ which will be used when we're refreshing routes in the future (will likely change by then)
28
+
29
+ - A subset of _queries.json (just query name -> result hash object) is written to a route-specific
30
+ `all-queries.json` which is used by `+layout.js` to load the prerendered data
31
+ */
32
+
33
+ /** @type {typeof writeFileSync} */
34
+ const writeToPossiblyNonexistentFile = (path, data) => {
35
+ mkdirSync(dirname(path), { recursive: true });
36
+ writeFileSync(path, data);
37
+ };
38
+
39
+ function getCacheFolder(route_hash, additional_hash) {
40
+ if (additional_hash) return `./.evidence-queries/cache/${route_hash}/${additional_hash}`;
41
+ else if (route_hash) return `./.evidence-queries/cache/${route_hash}`;
42
+ else return `./.evidence-queries/cache`;
43
+ }
44
+
45
+ /**
46
+ *
47
+ * @param {Uint8Array | string} buffer
48
+ * @returns {string}
49
+ */
50
+ function hash(buffer) {
51
+ return createHash('md5').update(buffer).digest('hex');
52
+ }
53
+
54
+ /**
55
+ * Points SQL strings to a hash of their output
56
+ * @type {Map<string, string>}
57
+ */
58
+ const sql_string_cache = new Map();
59
+
60
+ // while prerendering static sites, sveltekit finds endpoints based on what's used in load functions with
61
+ // the provided fetch - since the data isn't ready yet, instead we write the data to the place where sveltekit
62
+ // would have written it
63
+ const prerender_path = `.svelte-kit/output/prerendered/dependencies`;
64
+
65
+ /**
66
+ * Caches DuckDB queries that have been executed during SSR
67
+ * @param {string} sql_string SQL string executed
68
+ * @param {import("apache-arrow").Table} data result of the query
69
+ * @param {{ route_hash: string, additional_hash: string, query_name: string, prerendering: boolean }} cache_options
70
+ * @returns {void}
71
+ */
72
+ export function cache_for_hash(
73
+ sql_string,
74
+ data,
75
+ { route_hash, additional_hash, query_name, prerendering }
76
+ ) {
77
+ // if the it's in the cache then we shouldn't bother
78
+ let ipc_table_hash = sql_string_cache.get(sql_string);
79
+
80
+ // this can only be false during `prerendering`
81
+ if (!sql_string_cache.has(sql_string)) {
82
+ const ipc_table = tableToIPC(data);
83
+ ipc_table_hash = hash(ipc_table);
84
+
85
+ // write the data to cache
86
+ // query caches are grouped by the hash of the ipc_table
87
+ // this prevents duplicate queries (ie parameterized page queries) from exploding built site size
88
+ const base_cache_path = getCacheFolder();
89
+ writeToPossiblyNonexistentFile(`${base_cache_path}/${ipc_table_hash}.arrow`, ipc_table);
90
+ if (prerendering) {
91
+ // save the result of the query
92
+ writeToPossiblyNonexistentFile(
93
+ `${prerender_path}/api/prerendered_queries/${ipc_table_hash}.arrow`,
94
+ ipc_table
95
+ );
96
+ sql_string_cache.set(sql_string, ipc_table_hash);
97
+ }
98
+ }
99
+
100
+ // keeps a cache of the sql queries for each route
101
+ // for later refreshing without fully rebuilding the site
102
+ const double_cache_path = getCacheFolder(route_hash, additional_hash);
103
+ const sql_path = `${double_cache_path}/_queries.json`;
104
+ if (!existsSync(sql_path)) {
105
+ writeToPossiblyNonexistentFile(sql_path, '{}');
106
+ }
107
+ const sql_cache = JSON.parse(readFileSync(sql_path, 'utf-8'));
108
+ sql_cache[query_name] = { sql_string, query_hash: ipc_table_hash };
109
+ writeFileSync(sql_path, JSON.stringify(sql_cache));
110
+
111
+ // keep track of query hashes so `+layout.js` knows what to fetch
112
+ const sql_cache_with_hashed_query_strings = Object.fromEntries(
113
+ Object.entries(sql_cache).map((entry) => [entry[0], entry[1].query_hash])
114
+ );
115
+ /*
116
+ uses double cache path because parameterized pages could theoretically have
117
+ distinct queries run in them, such as:
118
+ {#if params.country === 'USA'}
119
+ <Dropdown />
120
+ {:else}
121
+ <Slider />
122
+ {/if}
123
+ */
124
+ writeToPossiblyNonexistentFile(
125
+ `${double_cache_path}/all-queries.json`,
126
+ JSON.stringify(sql_cache_with_hashed_query_strings)
127
+ );
128
+ if (prerendering) {
129
+ // keep track of the query names for the page
130
+ writeToPossiblyNonexistentFile(
131
+ `${prerender_path}/api/${route_hash}/${additional_hash}/all-queries.json`,
132
+ JSON.stringify(sql_cache_with_hashed_query_strings)
133
+ );
134
+ }
135
+ }
136
+
137
+ /**
138
+ * Gets the cached arrow response for a given route hash and query name
139
+ * @param {string} query_hash
140
+ * @returns {Buffer}
141
+ */
142
+ export function get_cache_for_hash(query_hash) {
143
+ const cache_path = getCacheFolder();
144
+ return readFileSync(`${cache_path}/${query_hash}.arrow`);
145
+ }
146
+
147
+ /**
148
+ * Gets all the queries that run on a given page
149
+ * @param {string} route_hash
150
+ * @param {string} additional_hash
151
+ * @returns {string}
152
+ */
153
+ export function get_all_page_queries(route_hash, additional_hash) {
154
+ try {
155
+ const cache_path = getCacheFolder(route_hash, additional_hash);
156
+
157
+ // Check if the file exists before attempting to read it
158
+ const file_path = `${cache_path}/all-queries.json`;
159
+ if (!existsSync(file_path)) {
160
+ return '{}'; // Return an empty object string as a fallback
161
+ }
162
+
163
+ // Read and return the file contents
164
+ return readFileSync(file_path, 'utf8');
165
+ } catch (error) {
166
+ // Log the error and return a fallback
167
+ console.error(`Error reading all-queries.json:`, error);
168
+ return '{}'; // Return an empty object string as a fallback
169
+ }
170
+ }
171
+
172
+ /**
173
+ * If a string has already been queried during the lifetime of the program
174
+ * fetch it from cache instead of re-executing it
175
+ *
176
+ * Only runs during build
177
+ * @param {string} sql
178
+ * @returns {import("apache-arrow").Table | null}
179
+ */
180
+ export function get_arrow_if_sql_already_run(sql) {
181
+ if (!sql_string_cache.has(sql)) return null;
182
+ return tableFromIPC(get_cache_for_hash(sql_string_cache.get(sql)));
183
+ }
@@ -0,0 +1,25 @@
1
+ /**
2
+ * @param {{ name: string; type: string }[]} columns
3
+ * @returns {number}
4
+ */
5
+ export function columnsToScore(
6
+ columns: {
7
+ name: string;
8
+ type: string;
9
+ }[]
10
+ ): number;
11
+
12
+ /**
13
+ * @param {{ name: string; evidenceType: string }[]} columns
14
+ * @returns {number}
15
+ */
16
+ export function evidenceColumnsToScore(columns: { name: string; evidenceType: string }[]): number;
17
+
18
+ /**
19
+ *
20
+ * @param {string} column_type
21
+ * @returns {'number' | 'boolean' | 'string' | 'date'}
22
+ */
23
+ export function duckdbTypeToEvidenceType(
24
+ column_type: string
25
+ ): 'number' | 'boolean' | 'string' | 'date';