@yolean/evidence-universal-sql 3.0.2 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/package.json +1 -1
- package/src/cache-duckdb.d.ts +7 -1
- package/src/cache-duckdb.js +22 -10
- package/src/client-duckdb/node.js +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# @evidence-dev/universal-sql
|
|
2
2
|
|
|
3
|
+
## 3.1.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 81977b64b: deployment.dataScope: pages with that route parameter load data/ and api/ from <basePath>/<value>/, and prerender their query results there.
|
|
8
|
+
|
|
3
9
|
## 3.0.2
|
|
4
10
|
|
|
5
11
|
### Patch Changes
|
|
@@ -17,10 +23,12 @@
|
|
|
17
23
|
### Major Changes
|
|
18
24
|
|
|
19
25
|
- b28f63f23: Update DuckDB to latest packages:
|
|
26
|
+
|
|
20
27
|
- Switch to @duckdb/node-api from duckdb-async
|
|
21
28
|
- Update duckdb-wasm to latest release
|
|
22
29
|
|
|
23
30
|
This release also has small data fixes across several packages:
|
|
31
|
+
|
|
24
32
|
- Better handling of NULL values when discovering column types
|
|
25
33
|
- Fix batch processing of parquet files
|
|
26
34
|
- Fix error with temporary parquet files when reloading data in dev environment
|
package/package.json
CHANGED
package/src/cache-duckdb.d.ts
CHANGED
|
@@ -8,7 +8,13 @@
|
|
|
8
8
|
export function cache_for_hash(
|
|
9
9
|
sql_string: string,
|
|
10
10
|
data: import('apache-arrow').Table,
|
|
11
|
-
cache_options: {
|
|
11
|
+
cache_options: {
|
|
12
|
+
route_hash: string;
|
|
13
|
+
additional_hash: string;
|
|
14
|
+
query_name: string;
|
|
15
|
+
prerendering: boolean;
|
|
16
|
+
scope?: string | null;
|
|
17
|
+
}
|
|
12
18
|
): void;
|
|
13
19
|
|
|
14
20
|
/**
|
package/src/cache-duckdb.js
CHANGED
|
@@ -57,6 +57,14 @@ function hash(buffer) {
|
|
|
57
57
|
*/
|
|
58
58
|
const sql_string_cache = new Map();
|
|
59
59
|
|
|
60
|
+
// Pages with a data scope (deployment.dataScope) prerender their results under /<scope>/,
|
|
61
|
+
// so a result is cached per scope: a query with the same result in two scopes is written twice
|
|
62
|
+
/**
|
|
63
|
+
* @param {string} sql
|
|
64
|
+
* @param {string | null | undefined} scope
|
|
65
|
+
*/
|
|
66
|
+
const cacheKey = (sql, scope) => (scope ? `${scope}\n${sql}` : sql);
|
|
67
|
+
|
|
60
68
|
// while prerendering static sites, sveltekit finds endpoints based on what's used in load functions with
|
|
61
69
|
// the provided fetch - since the data isn't ready yet, instead we write the data to the place where sveltekit
|
|
62
70
|
// would have written it
|
|
@@ -66,19 +74,21 @@ const prerender_path = `.svelte-kit/output/prerendered/dependencies`;
|
|
|
66
74
|
* Caches DuckDB queries that have been executed during SSR
|
|
67
75
|
* @param {string} sql_string SQL string executed
|
|
68
76
|
* @param {import("apache-arrow").Table} data result of the query
|
|
69
|
-
* @param {{ route_hash: string, additional_hash: string, query_name: string, prerendering: boolean }} cache_options
|
|
77
|
+
* @param {{ route_hash: string, additional_hash: string, query_name: string, prerendering: boolean, scope?: string | null }} cache_options
|
|
70
78
|
* @returns {void}
|
|
71
79
|
*/
|
|
72
80
|
export function cache_for_hash(
|
|
73
81
|
sql_string,
|
|
74
82
|
data,
|
|
75
|
-
{ route_hash, additional_hash, query_name, prerendering }
|
|
83
|
+
{ route_hash, additional_hash, query_name, prerendering, scope }
|
|
76
84
|
) {
|
|
85
|
+
const key = cacheKey(sql_string, scope);
|
|
86
|
+
const prerender_scope_path = scope ? `${prerender_path}/${scope}` : prerender_path;
|
|
77
87
|
// if the it's in the cache then we shouldn't bother
|
|
78
|
-
let ipc_table_hash = sql_string_cache.get(
|
|
88
|
+
let ipc_table_hash = sql_string_cache.get(key);
|
|
79
89
|
|
|
80
90
|
// this can only be false during `prerendering`
|
|
81
|
-
if (!sql_string_cache.has(
|
|
91
|
+
if (!sql_string_cache.has(key)) {
|
|
82
92
|
const ipc_table = tableToIPC(data);
|
|
83
93
|
ipc_table_hash = hash(ipc_table);
|
|
84
94
|
|
|
@@ -90,10 +100,10 @@ export function cache_for_hash(
|
|
|
90
100
|
if (prerendering) {
|
|
91
101
|
// save the result of the query
|
|
92
102
|
writeToPossiblyNonexistentFile(
|
|
93
|
-
`${
|
|
103
|
+
`${prerender_scope_path}/api/prerendered_queries/${ipc_table_hash}.arrow`,
|
|
94
104
|
ipc_table
|
|
95
105
|
);
|
|
96
|
-
sql_string_cache.set(
|
|
106
|
+
sql_string_cache.set(key, ipc_table_hash);
|
|
97
107
|
}
|
|
98
108
|
}
|
|
99
109
|
|
|
@@ -128,7 +138,7 @@ export function cache_for_hash(
|
|
|
128
138
|
if (prerendering) {
|
|
129
139
|
// keep track of the query names for the page
|
|
130
140
|
writeToPossiblyNonexistentFile(
|
|
131
|
-
`${
|
|
141
|
+
`${prerender_scope_path}/api/${route_hash}/${additional_hash}/all-queries.json`,
|
|
132
142
|
JSON.stringify(sql_cache_with_hashed_query_strings)
|
|
133
143
|
);
|
|
134
144
|
}
|
|
@@ -175,9 +185,11 @@ export function get_all_page_queries(route_hash, additional_hash) {
|
|
|
175
185
|
*
|
|
176
186
|
* Only runs during build
|
|
177
187
|
* @param {string} sql
|
|
188
|
+
* @param {string | null} [scope]
|
|
178
189
|
* @returns {import("apache-arrow").Table | null}
|
|
179
190
|
*/
|
|
180
|
-
export function get_arrow_if_sql_already_run(sql) {
|
|
181
|
-
|
|
182
|
-
|
|
191
|
+
export function get_arrow_if_sql_already_run(sql, scope) {
|
|
192
|
+
const key = cacheKey(sql, scope);
|
|
193
|
+
if (!sql_string_cache.has(key)) return null;
|
|
194
|
+
return tableFromIPC(get_cache_for_hash(sql_string_cache.get(key)));
|
|
183
195
|
}
|
|
@@ -147,7 +147,7 @@ export function query(sql, cache_options) {
|
|
|
147
147
|
// only cache during build, because
|
|
148
148
|
// parquet can/will eventually change during dev
|
|
149
149
|
if (cache_options?.prerendering) {
|
|
150
|
-
result = get_arrow_if_sql_already_run(sql);
|
|
150
|
+
result = get_arrow_if_sql_already_run(sql, cache_options.scope);
|
|
151
151
|
}
|
|
152
152
|
|
|
153
153
|
// TODO: This just fails, where is the process going?
|