@yolean/evidence-universal-sql 3.0.2 → 3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,5 +1,11 @@
1
1
  # @evidence-dev/universal-sql
2
2
 
3
+ ## 3.1.0
4
+
5
+ ### Minor Changes
6
+
7
+ - 81977b64b: deployment.dataScope: pages with that route parameter load data/ and api/ from <basePath>/<value>/, and prerender their query results there.
8
+
3
9
  ## 3.0.2
4
10
 
5
11
  ### Patch Changes
@@ -17,10 +23,12 @@
17
23
  ### Major Changes
18
24
 
19
25
  - b28f63f23: Update DuckDB to latest packages:
26
+
20
27
  - Switch to @duckdb/node-api from duckdb-async
21
28
  - Update duckdb-wasm to latest release
22
29
 
23
30
  This release also has small data fixes across several packages:
31
+
24
32
  - Better handling of NULL values when discovering column types
25
33
  - Fix batch processing of parquet files
26
34
  - Fix error with temporary parquet files when reloading data in dev environment
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yolean/evidence-universal-sql",
3
- "version": "3.0.2",
3
+ "version": "3.1.0",
4
4
  "description": "",
5
5
  "keywords": [],
6
6
  "author": "",
@@ -8,7 +8,13 @@
8
8
  export function cache_for_hash(
9
9
  sql_string: string,
10
10
  data: import('apache-arrow').Table,
11
- cache_options: { route_hash: string; query_name: string; prerendering: boolean }
11
+ cache_options: {
12
+ route_hash: string;
13
+ additional_hash: string;
14
+ query_name: string;
15
+ prerendering: boolean;
16
+ scope?: string | null;
17
+ }
12
18
  ): void;
13
19
 
14
20
  /**
@@ -57,6 +57,14 @@ function hash(buffer) {
57
57
  */
58
58
  const sql_string_cache = new Map();
59
59
 
60
+ // Pages with a data scope (deployment.dataScope) prerender their results under /<scope>/,
61
+ // so a result is cached per scope: a query with the same result in two scopes is written twice
62
+ /**
63
+ * @param {string} sql
64
+ * @param {string | null | undefined} scope
65
+ */
66
+ const cacheKey = (sql, scope) => (scope ? `${scope}\n${sql}` : sql);
67
+
60
68
  // while prerendering static sites, sveltekit finds endpoints based on what's used in load functions with
61
69
  // the provided fetch - since the data isn't ready yet, instead we write the data to the place where sveltekit
62
70
  // would have written it
@@ -66,19 +74,21 @@ const prerender_path = `.svelte-kit/output/prerendered/dependencies`;
66
74
  * Caches DuckDB queries that have been executed during SSR
67
75
  * @param {string} sql_string SQL string executed
68
76
  * @param {import("apache-arrow").Table} data result of the query
69
- * @param {{ route_hash: string, additional_hash: string, query_name: string, prerendering: boolean }} cache_options
77
+ * @param {{ route_hash: string, additional_hash: string, query_name: string, prerendering: boolean, scope?: string | null }} cache_options
70
78
  * @returns {void}
71
79
  */
72
80
  export function cache_for_hash(
73
81
  sql_string,
74
82
  data,
75
- { route_hash, additional_hash, query_name, prerendering }
83
+ { route_hash, additional_hash, query_name, prerendering, scope }
76
84
  ) {
85
+ const key = cacheKey(sql_string, scope);
86
+ const prerender_scope_path = scope ? `${prerender_path}/${scope}` : prerender_path;
77
87
  // if the it's in the cache then we shouldn't bother
78
- let ipc_table_hash = sql_string_cache.get(sql_string);
88
+ let ipc_table_hash = sql_string_cache.get(key);
79
89
 
80
90
  // this can only be false during `prerendering`
81
- if (!sql_string_cache.has(sql_string)) {
91
+ if (!sql_string_cache.has(key)) {
82
92
  const ipc_table = tableToIPC(data);
83
93
  ipc_table_hash = hash(ipc_table);
84
94
 
@@ -90,10 +100,10 @@ export function cache_for_hash(
90
100
  if (prerendering) {
91
101
  // save the result of the query
92
102
  writeToPossiblyNonexistentFile(
93
- `${prerender_path}/api/prerendered_queries/${ipc_table_hash}.arrow`,
103
+ `${prerender_scope_path}/api/prerendered_queries/${ipc_table_hash}.arrow`,
94
104
  ipc_table
95
105
  );
96
- sql_string_cache.set(sql_string, ipc_table_hash);
106
+ sql_string_cache.set(key, ipc_table_hash);
97
107
  }
98
108
  }
99
109
 
@@ -128,7 +138,7 @@ export function cache_for_hash(
128
138
  if (prerendering) {
129
139
  // keep track of the query names for the page
130
140
  writeToPossiblyNonexistentFile(
131
- `${prerender_path}/api/${route_hash}/${additional_hash}/all-queries.json`,
141
+ `${prerender_scope_path}/api/${route_hash}/${additional_hash}/all-queries.json`,
132
142
  JSON.stringify(sql_cache_with_hashed_query_strings)
133
143
  );
134
144
  }
@@ -175,9 +185,11 @@ export function get_all_page_queries(route_hash, additional_hash) {
175
185
  *
176
186
  * Only runs during build
177
187
  * @param {string} sql
188
+ * @param {string | null} [scope]
178
189
  * @returns {import("apache-arrow").Table | null}
179
190
  */
180
- export function get_arrow_if_sql_already_run(sql) {
181
- if (!sql_string_cache.has(sql)) return null;
182
- return tableFromIPC(get_cache_for_hash(sql_string_cache.get(sql)));
191
+ export function get_arrow_if_sql_already_run(sql, scope) {
192
+ const key = cacheKey(sql, scope);
193
+ if (!sql_string_cache.has(key)) return null;
194
+ return tableFromIPC(get_cache_for_hash(sql_string_cache.get(key)));
183
195
  }
@@ -147,7 +147,7 @@ export function query(sql, cache_options) {
147
147
  // only cache during build, because
148
148
  // parquet can/will eventually change during dev
149
149
  if (cache_options?.prerendering) {
150
- result = get_arrow_if_sql_already_run(sql);
150
+ result = get_arrow_if_sql_already_run(sql, cache_options.scope);
151
151
  }
152
152
 
153
153
  // TODO: This just fails, where is the process going?