@akohn/dashql 0.0.19-dev.305.gb0f831a47 → 0.0.19-dev.368.g7091b292b
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/core/api.d.ts +24 -18
- package/dist/core/api.js +42 -46
- package/dist/core/connection.d.ts +1 -1
- package/dist/core/index.d.ts +0 -4
- package/dist/core/index.js +1 -4
- package/dist/core/runtime.d.ts +1 -1
- package/dist/core/workbook.d.ts +1 -1
- package/dist/platform/compute/arrow_formatter.d.ts +3 -1
- package/dist/platform/compute/arrow_formatter.js +23 -20
- package/dist/platform/compute/computation_types.d.ts +7 -12
- package/dist/platform/compute/computation_types.js +6 -3
- package/dist/platform/compute/query_result_search_sql.d.ts +0 -7
- package/dist/platform/compute/query_result_search_sql.js +1 -19
- package/dist/platform/connections/connection_params.d.ts +5 -6
- package/dist/platform/connections/connection_params.js +14 -25
- package/dist/platform/connections/connection_state.d.ts +1 -1
- package/dist/platform/connections/connector_configs.d.ts +2 -3
- package/dist/platform/connections/connector_info.d.ts +4 -14
- package/dist/platform/connections/connector_info.js +1 -25
- package/dist/platform/connections/embedded_database_channel.d.ts +3 -3
- package/dist/platform/connections/embedded_database_channel.js +12 -6
- package/dist/platform/connections/hyper/hyper_error_info.d.ts +0 -15
- package/dist/platform/connections/hyper/hyper_error_info.js +1 -18
- package/dist/platform/connections/query_execution.js +9 -7
- package/dist/platform/connections/query_execution_args.d.ts +2 -2
- package/dist/platform/http_client.d.ts +1 -1
- package/dist/platform/logger/logger.d.ts +3 -3
- package/dist/platform/query/query_execution_state.d.ts +11 -38
- package/dist/platform/query/query_execution_state.js +6 -1
- package/dist/platform/query/query_progress.d.ts +0 -14
- package/dist/platform/query/query_result_cache.d.ts +2 -2
- package/dist/platform/storage/composite_storage_backend.d.ts +64 -0
- package/dist/platform/storage/composite_storage_backend.js +321 -0
- package/dist/platform/storage/native_storage_backend.d.ts +1 -1
- package/dist/platform/storage/native_storage_backend.js +1 -1
- package/dist/platform/storage/opfs_query_result_cache.d.ts +0 -2
- package/dist/platform/storage/opfs_query_result_cache.js +3 -5
- package/dist/platform/storage/opfs_storage_backend.d.ts +67 -0
- package/dist/platform/storage/opfs_storage_backend.js +494 -0
- package/dist/platform/storage/workbook_locator.d.ts +7 -0
- package/dist/platform/storage/workbook_locator.js +14 -0
- package/dist/platform/storage/workbook_tables.d.ts +1 -4
- package/dist/platform/storage/workbook_tables.js +3 -17
- package/dist/platform/storage/workbook_transfer.d.ts +9 -0
- package/dist/platform/storage/workbook_transfer.js +84 -0
- package/dist/platform/storage/workbook_tree.d.ts +30 -9
- package/dist/platform/storage/workbook_tree.js +17 -96
- package/dist/platform/utils/base64.js +4 -4
- package/dist/platform/web_http_client.d.ts +1 -4
- package/dist/platform/web_http_client.js +0 -5
- package/dist/platform/workbook/attachment_mounts.d.ts +12 -4
- package/dist/platform/workbook/attachment_mounts.js +69 -60
- package/dist/platform/workbook/run_events.js +1 -1
- package/dist/platform/workbook/script_compilation.d.ts +0 -1
- package/dist/platform/workbook/script_compilation.js +1 -14
- package/dist/platform/workbook/script_reference_check.js +7 -6
- package/dist/platform/workbook/script_types.d.ts +1 -1
- package/dist/platform/workbook/script_types.js +1 -10
- package/dist/platform/workbook/workbook_execution.d.ts +0 -2
- package/dist/platform/workbook/workbook_execution.js +1 -4
- package/dist/platform/workbook/workbook_inputs.d.ts +7 -7
- package/dist/platform/workbook/workbook_inputs.js +4 -4
- package/dist/platform/workbook/workbook_scripts.d.ts +2 -2
- package/dist/runtime/compute_platform.js +2 -2
- package/dist/runtime/node/node_platform.js +1 -1
- package/dist/runtime/platform_requests.js +1 -1
- package/dist/runtime/query/query_channels.d.ts +0 -2
- package/dist/runtime/query/query_channels.js +36 -55
- package/dist/runtime/run_mounts.d.ts +2 -1
- package/dist/runtime/run_mounts.js +4 -5
- package/dist/runtime/runtime.d.ts +3 -0
- package/dist/runtime/runtime.js +10 -6
- package/dist/runtime/runtime_storage.d.ts +2 -5
- package/dist/runtime/runtime_storage.js +4 -13
- package/dist/runtime/workbook_projection.js +2 -2
- package/dist/runtime/workbook_run_report.d.ts +3 -6
- package/dist/runtime/workbook_run_report.js +33 -47
- package/dist/wasm/dashql_core.d.ts +2363 -2090
- package/dist/wasm/dashql_core.js +362 -222
- package/dist/wasm/dashql_core.wasm +0 -0
- package/dist/wasm/dashql_dataflow.wasm +0 -0
- package/package.json +1 -1
- package/dist/core/format.d.ts +0 -4
- package/dist/core/format.js +0 -23
- package/dist/core/lookup.d.ts +0 -10
- package/dist/core/lookup.js +0 -129
- package/dist/core/tokens.d.ts +0 -9
- package/dist/core/tokens.js +0 -78
- package/dist/core/variant.d.ts +0 -4
- package/dist/core/variant.js +0 -0
- package/dist/platform/connections/hyper/local_cloud.d.ts +0 -15
- package/dist/platform/connections/hyper/local_cloud.js +0 -14
- package/dist/platform/connections/query_channel.d.ts +0 -3
- package/dist/platform/connections/query_channel.js +0 -11
- package/dist/platform/hyper_link_target.d.ts +0 -8
- package/dist/platform/hyper_link_target.js +0 -35
- package/dist/platform/sql/sql_format.d.ts +0 -9
- package/dist/platform/sql/sql_format.js +0 -40
- package/dist/platform/sql/sql_lint.d.ts +0 -27
- package/dist/platform/sql/sql_lint.js +0 -69
- package/dist/platform/sql/sql_report.d.ts +0 -18
- package/dist/platform/sql/sql_report.js +0 -80
- package/dist/platform/storage/storage_versions.d.ts +0 -3
- package/dist/platform/storage/storage_versions.js +0 -13
- package/dist/platform/utils/async_consumer.d.ts +0 -14
- package/dist/platform/utils/async_consumer.js +0 -24
- package/dist/platform/workbook/attachment_mirror.d.ts +0 -2
- package/dist/platform/workbook/attachment_mirror.js +0 -25
|
@@ -0,0 +1,321 @@
|
|
|
1
|
+
import { StorageBackendType } from "./storage_backend.js";
|
|
2
|
+
import { applyWorkbookTreeChange, emptyWorkbookTree, flattenWorkbookTree } from "./workbook_tree.js";
|
|
3
|
+
import { NativeStorageBackend } from "./native_storage_backend.js";
|
|
4
|
+
import { locationFromEntry } from "./workbook_locator.js";
|
|
5
|
+
import { copyWorkbook, verifyWorkbook } from "./workbook_transfer.js";
|
|
6
|
+
import { ELECTRON_FILE_SYSTEM } from "../electron_fs.js";
|
|
7
|
+
// dist/.stage/src/platform/storage/composite_storage_backend.ts
|
|
8
|
+
var LOG_CTX = "composite_storage_backend";
|
|
9
|
+
var CompositeStorageBackend = class {
|
|
10
|
+
opfs;
|
|
11
|
+
locations;
|
|
12
|
+
tree = emptyWorkbookTree();
|
|
13
|
+
nativeCache;
|
|
14
|
+
mirroredAttachments;
|
|
15
|
+
logger;
|
|
16
|
+
core;
|
|
17
|
+
constructor(core, opfs, logger) {
|
|
18
|
+
this.core = core;
|
|
19
|
+
this.opfs = opfs;
|
|
20
|
+
this.locations = /* @__PURE__ */ new Map();
|
|
21
|
+
this.nativeCache = /* @__PURE__ */ new Map();
|
|
22
|
+
this.mirroredAttachments = /* @__PURE__ */ new Map();
|
|
23
|
+
this.logger = logger;
|
|
24
|
+
}
|
|
25
|
+
getBackendType() {
|
|
26
|
+
return StorageBackendType.OPFS;
|
|
27
|
+
}
|
|
28
|
+
async initialize() {
|
|
29
|
+
await this.opfs.initialize?.();
|
|
30
|
+
await this.refreshLocations();
|
|
31
|
+
}
|
|
32
|
+
async refreshLocations() {
|
|
33
|
+
this.tree = await this.opfs.loadWorkbookTree();
|
|
34
|
+
this.locations.clear();
|
|
35
|
+
for (const entry of flattenWorkbookTree(this.tree)) this.locations.set(entry.path, locationFromEntry(entry));
|
|
36
|
+
}
|
|
37
|
+
async refreshTree() {
|
|
38
|
+
this.tree = await this.opfs.loadWorkbookTree();
|
|
39
|
+
}
|
|
40
|
+
locationOf(workbookId) {
|
|
41
|
+
return this.locations.get(workbookId) ?? { type: StorageBackendType.OPFS };
|
|
42
|
+
}
|
|
43
|
+
async backendFor(workbookId) {
|
|
44
|
+
return this.nativeWorkbookFolder(workbookId) ?? this.opfs;
|
|
45
|
+
}
|
|
46
|
+
nativeWorkbookFolder(workbookId) {
|
|
47
|
+
const loc = this.locationOf(workbookId);
|
|
48
|
+
if (loc.type !== StorageBackendType.Native || !loc.nativePath) return null;
|
|
49
|
+
let native = this.nativeCache.get(workbookId);
|
|
50
|
+
if (!native) {
|
|
51
|
+
native = new NativeStorageBackend(loc.nativePath, ELECTRON_FILE_SYSTEM);
|
|
52
|
+
this.nativeCache.set(workbookId, native);
|
|
53
|
+
}
|
|
54
|
+
return native;
|
|
55
|
+
}
|
|
56
|
+
listWorkbooks() {
|
|
57
|
+
return this.opfs.listWorkbooks();
|
|
58
|
+
}
|
|
59
|
+
loadWorkbookTree() {
|
|
60
|
+
return this.opfs.loadWorkbookTree();
|
|
61
|
+
}
|
|
62
|
+
loadAppSettings() {
|
|
63
|
+
return this.opfs.loadAppSettings();
|
|
64
|
+
}
|
|
65
|
+
saveAppSettings(settings) {
|
|
66
|
+
return this.opfs.saveAppSettings(settings);
|
|
67
|
+
}
|
|
68
|
+
loadManifest() {
|
|
69
|
+
return this.opfs.loadManifest();
|
|
70
|
+
}
|
|
71
|
+
updateConnections(update) {
|
|
72
|
+
return this.opfs.updateConnections(update);
|
|
73
|
+
}
|
|
74
|
+
loadConnectionFile(connectionId, fileName) {
|
|
75
|
+
return this.opfs.loadConnectionFile(connectionId, fileName);
|
|
76
|
+
}
|
|
77
|
+
saveConnectionFile(connectionId, fileName, text) {
|
|
78
|
+
return this.opfs.saveConnectionFile(connectionId, fileName, text);
|
|
79
|
+
}
|
|
80
|
+
createConnectionFolder(connectionId) {
|
|
81
|
+
return this.opfs.createConnectionFolder(connectionId);
|
|
82
|
+
}
|
|
83
|
+
deleteConnectionFiles(connectionId) {
|
|
84
|
+
return this.opfs.deleteConnectionFiles(connectionId);
|
|
85
|
+
}
|
|
86
|
+
async upsertWorkbookEntry(entry, folderPath = []) {
|
|
87
|
+
await this.opfs.upsertWorkbookEntry(entry, folderPath);
|
|
88
|
+
this.locations.set(entry.path, locationFromEntry(entry));
|
|
89
|
+
this.tree = applyWorkbookTreeChange(this.core, this.tree, {
|
|
90
|
+
kind: "upsert",
|
|
91
|
+
entry,
|
|
92
|
+
folderPath
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
async removeWorkbookEntry(workbookId) {
|
|
96
|
+
await this.opfs.removeWorkbookEntry(workbookId);
|
|
97
|
+
this.forgetWorkbook(workbookId);
|
|
98
|
+
}
|
|
99
|
+
async moveWorkbook(workbookId, folderPath, index) {
|
|
100
|
+
await this.opfs.moveWorkbook(workbookId, folderPath, index);
|
|
101
|
+
await this.refreshTree();
|
|
102
|
+
}
|
|
103
|
+
async createFolder(path) {
|
|
104
|
+
await this.opfs.createFolder(path);
|
|
105
|
+
await this.refreshTree();
|
|
106
|
+
}
|
|
107
|
+
async moveFolder(path, parentPath, name) {
|
|
108
|
+
await this.opfs.moveFolder(path, parentPath, name);
|
|
109
|
+
await this.refreshTree();
|
|
110
|
+
}
|
|
111
|
+
async deleteFolder(path) {
|
|
112
|
+
await this.opfs.deleteFolder(path);
|
|
113
|
+
await this.refreshTree();
|
|
114
|
+
}
|
|
115
|
+
forgetWorkbook(workbookId) {
|
|
116
|
+
this.locations.delete(workbookId);
|
|
117
|
+
this.nativeCache.delete(workbookId);
|
|
118
|
+
this.mirroredAttachments.delete(workbookId);
|
|
119
|
+
this.tree = applyWorkbookTreeChange(this.core, this.tree, {
|
|
120
|
+
kind: "remove",
|
|
121
|
+
workbookId
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
getWorkbookTree() {
|
|
125
|
+
return this.tree;
|
|
126
|
+
}
|
|
127
|
+
getWorkbookOrder() {
|
|
128
|
+
const order = flattenWorkbookTree(this.tree).map((entry) => entry.path);
|
|
129
|
+
const listed = new Set(order);
|
|
130
|
+
for (const id of this.locations.keys()) if (!listed.has(id)) order.push(id);
|
|
131
|
+
return order;
|
|
132
|
+
}
|
|
133
|
+
deleteWorkbookFiles(workbookId) {
|
|
134
|
+
return this.backendFor(workbookId).then((b) => {
|
|
135
|
+
if (b === this.opfs) return this.opfs.deleteWorkbookFiles(workbookId);
|
|
136
|
+
return b.deleteWorkbook(workbookId);
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
async loadWorkbook(workbookId) {
|
|
140
|
+
const workbook = await (await this.backendFor(workbookId)).loadWorkbook(workbookId);
|
|
141
|
+
await this.syncNativeAttachments(workbookId);
|
|
142
|
+
return workbook;
|
|
143
|
+
}
|
|
144
|
+
async syncNativeAttachments(workbookId) {
|
|
145
|
+
const native = this.nativeWorkbookFolder(workbookId);
|
|
146
|
+
if (native == null) return;
|
|
147
|
+
const files = (await native.stampFolder()).attachments;
|
|
148
|
+
let mirrored = this.mirroredAttachments.get(workbookId);
|
|
149
|
+
if (mirrored === void 0) {
|
|
150
|
+
mirrored = /* @__PURE__ */ new Map();
|
|
151
|
+
this.mirroredAttachments.set(workbookId, mirrored);
|
|
152
|
+
}
|
|
153
|
+
const names = /* @__PURE__ */ new Set();
|
|
154
|
+
for (const file of files) {
|
|
155
|
+
this.core.validateDataFileName(file.name);
|
|
156
|
+
names.add(file.name);
|
|
157
|
+
const stamp = mirrored.get(file.name);
|
|
158
|
+
if (stamp !== void 0 && stamp.size === file.size && stamp.mtimeMs === file.mtimeMs) continue;
|
|
159
|
+
mirrored.delete(file.name);
|
|
160
|
+
await this.opfs.saveDataFile(workbookId, file.name, await native.loadDataFile(workbookId, file.name));
|
|
161
|
+
mirrored.set(file.name, {
|
|
162
|
+
size: file.size,
|
|
163
|
+
mtimeMs: file.mtimeMs
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
for (const file of await this.opfs.listDataFiles(workbookId)) if (!names.has(file.name)) {
|
|
167
|
+
await this.opfs.deleteDataFile(workbookId, file.name);
|
|
168
|
+
mirrored.delete(file.name);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
async saveWorkbookManifest(workbookId, data, expected) {
|
|
172
|
+
const stamp = await (await this.backendFor(workbookId)).saveWorkbookManifest(workbookId, data, expected) ?? null;
|
|
173
|
+
await this.registerSavedWorkbook(workbookId);
|
|
174
|
+
return stamp;
|
|
175
|
+
}
|
|
176
|
+
async registerSavedWorkbook(workbookId) {
|
|
177
|
+
const loc = this.locationOf(workbookId);
|
|
178
|
+
if (loc.type === StorageBackendType.Native && loc.nativePath) await this.opfs.upsertWorkbookEntry({
|
|
179
|
+
path: workbookId,
|
|
180
|
+
storageType: StorageBackendType.Native,
|
|
181
|
+
nativePath: loc.nativePath
|
|
182
|
+
});
|
|
183
|
+
else {
|
|
184
|
+
this.locations.set(workbookId, loc);
|
|
185
|
+
this.tree = applyWorkbookTreeChange(this.core, this.tree, {
|
|
186
|
+
kind: "upsert",
|
|
187
|
+
entry: {
|
|
188
|
+
path: workbookId,
|
|
189
|
+
storageType: StorageBackendType.OPFS
|
|
190
|
+
}
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
async deleteWorkbook(workbookId) {
|
|
195
|
+
if (this.locationOf(workbookId).type === StorageBackendType.Native) {
|
|
196
|
+
await this.opfs.removeWorkbookEntry(workbookId);
|
|
197
|
+
await this.opfs.deleteWorkbookFiles(workbookId);
|
|
198
|
+
} else await this.opfs.deleteWorkbook(workbookId);
|
|
199
|
+
this.forgetWorkbook(workbookId);
|
|
200
|
+
}
|
|
201
|
+
async loadScripts(workbookId) {
|
|
202
|
+
return (await this.backendFor(workbookId)).loadScripts(workbookId);
|
|
203
|
+
}
|
|
204
|
+
async loadScript(workbookId, scriptName) {
|
|
205
|
+
return (await this.backendFor(workbookId)).loadScript(workbookId, scriptName);
|
|
206
|
+
}
|
|
207
|
+
async saveScript(workbookId, scriptName, sql, expected) {
|
|
208
|
+
return await (await this.backendFor(workbookId)).saveScript(workbookId, scriptName, sql, expected) ?? null;
|
|
209
|
+
}
|
|
210
|
+
async deleteScript(workbookId, scriptName, expected) {
|
|
211
|
+
return (await this.backendFor(workbookId)).deleteScript(workbookId, scriptName, expected);
|
|
212
|
+
}
|
|
213
|
+
async renameScript(workbookId, oldScriptName, newScriptName, expectedFrom, expectedTo) {
|
|
214
|
+
return (await this.backendFor(workbookId)).renameScript(workbookId, oldScriptName, newScriptName, expectedFrom, expectedTo);
|
|
215
|
+
}
|
|
216
|
+
async listDataFiles(workbookId) {
|
|
217
|
+
return (await this.backendFor(workbookId)).listDataFiles(workbookId);
|
|
218
|
+
}
|
|
219
|
+
async loadDataFile(workbookId, name) {
|
|
220
|
+
return (await this.backendFor(workbookId)).loadDataFile(workbookId, name);
|
|
221
|
+
}
|
|
222
|
+
async saveDataFile(workbookId, name, content) {
|
|
223
|
+
await (await this.backendFor(workbookId)).saveDataFile(workbookId, name, content);
|
|
224
|
+
await this.syncNativeAttachments(workbookId);
|
|
225
|
+
}
|
|
226
|
+
async deleteDataFile(workbookId, name) {
|
|
227
|
+
await (await this.backendFor(workbookId)).deleteDataFile(workbookId, name);
|
|
228
|
+
await this.syncNativeAttachments(workbookId);
|
|
229
|
+
}
|
|
230
|
+
async renameDataFile(workbookId, oldName, newName) {
|
|
231
|
+
await (await this.backendFor(workbookId)).renameDataFile(workbookId, oldName, newName);
|
|
232
|
+
await this.syncNativeAttachments(workbookId);
|
|
233
|
+
}
|
|
234
|
+
async clearAllStorage() {
|
|
235
|
+
this.nativeCache.clear();
|
|
236
|
+
this.mirroredAttachments.clear();
|
|
237
|
+
this.locations.clear();
|
|
238
|
+
this.tree = emptyWorkbookTree();
|
|
239
|
+
await this.opfs.clearAllStorage?.();
|
|
240
|
+
}
|
|
241
|
+
getWorkbookLocation(workbookId) {
|
|
242
|
+
return this.locationOf(workbookId);
|
|
243
|
+
}
|
|
244
|
+
async registerNativeWorkbook(dir, workbookId) {
|
|
245
|
+
const replaced = this.locations.get(workbookId);
|
|
246
|
+
const native = new NativeStorageBackend(dir, ELECTRON_FILE_SYSTEM);
|
|
247
|
+
await this.upsertWorkbookEntry({
|
|
248
|
+
path: workbookId,
|
|
249
|
+
storageType: StorageBackendType.Native,
|
|
250
|
+
nativePath: dir
|
|
251
|
+
});
|
|
252
|
+
this.nativeCache.set(workbookId, native);
|
|
253
|
+
if (replaced?.type === StorageBackendType.OPFS) await this.opfs.deleteWorkbookFiles(workbookId);
|
|
254
|
+
}
|
|
255
|
+
importFolder(dir) {
|
|
256
|
+
const native = new NativeStorageBackend(dir, ELECTRON_FILE_SYSTEM);
|
|
257
|
+
const failed = (error) => /* @__PURE__ */ new Error(`Failed to read workbook in ${dir}: ${error?.message ?? String(error)}`);
|
|
258
|
+
return {
|
|
259
|
+
errorPrefix: `Invalid workbook folder ${dir}`,
|
|
260
|
+
nativePath: dir,
|
|
261
|
+
readText: (path) => native.readText(path),
|
|
262
|
+
scripts: async () => {
|
|
263
|
+
try {
|
|
264
|
+
return (await native.loadScripts(dir)).map((script) => ({
|
|
265
|
+
name: script.name,
|
|
266
|
+
sql: script.sql
|
|
267
|
+
}));
|
|
268
|
+
} catch (error) {
|
|
269
|
+
throw failed(error);
|
|
270
|
+
}
|
|
271
|
+
},
|
|
272
|
+
attachments: async () => {
|
|
273
|
+
try {
|
|
274
|
+
return await Promise.all((await native.listDataFiles(dir)).map(async (file) => ({
|
|
275
|
+
name: file.name,
|
|
276
|
+
file: await native.loadDataFile(dir, file.name)
|
|
277
|
+
})));
|
|
278
|
+
} catch (error) {
|
|
279
|
+
throw failed(error);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
async relocateWorkbookToNative(uuid, dir) {
|
|
285
|
+
if (this.locationOf(uuid).type !== StorageBackendType.OPFS) throw new Error(`Workbook ${uuid} is not an OPFS workbook`);
|
|
286
|
+
const native = new NativeStorageBackend(dir, ELECTRON_FILE_SYSTEM);
|
|
287
|
+
await native.initialize();
|
|
288
|
+
const workbookData = await this.opfs.loadWorkbook(uuid);
|
|
289
|
+
await copyWorkbook(uuid, this.opfs, native, this.logger);
|
|
290
|
+
await native.saveWorkbookManifest(uuid, {
|
|
291
|
+
...workbookData,
|
|
292
|
+
storageType: StorageBackendType.Native,
|
|
293
|
+
nativePath: dir
|
|
294
|
+
});
|
|
295
|
+
if (!await verifyWorkbook(uuid, this.opfs, native)) throw new Error(`Relocation verification failed for workbook ${uuid} - keeping OPFS copy`);
|
|
296
|
+
await this.opfs.upsertWorkbookEntry({
|
|
297
|
+
path: uuid,
|
|
298
|
+
storageType: StorageBackendType.Native,
|
|
299
|
+
nativePath: dir
|
|
300
|
+
});
|
|
301
|
+
this.locations.set(uuid, {
|
|
302
|
+
type: StorageBackendType.Native,
|
|
303
|
+
nativePath: dir
|
|
304
|
+
});
|
|
305
|
+
this.tree = applyWorkbookTreeChange(this.core, this.tree, {
|
|
306
|
+
kind: "upsert",
|
|
307
|
+
entry: {
|
|
308
|
+
path: uuid,
|
|
309
|
+
storageType: StorageBackendType.Native,
|
|
310
|
+
nativePath: dir
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
this.nativeCache.set(uuid, native);
|
|
314
|
+
await this.opfs.deleteWorkbookFiles(uuid);
|
|
315
|
+
this.logger.info("relocated workbook to native storage", {
|
|
316
|
+
workbookId: uuid,
|
|
317
|
+
dir
|
|
318
|
+
}, LOG_CTX);
|
|
319
|
+
}
|
|
320
|
+
};
|
|
321
|
+
export { CompositeStorageBackend };
|
|
@@ -27,7 +27,7 @@ export declare class NativeStorageBackend implements StorageBackend {
|
|
|
27
27
|
readText(path: string): Promise<string | null>;
|
|
28
28
|
writeText(name: string, text: string): Promise<FileStamp>;
|
|
29
29
|
remove(name: string): Promise<void>;
|
|
30
|
-
loadWorkbook(
|
|
30
|
+
loadWorkbook(_workbookId: string): Promise<WorkbookData>;
|
|
31
31
|
private ensureGitignore;
|
|
32
32
|
saveWorkbookManifest(workbookId: string, data: WorkbookData, expected?: ExpectedStamp): Promise<FileStamp>;
|
|
33
33
|
deleteWorkbook(_workbookId: string): Promise<void>;
|
|
@@ -58,7 +58,7 @@ var NativeStorageBackend = class {
|
|
|
58
58
|
const path = await this.abs(name);
|
|
59
59
|
if (await this.fs.exists(path)) await this.fs.remove(path);
|
|
60
60
|
}
|
|
61
|
-
async loadWorkbook(
|
|
61
|
+
async loadWorkbook(_workbookId) {
|
|
62
62
|
const text = await this.readWorkbookFile(STORAGE_WORKBOOK_FILE);
|
|
63
63
|
if (text == null) throw new Error(`Missing ${STORAGE_WORKBOOK_FILE} in ${this.dir}`);
|
|
64
64
|
return JSON.parse(text);
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import type { QueryResultCache } from '../query/query_result_cache.js';
|
|
2
2
|
export declare const QUERY_RESULT_CACHE_DIRECTORY = "cache";
|
|
3
3
|
export declare const QUERY_RESULT_CACHE_MOUNT = "/mnt/external/cache";
|
|
4
|
-
export declare const DEFAULT_CACHE_MAX_BYTES: number;
|
|
5
|
-
export declare const DEFAULT_CACHE_MAX_FILES = 1000;
|
|
6
4
|
export interface OPFSQueryResultCacheOptions {
|
|
7
5
|
getRoot?: () => Promise<FileSystemDirectoryHandle>;
|
|
8
6
|
locks?: LockManager;
|
|
@@ -9,8 +9,6 @@ var KEY_FILE = /^([0-9a-f]{64})\.[0-9a-f-]{36}\.parquet$/;
|
|
|
9
9
|
var TRANSIENT_SUFFIX = ".transient.parquet";
|
|
10
10
|
var GARBAGE_AFTER_MS = 36e5;
|
|
11
11
|
var LOCK_PREFIX = "dashql-query-result-cache/";
|
|
12
|
-
var DEFAULT_CACHE_MAX_BYTES = 2147483648;
|
|
13
|
-
var DEFAULT_CACHE_MAX_FILES = 1e3;
|
|
14
12
|
function cacheFileKey(name) {
|
|
15
13
|
return KEY_FILE.exec(name)?.[1] ?? null;
|
|
16
14
|
}
|
|
@@ -20,8 +18,8 @@ function isTransientCacheFile(name) {
|
|
|
20
18
|
function createOPFSQueryResultCache(options = {}) {
|
|
21
19
|
const getRoot = options.getRoot ?? getOPFSRoot;
|
|
22
20
|
const locks = options.locks ?? navigator.locks;
|
|
23
|
-
const maxBytes = options.maxBytes ??
|
|
24
|
-
const maxFiles = options.maxFiles ??
|
|
21
|
+
const maxBytes = options.maxBytes ?? null;
|
|
22
|
+
const maxFiles = options.maxFiles ?? null;
|
|
25
23
|
const now = options.now ?? Date.now;
|
|
26
24
|
const directory = async () => await (await getRoot()).getDirectoryHandle(QUERY_RESULT_CACHE_DIRECTORY, { create: true });
|
|
27
25
|
const listFiles = async () => {
|
|
@@ -234,4 +232,4 @@ function createOPFSQueryResultCache(options = {}) {
|
|
|
234
232
|
remove: tryRemove
|
|
235
233
|
};
|
|
236
234
|
}
|
|
237
|
-
export {
|
|
235
|
+
export { QUERY_RESULT_CACHE_DIRECTORY, QUERY_RESULT_CACHE_MOUNT, cacheFileKey, createOPFSQueryResultCache, isTransientCacheFile };
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { type WorkbookRegistryBackend, type ConnectionEntry, type WorkbookData, type ScriptData, type WorkbookEntry, type StorageManifest, type AppSettings, StorageBackendType } from './storage_backend.js';
|
|
2
|
+
import { type DataFile, type DataFileContent } from './storage_backend.js';
|
|
3
|
+
import type * as core from '../../core/index.js';
|
|
4
|
+
import { type WorkbookTreeFolder } from './workbook_tree.js';
|
|
5
|
+
export declare const OPFS_MANIFEST_LOCK = "dashql-manifest";
|
|
6
|
+
export declare const OPFS_WORKBOOK_TREE_LOCK = "dashql-workbook-tree";
|
|
7
|
+
export declare class OPFSStorageBackend implements WorkbookRegistryBackend {
|
|
8
|
+
private readonly core;
|
|
9
|
+
private readonly getRoot;
|
|
10
|
+
private rootHandle;
|
|
11
|
+
private manifestChain;
|
|
12
|
+
private treeChain;
|
|
13
|
+
constructor(core: core.DashQL, getRoot?: () => Promise<FileSystemDirectoryHandle>);
|
|
14
|
+
getBackendType(): StorageBackendType;
|
|
15
|
+
initialize(): Promise<void>;
|
|
16
|
+
private createWorkbookTree;
|
|
17
|
+
private withManifestLock;
|
|
18
|
+
private withTreeLock;
|
|
19
|
+
private mutateManifest;
|
|
20
|
+
private ensureInitialized;
|
|
21
|
+
private workbookRelPath;
|
|
22
|
+
private naturalSort;
|
|
23
|
+
listWorkbooks(): Promise<WorkbookEntry[]>;
|
|
24
|
+
loadWorkbook(workbookId: string): Promise<WorkbookData>;
|
|
25
|
+
saveWorkbookManifest(workbookId: string, data: WorkbookData): Promise<void>;
|
|
26
|
+
deleteWorkbook(workbookId: string): Promise<void>;
|
|
27
|
+
deleteWorkbookFiles(workbookId: string): Promise<void>;
|
|
28
|
+
loadScripts(workbookId: string): Promise<ScriptData[]>;
|
|
29
|
+
private moveFile;
|
|
30
|
+
loadScript(workbookId: string, scriptName: string): Promise<ScriptData>;
|
|
31
|
+
saveScript(workbookId: string, scriptName: string, sql: string): Promise<void>;
|
|
32
|
+
deleteScript(workbookId: string, scriptName: string): Promise<void>;
|
|
33
|
+
renameScript(workbookId: string, oldScriptName: string, newScriptName: string): Promise<void>;
|
|
34
|
+
private dataDir;
|
|
35
|
+
listDataFiles(workbookId: string): Promise<DataFile[]>;
|
|
36
|
+
loadDataFile(workbookId: string, name: string): Promise<Uint8Array>;
|
|
37
|
+
saveDataFile(workbookId: string, name: string, content: DataFileContent): Promise<void>;
|
|
38
|
+
deleteDataFile(workbookId: string, name: string): Promise<void>;
|
|
39
|
+
renameDataFile(workbookId: string, oldName: string, newName: string): Promise<void>;
|
|
40
|
+
private writeEmptyFile;
|
|
41
|
+
clearAllStorage(): Promise<void>;
|
|
42
|
+
loadAppSettings(): Promise<AppSettings | null>;
|
|
43
|
+
saveAppSettings(settings: AppSettings): Promise<void>;
|
|
44
|
+
loadManifest(): Promise<StorageManifest>;
|
|
45
|
+
updateConnections(update: (connections: ConnectionEntry[] | undefined) => ConnectionEntry[] | null): Promise<void>;
|
|
46
|
+
private connectionRelPath;
|
|
47
|
+
loadConnectionFile(connectionId: string, fileName: string): Promise<string | null>;
|
|
48
|
+
saveConnectionFile(connectionId: string, fileName: string, text: string): Promise<void>;
|
|
49
|
+
createConnectionFolder(connectionId: string): Promise<void>;
|
|
50
|
+
deleteConnectionFiles(connectionId: string): Promise<void>;
|
|
51
|
+
private readManifest;
|
|
52
|
+
private writeManifest;
|
|
53
|
+
loadWorkbookTree(): Promise<WorkbookTreeFolder>;
|
|
54
|
+
upsertWorkbookEntry(entry: WorkbookEntry, folderPath?: readonly string[]): Promise<void>;
|
|
55
|
+
removeWorkbookEntry(workbookId: string): Promise<void>;
|
|
56
|
+
moveWorkbook(workbookId: string, folderPath: string[], index: number): Promise<void>;
|
|
57
|
+
createFolder(path: string[]): Promise<void>;
|
|
58
|
+
moveFolder(path: string[], parentPath: string[], name?: string): Promise<void>;
|
|
59
|
+
deleteFolder(path: string[]): Promise<void>;
|
|
60
|
+
private readTree;
|
|
61
|
+
private mutateTree;
|
|
62
|
+
private folderDir;
|
|
63
|
+
private writeFolderFile;
|
|
64
|
+
private writeFolderText;
|
|
65
|
+
private copyFolderUnlocked;
|
|
66
|
+
private getWorkbookDir;
|
|
67
|
+
}
|