@akohn/dashql 0.0.19-dev.305.gb0f831a47 → 0.0.19-dev.369.g290145cb4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -2
- package/dist/core/api.d.ts +24 -18
- package/dist/core/api.js +42 -46
- package/dist/core/connection.d.ts +1 -1
- package/dist/core/index.d.ts +0 -4
- package/dist/core/index.js +1 -4
- package/dist/core/runtime.d.ts +1 -1
- package/dist/core/workbook.d.ts +1 -1
- package/dist/platform/compute/arrow_formatter.d.ts +3 -1
- package/dist/platform/compute/arrow_formatter.js +23 -20
- package/dist/platform/compute/computation_types.d.ts +7 -12
- package/dist/platform/compute/computation_types.js +6 -3
- package/dist/platform/compute/query_result_search_sql.d.ts +0 -7
- package/dist/platform/compute/query_result_search_sql.js +1 -19
- package/dist/platform/connections/connection_params.d.ts +5 -6
- package/dist/platform/connections/connection_params.js +14 -25
- package/dist/platform/connections/connection_state.d.ts +1 -1
- package/dist/platform/connections/connector_configs.d.ts +2 -3
- package/dist/platform/connections/connector_info.d.ts +4 -14
- package/dist/platform/connections/connector_info.js +1 -25
- package/dist/platform/connections/embedded_database_channel.d.ts +3 -3
- package/dist/platform/connections/embedded_database_channel.js +12 -6
- package/dist/platform/connections/hyper/hyper_error_info.d.ts +0 -15
- package/dist/platform/connections/hyper/hyper_error_info.js +1 -18
- package/dist/platform/connections/query_execution.js +9 -7
- package/dist/platform/connections/query_execution_args.d.ts +2 -2
- package/dist/platform/http_client.d.ts +1 -1
- package/dist/platform/logger/logger.d.ts +3 -3
- package/dist/platform/query/query_execution_state.d.ts +11 -38
- package/dist/platform/query/query_execution_state.js +6 -1
- package/dist/platform/query/query_progress.d.ts +0 -14
- package/dist/platform/query/query_result_cache.d.ts +2 -2
- package/dist/platform/storage/composite_storage_backend.d.ts +64 -0
- package/dist/platform/storage/composite_storage_backend.js +321 -0
- package/dist/platform/storage/native_storage_backend.d.ts +1 -1
- package/dist/platform/storage/native_storage_backend.js +1 -1
- package/dist/platform/storage/opfs_query_result_cache.d.ts +0 -2
- package/dist/platform/storage/opfs_query_result_cache.js +3 -5
- package/dist/platform/storage/opfs_storage_backend.d.ts +67 -0
- package/dist/platform/storage/opfs_storage_backend.js +494 -0
- package/dist/platform/storage/workbook_locator.d.ts +7 -0
- package/dist/platform/storage/workbook_locator.js +14 -0
- package/dist/platform/storage/workbook_tables.d.ts +1 -4
- package/dist/platform/storage/workbook_tables.js +3 -17
- package/dist/platform/storage/workbook_transfer.d.ts +9 -0
- package/dist/platform/storage/workbook_transfer.js +84 -0
- package/dist/platform/storage/workbook_tree.d.ts +30 -9
- package/dist/platform/storage/workbook_tree.js +17 -96
- package/dist/platform/utils/base64.js +4 -4
- package/dist/platform/web_http_client.d.ts +1 -4
- package/dist/platform/web_http_client.js +0 -5
- package/dist/platform/workbook/attachment_mounts.d.ts +12 -4
- package/dist/platform/workbook/attachment_mounts.js +69 -60
- package/dist/platform/workbook/run_events.js +1 -1
- package/dist/platform/workbook/script_compilation.d.ts +0 -1
- package/dist/platform/workbook/script_compilation.js +1 -14
- package/dist/platform/workbook/script_reference_check.js +7 -6
- package/dist/platform/workbook/script_types.d.ts +1 -1
- package/dist/platform/workbook/script_types.js +1 -10
- package/dist/platform/workbook/workbook_execution.d.ts +0 -2
- package/dist/platform/workbook/workbook_execution.js +1 -4
- package/dist/platform/workbook/workbook_inputs.d.ts +7 -7
- package/dist/platform/workbook/workbook_inputs.js +4 -4
- package/dist/platform/workbook/workbook_scripts.d.ts +2 -2
- package/dist/runtime/compute_platform.js +2 -2
- package/dist/runtime/node/node_platform.js +1 -1
- package/dist/runtime/platform_requests.js +1 -1
- package/dist/runtime/query/query_channels.d.ts +0 -2
- package/dist/runtime/query/query_channels.js +36 -55
- package/dist/runtime/run_mounts.d.ts +2 -1
- package/dist/runtime/run_mounts.js +4 -5
- package/dist/runtime/runtime.d.ts +3 -0
- package/dist/runtime/runtime.js +10 -6
- package/dist/runtime/runtime_storage.d.ts +2 -5
- package/dist/runtime/runtime_storage.js +4 -13
- package/dist/runtime/workbook_projection.js +2 -2
- package/dist/runtime/workbook_run_report.d.ts +3 -6
- package/dist/runtime/workbook_run_report.js +33 -47
- package/dist/wasm/dashql_core.d.ts +2363 -2090
- package/dist/wasm/dashql_core.js +362 -222
- package/dist/wasm/dashql_core.wasm +0 -0
- package/dist/wasm/dashql_dataflow.wasm +0 -0
- package/package.json +1 -1
- package/dist/core/format.d.ts +0 -4
- package/dist/core/format.js +0 -23
- package/dist/core/lookup.d.ts +0 -10
- package/dist/core/lookup.js +0 -129
- package/dist/core/tokens.d.ts +0 -9
- package/dist/core/tokens.js +0 -78
- package/dist/core/variant.d.ts +0 -4
- package/dist/core/variant.js +0 -0
- package/dist/platform/connections/hyper/local_cloud.d.ts +0 -15
- package/dist/platform/connections/hyper/local_cloud.js +0 -14
- package/dist/platform/connections/query_channel.d.ts +0 -3
- package/dist/platform/connections/query_channel.js +0 -11
- package/dist/platform/hyper_link_target.d.ts +0 -8
- package/dist/platform/hyper_link_target.js +0 -35
- package/dist/platform/sql/sql_format.d.ts +0 -9
- package/dist/platform/sql/sql_format.js +0 -40
- package/dist/platform/sql/sql_lint.d.ts +0 -27
- package/dist/platform/sql/sql_lint.js +0 -69
- package/dist/platform/sql/sql_report.d.ts +0 -18
- package/dist/platform/sql/sql_report.js +0 -80
- package/dist/platform/storage/storage_versions.d.ts +0 -3
- package/dist/platform/storage/storage_versions.js +0 -13
- package/dist/platform/utils/async_consumer.d.ts +0 -14
- package/dist/platform/utils/async_consumer.js +0 -24
- package/dist/platform/workbook/attachment_mirror.d.ts +0 -2
- package/dist/platform/workbook/attachment_mirror.js +0 -25
|
@@ -55,7 +55,7 @@ var PlatformRequests = class {
|
|
|
55
55
|
};
|
|
56
56
|
} catch (error) {
|
|
57
57
|
const cause = error instanceof Error ? error.cause : void 0;
|
|
58
|
-
if (error instanceof Error && cause instanceof Error) throw new Error(`${error.message}: ${cause.message}
|
|
58
|
+
if (error instanceof Error && cause instanceof Error) throw new Error(`${error.message}: ${cause.message}`, { cause: error });
|
|
59
59
|
throw error;
|
|
60
60
|
} finally {
|
|
61
61
|
this.fetches.delete(id);
|
|
@@ -27,8 +27,6 @@ export declare class QueryChannels {
|
|
|
27
27
|
execute(runtime: core.Runtime, connectionId: string, args: QueryExecutionArgs): [number, Promise<arrow.Table | null>];
|
|
28
28
|
private errorOf;
|
|
29
29
|
private recordOf;
|
|
30
|
-
private pushMessage;
|
|
31
|
-
private produce;
|
|
32
30
|
private nextMessage;
|
|
33
31
|
}
|
|
34
32
|
export {};
|
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { salesforce_hyperdb_grpc_v1 } from "../../platform/proto.js";
|
|
2
2
|
import { QueryCacheMissError } from "../../platform/query/query_result_cache.js";
|
|
3
3
|
import { createTrace } from "../../platform/logger/trace_context.js";
|
|
4
4
|
import { LoggableException, stringifyError } from "../../platform/logger/logger.js";
|
|
5
5
|
import { supportsParquet } from "../../platform/hyperdb/hyperdb_parquet_export.js";
|
|
6
6
|
import * as arrow from "apache-arrow";
|
|
7
|
+
import * as buf from "@bufbuild/protobuf";
|
|
7
8
|
// dist/.stage/src/runtime/query/query_channels.ts
|
|
8
9
|
var LOG_CTX = "query_executor";
|
|
9
10
|
function newRecord() {
|
|
@@ -13,10 +14,9 @@ function newRecord() {
|
|
|
13
14
|
cancellation: new AbortController(),
|
|
14
15
|
stream: null,
|
|
15
16
|
streamCancel: null,
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
wakeMessages: null
|
|
17
|
+
items: null,
|
|
18
|
+
batches: 0,
|
|
19
|
+
table: null
|
|
20
20
|
};
|
|
21
21
|
}
|
|
22
22
|
function abortError() {
|
|
@@ -43,13 +43,18 @@ var QueryChannels = class {
|
|
|
43
43
|
return { type: "host" };
|
|
44
44
|
},
|
|
45
45
|
forget: (queryId) => {
|
|
46
|
+
const record = this.records.get(queryId);
|
|
46
47
|
this.records.delete(queryId);
|
|
48
|
+
record?.items?.return?.()?.catch(() => {});
|
|
47
49
|
},
|
|
48
50
|
open: async (queryId, table) => {
|
|
49
51
|
const record = this.recordOf(queryId);
|
|
50
52
|
const query = record.query;
|
|
51
53
|
if (query == null) throw new Error(`The query ${queryId} was not described`);
|
|
52
|
-
const
|
|
54
|
+
const channel = record.channel;
|
|
55
|
+
if (channel == null) throw new Error(`${query.connector} channel is not set up`);
|
|
56
|
+
const param = buf.create(salesforce_hyperdb_grpc_v1.pb.QueryParamSchema, { query: query.text });
|
|
57
|
+
const stream = await channel.executeQuery(param, record.cancellation.signal, { readTimeoutMs: query.readTimeoutMs ?? void 0 });
|
|
53
58
|
if (stream == null) return false;
|
|
54
59
|
record.stream = stream;
|
|
55
60
|
record.table = table;
|
|
@@ -193,58 +198,34 @@ var QueryChannels = class {
|
|
|
193
198
|
if (record == null) throw new Error(`Unknown query ${queryId}`);
|
|
194
199
|
return record;
|
|
195
200
|
}
|
|
196
|
-
|
|
197
|
-
record.
|
|
198
|
-
|
|
199
|
-
record.wakeMessages = null;
|
|
200
|
-
wake?.();
|
|
201
|
-
}
|
|
202
|
-
produce(record, stream, table) {
|
|
201
|
+
async nextMessage(record) {
|
|
202
|
+
const stream = record.stream;
|
|
203
|
+
if (stream == null || record.table == null) throw new Error("The query has no stream");
|
|
203
204
|
const publish = record.query?.publishResults !== false;
|
|
204
|
-
const result = this.deps.tables.stream(table);
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
type: "batch",
|
|
205
|
+
const result = this.deps.tables.stream(record.table);
|
|
206
|
+
record.items ??= stream.read(record.cancellation.signal)[Symbol.asyncIterator]();
|
|
207
|
+
for (;;) {
|
|
208
|
+
const next = await record.items.next();
|
|
209
|
+
if (next.done) {
|
|
210
|
+
if (publish && record.batches === 0) result?.setSchema(await stream.getSchema() ?? new arrow.Schema());
|
|
211
|
+
return {
|
|
212
|
+
type: "end",
|
|
213
213
|
metrics: stream.getMetrics()
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
};
|
|
227
|
-
record.producing = true;
|
|
228
|
-
Promise.resolve().then(() => stream.produce(batchConsumer, progressConsumer, record.cancellation.signal)).then(async () => {
|
|
229
|
-
if (publish && batches === 0) result?.setSchema(await stream.getSchema() ?? new arrow.Schema());
|
|
230
|
-
this.pushMessage(record, {
|
|
231
|
-
type: "end",
|
|
214
|
+
};
|
|
215
|
+
}
|
|
216
|
+
const item = next.value;
|
|
217
|
+
if (item.type === "progress") return {
|
|
218
|
+
type: "progress",
|
|
219
|
+
progress: item.progress
|
|
220
|
+
};
|
|
221
|
+
if (!publish) continue;
|
|
222
|
+
++record.batches;
|
|
223
|
+
result?.pushBatch(item.batch);
|
|
224
|
+
return {
|
|
225
|
+
type: "batch",
|
|
232
226
|
metrics: stream.getMetrics()
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
type: "error",
|
|
236
|
-
error
|
|
237
|
-
}));
|
|
238
|
-
}
|
|
239
|
-
async nextMessage(record) {
|
|
240
|
-
if (record.stream == null) throw new Error("The query has no stream");
|
|
241
|
-
if (!record.producing && record.table != null) this.produce(record, record.stream, record.table);
|
|
242
|
-
while (record.messages.length === 0) await new Promise((resolve) => {
|
|
243
|
-
record.wakeMessages = resolve;
|
|
244
|
-
});
|
|
245
|
-
const message = record.messages.shift();
|
|
246
|
-
if (message.type === "error") throw message.error;
|
|
247
|
-
return message;
|
|
227
|
+
};
|
|
228
|
+
}
|
|
248
229
|
}
|
|
249
230
|
};
|
|
250
231
|
export { QueryChannels };
|
|
@@ -3,4 +3,5 @@ import type { HandleTable } from './store/handles.js';
|
|
|
3
3
|
import type { CachePins } from './query/cache_pins.js';
|
|
4
4
|
import type { EmbeddedComputeDatabase } from '../platform/embedded_database.js';
|
|
5
5
|
import type { QueryResultCache } from '../platform/query/query_result_cache.js';
|
|
6
|
-
|
|
6
|
+
import type { AttachmentMounts } from '../platform/workbook/attachment_mounts.js';
|
|
7
|
+
export declare function createRunMounts(computeDatabase: () => Promise<EmbeddedComputeDatabase>, cache: QueryResultCache | null, handles: HandleTable, pins: CachePins, attachments: AttachmentMounts): core.RuntimeMounts;
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { supportsParquet } from "../platform/hyperdb/hyperdb_parquet_export.js";
|
|
2
|
-
import { mountWorkbookAttachments, readAttachmentColumns, unmountWorkbookAttachments } from "../platform/workbook/attachment_mounts.js";
|
|
3
2
|
// dist/.stage/src/runtime/run_mounts.ts
|
|
4
3
|
function memoryFiles(handles, files) {
|
|
5
4
|
if (files == null) return null;
|
|
@@ -11,7 +10,7 @@ function memoryFiles(handles, files) {
|
|
|
11
10
|
}
|
|
12
11
|
return blobs;
|
|
13
12
|
}
|
|
14
|
-
function createRunMounts(computeDatabase, cache, handles, pins) {
|
|
13
|
+
function createRunMounts(computeDatabase, cache, handles, pins, attachments) {
|
|
15
14
|
const database = async () => {
|
|
16
15
|
const database = await computeDatabase();
|
|
17
16
|
if (!supportsParquet(database)) throw new Error("Scripts that read the results of other scripts or attachments need a compute database that reads Parquet files");
|
|
@@ -45,10 +44,10 @@ function createRunMounts(computeDatabase, cache, handles, pins) {
|
|
|
45
44
|
return table.numRows;
|
|
46
45
|
},
|
|
47
46
|
attachments: async (workbookId, files) => {
|
|
48
|
-
await
|
|
47
|
+
await attachments.mount(await database(), workbookId, memoryFiles(handles, files));
|
|
49
48
|
},
|
|
50
|
-
attachmentColumns: async (workbookId, fileName, files) =>
|
|
51
|
-
dropAttachments: (workbookId) =>
|
|
49
|
+
attachmentColumns: async (workbookId, fileName, files) => attachments.readColumns(await database(), workbookId, fileName, memoryFiles(handles, files)),
|
|
50
|
+
dropAttachments: (workbookId) => attachments.unmount(workbookId)
|
|
52
51
|
};
|
|
53
52
|
}
|
|
54
53
|
export { createRunMounts };
|
|
@@ -16,6 +16,7 @@ import type { OAuthFlow } from '../platform/oauth_flow.js';
|
|
|
16
16
|
import { StorageCommands } from './runtime_storage.js';
|
|
17
17
|
import type { StorageBackend } from '../platform/storage/storage_backend.js';
|
|
18
18
|
import type { ScriptData, WorkbookScripts } from '../platform/workbook/workbook_scripts.js';
|
|
19
|
+
import { AttachmentMounts } from '../platform/workbook/attachment_mounts.js';
|
|
19
20
|
import { type ScriptRunEvent } from '../platform/workbook/workbook_execution.js';
|
|
20
21
|
import { type WorkbookTables } from '../platform/storage/workbook_tables.js';
|
|
21
22
|
export type { ComputeDatabaseProvider };
|
|
@@ -94,6 +95,7 @@ export declare class Runtime {
|
|
|
94
95
|
private readonly tables;
|
|
95
96
|
private readonly queries;
|
|
96
97
|
private readonly pins;
|
|
98
|
+
readonly attachmentMounts: AttachmentMounts;
|
|
97
99
|
private readonly catalogs;
|
|
98
100
|
private readonly requests;
|
|
99
101
|
private readonly scriptProjection;
|
|
@@ -113,6 +115,7 @@ export declare class Runtime {
|
|
|
113
115
|
connection(connectionId: string): ConnectionState | null;
|
|
114
116
|
allocateId(): number;
|
|
115
117
|
catalogRelationsFile(connectionId: string): string | null;
|
|
118
|
+
bindHyperLink(endpoint: string, token: string, recentConnectionId: string | null): core.HyperLinkView | null;
|
|
116
119
|
revealConnectionSecrets(connectionId: string): core.ConnectionSecrets | null;
|
|
117
120
|
exportConnection(connectionId: string, options: core.ConnectionLinkOptions): Promise<string>;
|
|
118
121
|
catalog(connectionId: string): core.DashQLCatalog | null;
|
package/dist/runtime/runtime.js
CHANGED
|
@@ -18,10 +18,10 @@ import { PlatformRequests } from "./platform_requests.js";
|
|
|
18
18
|
import { StorageCommands, createRuntimeStorage } from "./runtime_storage.js";
|
|
19
19
|
import { ScriptProjection, WORKBOOK_PROJECTION, WorkbookDerivedProjection } from "./workbook_projection.js";
|
|
20
20
|
import { createRunMounts } from "./run_mounts.js";
|
|
21
|
+
import { AttachmentMounts } from "../platform/workbook/attachment_mounts.js";
|
|
21
22
|
import { CachePins } from "./query/cache_pins.js";
|
|
22
23
|
import { scriptRunEvent, workbookRunError } from "../platform/workbook/workbook_execution.js";
|
|
23
|
-
import { WORKBOOK_TABLE_NAMES, emptyWorkbookTables
|
|
24
|
-
import { provideStorageVersions } from "../platform/storage/storage_versions.js";
|
|
24
|
+
import { WORKBOOK_TABLE_NAMES, emptyWorkbookTables } from "../platform/storage/workbook_tables.js";
|
|
25
25
|
import * as flatbuffers from "flatbuffers";
|
|
26
26
|
// dist/.stage/src/runtime/runtime.ts
|
|
27
27
|
var LOG_CTX = "runtime";
|
|
@@ -46,6 +46,7 @@ var Runtime = class {
|
|
|
46
46
|
tables;
|
|
47
47
|
queries;
|
|
48
48
|
pins = new CachePins();
|
|
49
|
+
attachmentMounts = new AttachmentMounts();
|
|
49
50
|
catalogs = /* @__PURE__ */ new Map();
|
|
50
51
|
requests;
|
|
51
52
|
scriptProjection;
|
|
@@ -70,8 +71,6 @@ var Runtime = class {
|
|
|
70
71
|
resultCache: options.resultCache ?? null,
|
|
71
72
|
pins: this.pins
|
|
72
73
|
});
|
|
73
|
-
provideWorkbookTables(this.core);
|
|
74
|
-
provideStorageVersions(this.core);
|
|
75
74
|
this.module = this.core.createRuntime(this.modulePlatform(database));
|
|
76
75
|
this.module.setProgressLayout(createProgressLayoutConfig());
|
|
77
76
|
this.module.runCatalogLoader(options.catalogLoader !== false);
|
|
@@ -113,14 +112,14 @@ var Runtime = class {
|
|
|
113
112
|
hyperLinkToken: (connectionId) => platform.channels?.hyperLinkToken?.(connectionId) ?? null
|
|
114
113
|
},
|
|
115
114
|
resultCache: this.queries.resultCache,
|
|
116
|
-
mounts: database == null ? null : createRunMounts(database, platform.resultCache ?? null, this.handles, this.pins),
|
|
115
|
+
mounts: database == null ? null : createRunMounts(database, platform.resultCache ?? null, this.handles, this.pins, this.attachmentMounts),
|
|
117
116
|
crypto: {
|
|
118
117
|
randomUuid: () => crypto.randomUUID(),
|
|
119
118
|
sha256: async (bytes) => new Uint8Array(await crypto.subtle.digest("SHA-256", bytes))
|
|
120
119
|
},
|
|
121
120
|
oauth: this.requests.oauth(),
|
|
122
121
|
extensions: { loadSpec: async (extension) => JSON.stringify(await loadClientFunctionSpec(extension)) },
|
|
123
|
-
storage: platform.storage == null ? null : createRuntimeStorage(platform.storage),
|
|
122
|
+
storage: platform.storage == null ? null : createRuntimeStorage(platform.storage, this.attachmentMounts),
|
|
124
123
|
archives: platform.archives?.(this.handles) ?? null,
|
|
125
124
|
changed: (urgent) => this.store.changed(urgent),
|
|
126
125
|
settings: platform.settings ?? void 0
|
|
@@ -188,6 +187,10 @@ var Runtime = class {
|
|
|
188
187
|
if (this.stopping) return null;
|
|
189
188
|
return this.module.catalogRelationsFile(connectionId) ?? null;
|
|
190
189
|
}
|
|
190
|
+
bindHyperLink(endpoint, token, recentConnectionId) {
|
|
191
|
+
if (this.stopping) return null;
|
|
192
|
+
return this.module.bindHyperLink(endpoint, token, recentConnectionId ?? void 0);
|
|
193
|
+
}
|
|
191
194
|
revealConnectionSecrets(connectionId) {
|
|
192
195
|
if (this.stopping) return null;
|
|
193
196
|
return this.module.revealConnectionSecrets(connectionId);
|
|
@@ -449,6 +452,7 @@ var Runtime = class {
|
|
|
449
452
|
this.store.dispose();
|
|
450
453
|
this.requests.abortAll();
|
|
451
454
|
this.queries.dispose();
|
|
455
|
+
await this.attachmentMounts.unmountAll();
|
|
452
456
|
await this.pins.close();
|
|
453
457
|
}
|
|
454
458
|
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type * as core from '../core/index.js';
|
|
2
2
|
import type { StorageBackend } from '../platform/storage/storage_backend.js';
|
|
3
3
|
import { NativeStorageBackend } from '../platform/storage/native_storage_backend.js';
|
|
4
|
+
import type { AttachmentMounts } from '../platform/workbook/attachment_mounts.js';
|
|
4
5
|
export type StorageWriteResult = {
|
|
5
6
|
status: 'written';
|
|
6
7
|
} | {
|
|
@@ -13,16 +14,12 @@ export type StorageWriteResult = {
|
|
|
13
14
|
status: 'failed';
|
|
14
15
|
error: unknown;
|
|
15
16
|
};
|
|
16
|
-
export declare class WorkbookUnavailableError extends Error {
|
|
17
|
-
readonly workbookId: string;
|
|
18
|
-
constructor(workbookId: string, cause: unknown);
|
|
19
|
-
}
|
|
20
17
|
export declare function writeResultOf(outcome: core.StorageWriteOutcome): StorageWriteResult;
|
|
21
18
|
export interface NativeWorkbookRegistry {
|
|
22
19
|
registerNativeWorkbook(dir: string, workbookId: string): Promise<void>;
|
|
23
20
|
nativeWorkbookFolder(workbookId: string): NativeStorageBackend | null;
|
|
24
21
|
}
|
|
25
|
-
export declare function createRuntimeStorage(backend: StorageBackend): core.RuntimeStorage;
|
|
22
|
+
export declare function createRuntimeStorage(backend: StorageBackend, attachments: AttachmentMounts): core.RuntimeStorage;
|
|
26
23
|
export declare class StorageCommands {
|
|
27
24
|
private readonly runtime;
|
|
28
25
|
constructor(runtime: {
|
|
@@ -1,24 +1,15 @@
|
|
|
1
1
|
import { stringifyError } from "../platform/logger/logger.js";
|
|
2
2
|
import { isWorkbookRegistryBackend } from "../platform/storage/storage_backend.js";
|
|
3
3
|
import { NativeStorageBackend } from "../platform/storage/native_storage_backend.js";
|
|
4
|
-
import { markWorkbookAttachmentsChanged } from "../platform/workbook/attachment_mounts.js";
|
|
5
4
|
// dist/.stage/src/runtime/runtime_storage.ts
|
|
6
|
-
var WorkbookUnavailableError = class extends Error {
|
|
7
|
-
workbookId;
|
|
8
|
-
constructor(workbookId, cause) {
|
|
9
|
-
super(`workbook ${workbookId} is not available: ${stringifyError(cause)}`);
|
|
10
|
-
this.name = "WorkbookUnavailableError";
|
|
11
|
-
this.workbookId = workbookId;
|
|
12
|
-
}
|
|
13
|
-
};
|
|
14
5
|
function writeResultOf(outcome) {
|
|
15
6
|
if (outcome.status !== "failed") return { status: outcome.status };
|
|
16
7
|
return {
|
|
17
8
|
status: "failed",
|
|
18
|
-
error: outcome.unavailableWorkbook != null ? new
|
|
9
|
+
error: outcome.unavailableWorkbook != null ? /* @__PURE__ */ new Error(`workbook ${outcome.unavailableWorkbook} is not available: ${stringifyError(outcome.error)}`) : outcome.error
|
|
19
10
|
};
|
|
20
11
|
}
|
|
21
|
-
function createRuntimeStorage(backend) {
|
|
12
|
+
function createRuntimeStorage(backend, attachments) {
|
|
22
13
|
const registry = () => {
|
|
23
14
|
if (!isWorkbookRegistryBackend(backend)) throw new Error("the storage backend does not own a workbook registry");
|
|
24
15
|
return backend;
|
|
@@ -55,7 +46,7 @@ function createRuntimeStorage(backend) {
|
|
|
55
46
|
await native.registerNativeWorkbook(dir, workbookId);
|
|
56
47
|
},
|
|
57
48
|
workbookFolder: (workbookId) => backend instanceof NativeStorageBackend ? backend : backend.nativeWorkbookFolder?.(workbookId) ?? null,
|
|
58
|
-
attachmentsChanged: (workbookId) =>
|
|
49
|
+
attachmentsChanged: (workbookId) => attachments.changed(workbookId)
|
|
59
50
|
};
|
|
60
51
|
}
|
|
61
52
|
var StorageCommands = class {
|
|
@@ -115,4 +106,4 @@ var StorageCommands = class {
|
|
|
115
106
|
return this.module?.storageStatistics(workbookId) ?? [];
|
|
116
107
|
}
|
|
117
108
|
};
|
|
118
|
-
export { StorageCommands,
|
|
109
|
+
export { StorageCommands, createRuntimeStorage, writeResultOf };
|
|
@@ -9,7 +9,7 @@ import { FlatBufferCache, readFlatBuffer } from "../core/flatbuffer_view.js";
|
|
|
9
9
|
import { scriptDiffBuffer } from "../core/workbook.js";
|
|
10
10
|
import "../core/index.js";
|
|
11
11
|
import { share } from "./store/projection.js";
|
|
12
|
-
import {
|
|
12
|
+
import { readWorkbookInputs } from "../platform/workbook/workbook_inputs.js";
|
|
13
13
|
import { resolveCompiledVisualization } from "../platform/workbook/script_compilation.js";
|
|
14
14
|
import { readScriptReferenceReports } from "../platform/workbook/script_reference_check.js";
|
|
15
15
|
import { createScriptRef } from "../platform/workbook/script_types.js";
|
|
@@ -161,7 +161,7 @@ var WorkbookDerivedProjection = class {
|
|
|
161
161
|
const [referencesHash, dataflowHash, inputsHash] = view.hashes;
|
|
162
162
|
const references = prev != null && before != null && sameScripts && before.names === names && references0 === referencesHash ? prev.references : readScriptReferenceReports(readFlatBuffer(view.references, (bb) => ScriptReferenceCheck.getRootAsScriptReferenceCheck(bb)).unpack(), view.scriptKeys, files, attachments, prev?.references ?? null);
|
|
163
163
|
const dataflow = prev != null && before != null && dataflow0 === dataflowHash ? prev.dataflow : readFlatBuffer(view.dataflow, (bb) => SessionDataflow.getRootAsSessionDataflow(bb)).unpack();
|
|
164
|
-
const inputs = prev != null && before != null && sameScripts && inputs0 === inputsHash ? prev.inputs : share(prev?.inputs,
|
|
164
|
+
const inputs = prev != null && before != null && sameScripts && inputs0 === inputsHash ? prev.inputs : share(prev?.inputs, readWorkbookInputs(readFlatBuffer(view.inputs, (bb) => SessionInputs.getRootAsSessionInputs(bb)).unpack(), view.scriptKeys));
|
|
165
165
|
if (prev != null && prev.scriptKeys === scriptKeys && prev.references === references && prev.dataflow === dataflow && prev.inputs === inputs) return prev;
|
|
166
166
|
return {
|
|
167
167
|
workbookId,
|
|
@@ -1,13 +1,10 @@
|
|
|
1
|
+
import type * as core from '../core/index.js';
|
|
1
2
|
import type { ConnectionState } from '../platform/connections/connection_state.js';
|
|
2
3
|
import type { RunEvent } from '../platform/workbook/run_events.js';
|
|
3
4
|
import { type WorkbookScripts } from '../platform/workbook/workbook_scripts.js';
|
|
4
5
|
import type { Runtime } from './runtime.js';
|
|
5
6
|
export declare function describeRunError(error: unknown): string;
|
|
6
|
-
export declare function runScriptsOf(workbook: WorkbookScripts, names: readonly string[]):
|
|
7
|
-
selected: string[];
|
|
8
|
-
requested: boolean;
|
|
9
|
-
unknown: string[];
|
|
10
|
-
};
|
|
7
|
+
export declare function runScriptsOf(instance: core.DashQL, workbook: WorkbookScripts, names: readonly string[]): core.RunScriptSelection;
|
|
11
8
|
export interface ReportedRunOptions {
|
|
12
9
|
label: string;
|
|
13
10
|
scripts: readonly string[];
|
|
@@ -18,5 +15,5 @@ export interface ReportedRunOptions {
|
|
|
18
15
|
resultFile?: (key: string) => Promise<string | null>;
|
|
19
16
|
onReportError?: (error: unknown) => void;
|
|
20
17
|
}
|
|
21
|
-
export type ReportedRunEnd = '
|
|
18
|
+
export type ReportedRunEnd = core.RunReportEnd['end'];
|
|
22
19
|
export declare function runWorkbookReported(runtime: Runtime, workbook: WorkbookScripts, options: ReportedRunOptions, emit: (event: RunEvent) => void): Promise<ReportedRunEnd>;
|
|
@@ -16,27 +16,13 @@ function describeRunError(error) {
|
|
|
16
16
|
if (!(error instanceof Error)) return String(error);
|
|
17
17
|
return error.cause instanceof Error ? `${error.message}: ${error.cause.message}` : error.message;
|
|
18
18
|
}
|
|
19
|
-
function
|
|
20
|
-
|
|
21
|
-
if ("trino" in params) return `trino ${params.trino.endpoint}`;
|
|
22
|
-
if ("hyper" in params) return params.hyper.protocol === "WASM" ? "the embedded hyper" : `hyper ${params.hyper.endpoint ?? ""}`.trim();
|
|
23
|
-
if ("salesforce" in params) return "salesforce";
|
|
24
|
-
return "dbcom";
|
|
25
|
-
}
|
|
26
|
-
function runScriptsOf(workbook, names) {
|
|
27
|
-
const files = [...workbook.scriptOrder];
|
|
28
|
-
const requested = new Set(names.map((name) => name.split(/[\\/]/).pop() ?? name));
|
|
29
|
-
const unknown = [...requested].filter((file) => !files.includes(file));
|
|
30
|
-
return {
|
|
31
|
-
selected: requested.size === 0 ? files : files.filter((file) => requested.has(file)),
|
|
32
|
-
requested: requested.size > 0,
|
|
33
|
-
unknown
|
|
34
|
-
};
|
|
19
|
+
function runScriptsOf(instance, workbook, names) {
|
|
20
|
+
return instance.runScripts([...workbook.scriptOrder], [...names]);
|
|
35
21
|
}
|
|
36
22
|
async function runWorkbookReported(runtime, workbook, options, emit) {
|
|
37
23
|
const { workbookId } = workbook;
|
|
38
24
|
const files = [...workbook.scriptOrder];
|
|
39
|
-
const { selected, requested } = runScriptsOf(workbook, options.scripts);
|
|
25
|
+
const { selected, requested } = runScriptsOf(runtime.core, workbook, options.scripts);
|
|
40
26
|
const scriptOfOwner = new Map(files.map((file) => [scriptResultOwner(workbookId, workbook.scriptRefs[file].scriptId), file]));
|
|
41
27
|
const { signal } = options;
|
|
42
28
|
const runStarted = performance.now();
|
|
@@ -49,7 +35,7 @@ async function runWorkbookReported(runtime, workbook, options, emit) {
|
|
|
49
35
|
const prepareConnection = prepare == null ? void 0 : async (connectionId) => {
|
|
50
36
|
const conn = runtime.connection(connectionId);
|
|
51
37
|
if (conn == null) throw new Error(`the workbook has no connection ${connectionId}`);
|
|
52
|
-
const label = runConnectionLabel(conn);
|
|
38
|
+
const label = runtime.core.runConnectionLabel(conn.params);
|
|
53
39
|
const started = performance.now();
|
|
54
40
|
emit({
|
|
55
41
|
type: "connection_setup",
|
|
@@ -112,7 +98,11 @@ async function runWorkbookReported(runtime, workbook, options, emit) {
|
|
|
112
98
|
};
|
|
113
99
|
}
|
|
114
100
|
try {
|
|
115
|
-
const outcomes =
|
|
101
|
+
const outcomes = [];
|
|
102
|
+
const settle = (script, outcome) => outcomes.push({
|
|
103
|
+
script,
|
|
104
|
+
outcome
|
|
105
|
+
});
|
|
116
106
|
const startedAt = /* @__PURE__ */ new Map();
|
|
117
107
|
const elapsed = (script) => performance.now() - (startedAt.get(script) ?? runStarted);
|
|
118
108
|
let reports = Promise.resolve();
|
|
@@ -155,13 +145,13 @@ async function runWorkbookReported(runtime, workbook, options, emit) {
|
|
|
155
145
|
break;
|
|
156
146
|
case "cached":
|
|
157
147
|
case "done": {
|
|
158
|
-
|
|
148
|
+
settle(script, "ok");
|
|
159
149
|
const cached = event.type === "cached";
|
|
160
150
|
report(() => resultOf(script, event.queryId, event.key, cached));
|
|
161
151
|
break;
|
|
162
152
|
}
|
|
163
153
|
case "failed": {
|
|
164
|
-
|
|
154
|
+
settle(script, "failed");
|
|
165
155
|
const ms = elapsed(script);
|
|
166
156
|
const statement = event.queryId == null ? null : queryOf(event.queryId)?.queryText ?? null;
|
|
167
157
|
report(async () => ({
|
|
@@ -174,7 +164,7 @@ async function runWorkbookReported(runtime, workbook, options, emit) {
|
|
|
174
164
|
break;
|
|
175
165
|
}
|
|
176
166
|
case "cancelled": {
|
|
177
|
-
|
|
167
|
+
settle(script, "cancelled");
|
|
178
168
|
const ms = elapsed(script);
|
|
179
169
|
report(async () => ({
|
|
180
170
|
type: "script_cancelled",
|
|
@@ -184,7 +174,7 @@ async function runWorkbookReported(runtime, workbook, options, emit) {
|
|
|
184
174
|
break;
|
|
185
175
|
}
|
|
186
176
|
case "skipped":
|
|
187
|
-
|
|
177
|
+
settle(script, "notRun");
|
|
188
178
|
report(async () => ({
|
|
189
179
|
type: "script_not_run",
|
|
190
180
|
script,
|
|
@@ -210,37 +200,33 @@ async function runWorkbookReported(runtime, workbook, options, emit) {
|
|
|
210
200
|
await reports;
|
|
211
201
|
stopProgress?.();
|
|
212
202
|
const interrupted = signal.aborted;
|
|
213
|
-
const
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
203
|
+
const failure = run?.outcome.failure;
|
|
204
|
+
const end = runtime.core.finishRunReport({
|
|
205
|
+
selected,
|
|
206
|
+
outcomes,
|
|
207
|
+
interrupted,
|
|
208
|
+
refused: run == null || run.outcome.refused != null,
|
|
209
|
+
failure: failure != null && failure.kind !== "cancelled" ? failure.message : void 0
|
|
210
|
+
});
|
|
211
|
+
if (end.runError != null) emit({
|
|
220
212
|
type: "run_failed",
|
|
221
|
-
error
|
|
213
|
+
error: end.runError
|
|
214
|
+
});
|
|
215
|
+
for (const { script, reason } of end.notRun) emit({
|
|
216
|
+
type: "script_not_run",
|
|
217
|
+
script,
|
|
218
|
+
reason
|
|
222
219
|
});
|
|
223
|
-
for (const script of selected) {
|
|
224
|
-
if (outcomes.has(script)) continue;
|
|
225
|
-
outcomes.set(script, "not run");
|
|
226
|
-
emit({
|
|
227
|
-
type: "script_not_run",
|
|
228
|
-
script,
|
|
229
|
-
reason: interrupted ? "interrupted" : scriptFailed ? "the run stopped" : "the run failed"
|
|
230
|
-
});
|
|
231
|
-
}
|
|
232
|
-
const tally = (outcome) => [...outcomes.entries()].filter(([script, value]) => value === outcome && (outcome !== "ok" || selected.includes(script))).length;
|
|
233
220
|
emit({
|
|
234
221
|
type: "run_finished",
|
|
235
222
|
ms: performance.now() - runStarted,
|
|
236
223
|
interrupted,
|
|
237
|
-
ok:
|
|
238
|
-
failed:
|
|
239
|
-
cancelled:
|
|
240
|
-
notRun:
|
|
224
|
+
ok: end.ok,
|
|
225
|
+
failed: end.failed,
|
|
226
|
+
cancelled: end.cancelled,
|
|
227
|
+
notRun: end.notRunCount
|
|
241
228
|
});
|
|
242
|
-
|
|
243
|
-
return error == null && selected.every((script) => outcomes.get(script) === "ok") && tally("failed") + tally("cancelled") + tally("not run") === 0 ? "ok" : "failed";
|
|
229
|
+
return end.end;
|
|
244
230
|
} finally {
|
|
245
231
|
stopProgress?.();
|
|
246
232
|
}
|