@cyanheads/mcp-ts-core 0.13.8 → 0.13.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +17 -13
- package/CLAUDE.md +17 -13
- package/README.md +2 -2
- package/changelog/0.13.x/0.13.9.md +113 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +20 -9
- package/dist/config/index.js.map +1 -1
- package/dist/core/app.d.ts +6 -3
- package/dist/core/app.d.ts.map +1 -1
- package/dist/core/app.js +6 -4
- package/dist/core/app.js.map +1 -1
- package/dist/core/context.d.ts +25 -1
- package/dist/core/context.d.ts.map +1 -1
- package/dist/core/context.js.map +1 -1
- package/dist/core/serverManifest.d.ts +6 -0
- package/dist/core/serverManifest.d.ts.map +1 -1
- package/dist/core/serverManifest.js +6 -0
- package/dist/core/serverManifest.js.map +1 -1
- package/dist/linter/rules/tool-rules.d.ts +2 -1
- package/dist/linter/rules/tool-rules.d.ts.map +1 -1
- package/dist/linter/rules/tool-rules.js +36 -1
- package/dist/linter/rules/tool-rules.js.map +1 -1
- package/dist/mcp-server/inputRequired.d.ts +14 -5
- package/dist/mcp-server/inputRequired.d.ts.map +1 -1
- package/dist/mcp-server/inputRequired.js +15 -8
- package/dist/mcp-server/inputRequired.js.map +1 -1
- package/dist/mcp-server/tools/utils/inputPrevalidation.d.ts +163 -40
- package/dist/mcp-server/tools/utils/inputPrevalidation.d.ts.map +1 -1
- package/dist/mcp-server/tools/utils/inputPrevalidation.js +330 -114
- package/dist/mcp-server/tools/utils/inputPrevalidation.js.map +1 -1
- package/dist/mcp-server/tools/utils/toolHandlerFactory.d.ts +23 -10
- package/dist/mcp-server/tools/utils/toolHandlerFactory.d.ts.map +1 -1
- package/dist/mcp-server/tools/utils/toolHandlerFactory.js +296 -80
- package/dist/mcp-server/tools/utils/toolHandlerFactory.js.map +1 -1
- package/dist/mcp-server/transports/http/httpTransport.d.ts.map +1 -1
- package/dist/mcp-server/transports/http/httpTransport.js +65 -9
- package/dist/mcp-server/transports/http/httpTransport.js.map +1 -1
- package/dist/mcp-server/transports/stdio/stdioTransport.d.ts +9 -5
- package/dist/mcp-server/transports/stdio/stdioTransport.d.ts.map +1 -1
- package/dist/mcp-server/transports/stdio/stdioTransport.js +9 -5
- package/dist/mcp-server/transports/stdio/stdioTransport.js.map +1 -1
- package/dist/services/canvas/core/CanvasRegistry.d.ts +6 -2
- package/dist/services/canvas/core/CanvasRegistry.d.ts.map +1 -1
- package/dist/services/canvas/core/CanvasRegistry.js +7 -3
- package/dist/services/canvas/core/CanvasRegistry.js.map +1 -1
- package/dist/services/canvas/providers/duckdb/DuckdbProvider.d.ts +82 -18
- package/dist/services/canvas/providers/duckdb/DuckdbProvider.d.ts.map +1 -1
- package/dist/services/canvas/providers/duckdb/DuckdbProvider.js +620 -328
- package/dist/services/canvas/providers/duckdb/DuckdbProvider.js.map +1 -1
- package/dist/services/canvas/providers/duckdb/exportWriter.d.ts +11 -7
- package/dist/services/canvas/providers/duckdb/exportWriter.d.ts.map +1 -1
- package/dist/services/canvas/providers/duckdb/exportWriter.js +19 -16
- package/dist/services/canvas/providers/duckdb/exportWriter.js.map +1 -1
- package/dist/services/mirror/core/defineMirror.d.ts +1 -0
- package/dist/services/mirror/core/defineMirror.d.ts.map +1 -1
- package/dist/services/mirror/core/defineMirror.js +1 -0
- package/dist/services/mirror/core/defineMirror.js.map +1 -1
- package/dist/utils/index.d.ts +1 -1
- package/dist/utils/index.d.ts.map +1 -1
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/network/pacer.d.ts +38 -5
- package/dist/utils/network/pacer.d.ts.map +1 -1
- package/dist/utils/network/pacer.js +87 -25
- package/dist/utils/network/pacer.js.map +1 -1
- package/dist/utils/telemetry/attributes.d.ts +5 -1
- package/dist/utils/telemetry/attributes.d.ts.map +1 -1
- package/dist/utils/telemetry/attributes.js +5 -1
- package/dist/utils/telemetry/attributes.js.map +1 -1
- package/framework-skills/add-app-tool/SKILL.md +3 -3
- package/framework-skills/add-export/SKILL.md +5 -16
- package/framework-skills/add-prompt/SKILL.md +7 -3
- package/framework-skills/add-resource/SKILL.md +7 -5
- package/framework-skills/add-tool/SKILL.md +12 -10
- package/framework-skills/api-auth/SKILL.md +2 -2
- package/framework-skills/api-canvas/SKILL.md +17 -8
- package/framework-skills/api-config/SKILL.md +4 -4
- package/framework-skills/api-context/SKILL.md +14 -3
- package/framework-skills/api-errors/SKILL.md +8 -7
- package/framework-skills/api-linter/SKILL.md +26 -7
- package/framework-skills/api-mirror/SKILL.md +2 -1
- package/framework-skills/api-telemetry/SKILL.md +4 -4
- package/framework-skills/api-utils/SKILL.md +2 -2
- package/framework-skills/design-mcp-server/SKILL.md +2 -2
- package/framework-skills/field-test/SKILL.md +4 -4
- package/framework-skills/git-wrapup/SKILL.md +8 -6
- package/framework-skills/orchestrations/SKILL.md +7 -6
- package/framework-skills/orchestrations/workflows/field-test-fix.md +9 -19
- package/framework-skills/orchestrations/workflows/fix-wrapup-release.md +7 -7
- package/framework-skills/orchestrations/workflows/greenfield-build.md +8 -5
- package/framework-skills/orchestrations/workflows/maintenance-release.md +8 -8
- package/framework-skills/polish-docs-meta/SKILL.md +4 -4
- package/framework-skills/release-and-publish/SKILL.md +6 -4
- package/framework-skills/release-pr-review/SKILL.md +37 -23
- package/framework-skills/report-issue-framework/SKILL.md +7 -5
- package/framework-skills/report-issue-local/SKILL.md +8 -6
- package/framework-skills/security-pass/SKILL.md +8 -8
- package/package.json +3 -3
- package/scripts/devcheck.ts +7 -6
- package/scripts/lint-mcp.ts +87 -27
- package/scripts/lint-packaging.ts +61 -0
- package/scripts/release-github.ts +117 -5
- package/templates/_.mcpbignore +2 -0
|
@@ -6,10 +6,63 @@
|
|
|
6
6
|
* a true peer dependency.
|
|
7
7
|
* @module src/services/canvas/providers/duckdb/DuckdbProvider
|
|
8
8
|
*/
|
|
9
|
-
|
|
9
|
+
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
10
|
+
if (value !== null && value !== void 0) {
|
|
11
|
+
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
12
|
+
var dispose, inner;
|
|
13
|
+
if (async) {
|
|
14
|
+
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
15
|
+
dispose = value[Symbol.asyncDispose];
|
|
16
|
+
}
|
|
17
|
+
if (dispose === void 0) {
|
|
18
|
+
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
19
|
+
dispose = value[Symbol.dispose];
|
|
20
|
+
if (async) inner = dispose;
|
|
21
|
+
}
|
|
22
|
+
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
23
|
+
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
24
|
+
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
25
|
+
}
|
|
26
|
+
else if (async) {
|
|
27
|
+
env.stack.push({ async: true });
|
|
28
|
+
}
|
|
29
|
+
return value;
|
|
30
|
+
};
|
|
31
|
+
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
32
|
+
return function (env) {
|
|
33
|
+
function fail(e) {
|
|
34
|
+
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
35
|
+
env.hasError = true;
|
|
36
|
+
}
|
|
37
|
+
var r, s = 0;
|
|
38
|
+
function next() {
|
|
39
|
+
while (r = env.stack.pop()) {
|
|
40
|
+
try {
|
|
41
|
+
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
42
|
+
if (r.dispose) {
|
|
43
|
+
var result = r.dispose.call(r.value);
|
|
44
|
+
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
45
|
+
}
|
|
46
|
+
else s |= 1;
|
|
47
|
+
}
|
|
48
|
+
catch (e) {
|
|
49
|
+
fail(e);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
53
|
+
if (env.hasError) throw env.error;
|
|
54
|
+
}
|
|
55
|
+
return next();
|
|
56
|
+
};
|
|
57
|
+
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
58
|
+
var e = new Error(message);
|
|
59
|
+
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
60
|
+
});
|
|
61
|
+
import { randomUUID } from 'node:crypto';
|
|
62
|
+
import { mkdir, mkdtemp, rm, unlink } from 'node:fs/promises';
|
|
10
63
|
import { tmpdir } from 'node:os';
|
|
11
64
|
import { join, resolve } from 'node:path';
|
|
12
|
-
import { databaseError, McpError, notFound, timeout, validationError, } from '../../../../types-global/errors.js';
|
|
65
|
+
import { configurationError, databaseError, McpError, notFound, serviceUnavailable, timeout, validationError, } from '../../../../types-global/errors.js';
|
|
13
66
|
import { lazyImport } from '../../../../utils/internal/lazyImport.js';
|
|
14
67
|
import { logger } from '../../../../utils/internal/logger.js';
|
|
15
68
|
import { requestContextService, withExtra, } from '../../../../utils/internal/requestContext.js';
|
|
@@ -22,6 +75,10 @@ export class DuckdbProvider {
|
|
|
22
75
|
options;
|
|
23
76
|
name = 'duckdb';
|
|
24
77
|
canvases = new Map();
|
|
78
|
+
/** One pending promise per held canvas record, resolved on release — see {@link holdCanvas}. */
|
|
79
|
+
holds = new Set();
|
|
80
|
+
/** The private scratch directory once first requested — see {@link ensureTempRoot}. */
|
|
81
|
+
tempRoot;
|
|
25
82
|
constructor(options) {
|
|
26
83
|
this.options = options;
|
|
27
84
|
}
|
|
@@ -29,36 +86,83 @@ export class DuckdbProvider {
|
|
|
29
86
|
// Lifecycle
|
|
30
87
|
// ---------------------------------------------------------------------
|
|
31
88
|
/**
|
|
32
|
-
*
|
|
89
|
+
* The provider's private scratch directory, created once on first use.
|
|
33
90
|
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
38
|
-
*
|
|
91
|
+
* `mkdtemp` makes a directory no other local user can have pre-created,
|
|
92
|
+
* planted links in, or read (`0700` on POSIX; on Windows it inherits the
|
|
93
|
+
* parent's ACL, so there the parent must not grant other users access), so
|
|
94
|
+
* no scratch path resolves through a fixed shared name (#554). It also
|
|
95
|
+
* keeps scratch off the process cwd: DuckDB defaults an in-memory
|
|
96
|
+
* database's `temp_directory` to a cwd-relative `.tmp`, which fails on a
|
|
97
|
+
* non-root or read-only rootfs. Concurrent first calls share one directory;
|
|
98
|
+
* a failed attempt is dropped so the next call retries.
|
|
39
99
|
*/
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
100
|
+
ensureTempRoot() {
|
|
101
|
+
if (!this.tempRoot) {
|
|
102
|
+
const pending = this.createTempRoot();
|
|
103
|
+
this.tempRoot = pending;
|
|
104
|
+
pending.catch(() => {
|
|
105
|
+
if (this.tempRoot === pending)
|
|
106
|
+
this.tempRoot = undefined;
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
return this.tempRoot;
|
|
110
|
+
}
|
|
111
|
+
async createTempRoot() {
|
|
112
|
+
const parent = resolve(this.options.tempRootPath ?? tmpdir());
|
|
113
|
+
try {
|
|
114
|
+
await mkdir(parent, { recursive: true });
|
|
115
|
+
return await mkdtemp(join(parent, 'mcp-canvas-'));
|
|
116
|
+
}
|
|
117
|
+
catch (err) {
|
|
118
|
+
throw configurationError('Canvas scratch directory could not be created: CANVAS_TEMP_PATH (the OS temp directory when unset) must be writable by the server process.', undefined, { cause: err });
|
|
119
|
+
}
|
|
44
120
|
}
|
|
45
121
|
async initCanvas(canvasId, _context) {
|
|
46
122
|
if (this.canvases.has(canvasId))
|
|
47
123
|
return;
|
|
48
124
|
const duck = await importDuckDB();
|
|
49
|
-
const
|
|
125
|
+
const root = this.ensureTempRoot();
|
|
126
|
+
const scratchDir = await root;
|
|
127
|
+
// DuckDB names spill files by block size alone, so instances sharing a
|
|
128
|
+
// temp_directory overwrite each other's evicted blocks (#561).
|
|
129
|
+
const spillDir = join(scratchDir, randomUUID());
|
|
50
130
|
const instance = await duck.DuckDBInstance.create(':memory:', {
|
|
51
131
|
memory_limit: `${this.options.memoryLimitMb}MB`,
|
|
52
|
-
temp_directory:
|
|
132
|
+
temp_directory: spillDir,
|
|
53
133
|
// Disable extension install/load paths in canvas mode.
|
|
54
134
|
autoinstall_known_extensions: 'false',
|
|
55
135
|
autoload_known_extensions: 'false',
|
|
56
136
|
});
|
|
57
137
|
const controlConnection = await instance.connect();
|
|
58
138
|
await controlConnection.run(`SET memory_limit = '${this.options.memoryLimitMb}MB'`);
|
|
59
|
-
|
|
139
|
+
// A shutdown that began meanwhile retired `scratchDir` and may already
|
|
140
|
+
// have removed it, freeing the name for another local user to re-create;
|
|
141
|
+
// DuckDB creates only the leaf of its temp_directory and would spill into
|
|
142
|
+
// whatever sits there. The check and the insert share one synchronous run,
|
|
143
|
+
// so no shutdown can begin between them.
|
|
144
|
+
if (this.tempRoot !== root) {
|
|
145
|
+
controlConnection.closeSync();
|
|
146
|
+
instance.closeSync();
|
|
147
|
+
throw serviceUnavailable('Canvas provider shut down while the canvas was being created.');
|
|
148
|
+
}
|
|
149
|
+
this.canvases.set(canvasId, {
|
|
150
|
+
instance,
|
|
151
|
+
controlConnection,
|
|
152
|
+
holds: new Set(),
|
|
153
|
+
scratchDir,
|
|
154
|
+
spillDir,
|
|
155
|
+
});
|
|
60
156
|
}
|
|
61
|
-
|
|
157
|
+
/**
|
|
158
|
+
* Close the canvas's instance and remove its spill directory. A call still
|
|
159
|
+
* running on the canvas keeps the database open on its own connection and
|
|
160
|
+
* can still read spilled blocks back from that directory, so the directory
|
|
161
|
+
* goes once the last such call settles, and this does not wait for it.
|
|
162
|
+
* DuckDB removes a `temp_directory` it created when the database finally
|
|
163
|
+
* closes; the removal here covers every other case, so nothing of the
|
|
164
|
+
* canvas outlives it.
|
|
165
|
+
*/
|
|
62
166
|
async destroyCanvas(canvasId, _context) {
|
|
63
167
|
const record = this.canvases.get(canvasId);
|
|
64
168
|
if (!record)
|
|
@@ -80,6 +184,14 @@ export class DuckdbProvider {
|
|
|
80
184
|
catch (err) {
|
|
81
185
|
logger.warning('DuckDB instance close failed.', withExtra(closeContext, { error: err instanceof Error ? err.message : String(err) }));
|
|
82
186
|
}
|
|
187
|
+
const remove = () => rm(record.spillDir, { recursive: true, force: true }).catch((err) => {
|
|
188
|
+
logger.warning('Canvas spill directory removal failed.', withExtra(closeContext, { error: err instanceof Error ? err.message : String(err) }));
|
|
189
|
+
});
|
|
190
|
+
if (record.holds.size > 0) {
|
|
191
|
+
void Promise.all(record.holds).then(remove);
|
|
192
|
+
return;
|
|
193
|
+
}
|
|
194
|
+
await remove();
|
|
83
195
|
}
|
|
84
196
|
async healthCheck() {
|
|
85
197
|
try {
|
|
@@ -95,312 +207,413 @@ export class DuckdbProvider {
|
|
|
95
207
|
return false;
|
|
96
208
|
}
|
|
97
209
|
}
|
|
210
|
+
/**
|
|
211
|
+
* Destroy every canvas, then remove the private scratch directory. A call
|
|
212
|
+
* that reached a canvas before shutdown began can still be writing there —
|
|
213
|
+
* an open per-query connection outlives its closed instance — so the
|
|
214
|
+
* directory goes once the last such call settles, and shutdown does not
|
|
215
|
+
* wait for it. Until then the directory stays in place and private, which
|
|
216
|
+
* keeps another local user from re-creating its name while that call still
|
|
217
|
+
* writes inside it. A canvas whose creation straddles the shutdown is
|
|
218
|
+
* refused rather than bound to the retired directory. A provider used
|
|
219
|
+
* again afterwards creates a fresh one. After an abnormal exit the
|
|
220
|
+
* directory stays behind, still private; nothing sweeps it on the next
|
|
221
|
+
* start.
|
|
222
|
+
*/
|
|
98
223
|
async shutdown() {
|
|
99
224
|
const context = requestContextService.createRequestContext({
|
|
100
225
|
operation: 'DuckdbProvider.shutdown',
|
|
101
226
|
});
|
|
102
227
|
const ids = [...this.canvases.keys()];
|
|
228
|
+
const held = [...this.holds];
|
|
229
|
+
const tempRoot = this.tempRoot;
|
|
230
|
+
this.tempRoot = undefined;
|
|
103
231
|
await Promise.allSettled(ids.map((id) => this.destroyCanvas(id, context)));
|
|
232
|
+
const dir = await tempRoot?.catch(() => undefined);
|
|
233
|
+
if (!dir)
|
|
234
|
+
return;
|
|
235
|
+
const remove = () => rm(dir, { recursive: true, force: true }).catch((err) => {
|
|
236
|
+
logger.warning('Canvas scratch directory removal failed; it stays in place, still private.', withExtra(context, { error: err instanceof Error ? err.message : String(err) }));
|
|
237
|
+
});
|
|
238
|
+
const pending = held.filter((hold) => this.holds.has(hold));
|
|
239
|
+
if (pending.length > 0) {
|
|
240
|
+
void Promise.all(pending).then(remove);
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
await remove();
|
|
104
244
|
}
|
|
105
245
|
// ---------------------------------------------------------------------
|
|
106
246
|
// Data plane
|
|
107
247
|
// ---------------------------------------------------------------------
|
|
108
248
|
async registerTable(canvasId, name, rows, _context, options) {
|
|
109
|
-
const
|
|
110
|
-
const duck = await importDuckDB();
|
|
111
|
-
assertValidIdentifier(name, 'table');
|
|
112
|
-
options?.signal?.throwIfAborted();
|
|
113
|
-
const isAsyncIterable = typeof rows[Symbol.asyncIterator] === 'function';
|
|
114
|
-
let schema;
|
|
115
|
-
let bufferedRows;
|
|
116
|
-
let remainingSync;
|
|
117
|
-
if (options?.schema) {
|
|
118
|
-
schema = options.schema;
|
|
119
|
-
}
|
|
120
|
-
else if (isAsyncIterable) {
|
|
121
|
-
throw validationError('Schema must be provided explicitly when registering rows from an AsyncIterable.', { reason: 'async_iterable_requires_schema', tableName: name });
|
|
122
|
-
}
|
|
123
|
-
else {
|
|
124
|
-
const sniffed = sniffSchema(rows, this.options.schemaSniffRows);
|
|
125
|
-
schema = sniffed.schema;
|
|
126
|
-
bufferedRows = sniffed.sniffedRows;
|
|
127
|
-
remainingSync = sniffed.remaining;
|
|
128
|
-
}
|
|
129
|
-
for (const col of schema)
|
|
130
|
-
assertValidIdentifier(col.name, 'column');
|
|
131
|
-
// Drop+create makes register idempotent under re-registration of a name.
|
|
132
|
-
const ddl = buildCreateTableSql(name, schema);
|
|
133
|
-
await record.controlConnection.run(`DROP TABLE IF EXISTS ${quoteIdentifier(name)}`);
|
|
134
|
-
await record.controlConnection.run(ddl);
|
|
135
|
-
options?.signal?.throwIfAborted();
|
|
136
|
-
const appender = await record.controlConnection.createAppender(name);
|
|
137
|
-
let count = 0;
|
|
249
|
+
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
138
250
|
try {
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
appendOne(row);
|
|
150
|
-
}
|
|
151
|
-
}
|
|
152
|
-
if (isAsyncIterable) {
|
|
153
|
-
for await (const row of rows) {
|
|
154
|
-
options?.signal?.throwIfAborted();
|
|
155
|
-
appendOne(row);
|
|
156
|
-
}
|
|
251
|
+
const record = __addDisposableResource(env_1, this.holdCanvas(canvasId), false);
|
|
252
|
+
const duck = await importDuckDB();
|
|
253
|
+
assertValidIdentifier(name, 'table');
|
|
254
|
+
options?.signal?.throwIfAborted();
|
|
255
|
+
const isAsyncIterable = typeof rows[Symbol.asyncIterator] === 'function';
|
|
256
|
+
let schema;
|
|
257
|
+
let bufferedRows;
|
|
258
|
+
let remainingSync;
|
|
259
|
+
if (options?.schema) {
|
|
260
|
+
schema = options.schema;
|
|
157
261
|
}
|
|
158
|
-
else if (
|
|
159
|
-
|
|
160
|
-
// bufferedRows so we don't re-iterate (which would drop data on
|
|
161
|
-
// generators or duplicate rows from fresh-iterator iterables).
|
|
162
|
-
let next = remainingSync.next();
|
|
163
|
-
while (!next.done) {
|
|
164
|
-
options?.signal?.throwIfAborted();
|
|
165
|
-
appendOne(next.value);
|
|
166
|
-
next = remainingSync.next();
|
|
167
|
-
}
|
|
262
|
+
else if (isAsyncIterable) {
|
|
263
|
+
throw validationError('Schema must be provided explicitly when registering rows from an AsyncIterable.', { reason: 'async_iterable_requires_schema', tableName: name });
|
|
168
264
|
}
|
|
169
265
|
else {
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
266
|
+
const sniffed = sniffSchema(rows, this.options.schemaSniffRows);
|
|
267
|
+
schema = sniffed.schema;
|
|
268
|
+
bufferedRows = sniffed.sniffedRows;
|
|
269
|
+
remainingSync = sniffed.remaining;
|
|
270
|
+
}
|
|
271
|
+
for (const col of schema)
|
|
272
|
+
assertValidIdentifier(col.name, 'column');
|
|
273
|
+
// Drop+create makes register idempotent under re-registration of a name.
|
|
274
|
+
const ddl = buildCreateTableSql(name, schema);
|
|
275
|
+
await record.controlConnection.run(`DROP TABLE IF EXISTS ${quoteIdentifier(name)}`);
|
|
276
|
+
await record.controlConnection.run(ddl);
|
|
277
|
+
options?.signal?.throwIfAborted();
|
|
278
|
+
let count = 0;
|
|
279
|
+
try {
|
|
280
|
+
const appender = await record.controlConnection.createAppender(name);
|
|
281
|
+
try {
|
|
282
|
+
const appendOne = (row) => {
|
|
283
|
+
for (const col of schema) {
|
|
284
|
+
appendValue(appender, col, row[col.name], duck);
|
|
285
|
+
}
|
|
286
|
+
appender.endRow();
|
|
287
|
+
count += 1;
|
|
288
|
+
};
|
|
289
|
+
if (bufferedRows) {
|
|
290
|
+
for (const row of bufferedRows) {
|
|
291
|
+
options?.signal?.throwIfAborted();
|
|
292
|
+
appendOne(row);
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (isAsyncIterable) {
|
|
296
|
+
for await (const row of rows) {
|
|
297
|
+
options?.signal?.throwIfAborted();
|
|
298
|
+
appendOne(row);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
else if (remainingSync) {
|
|
302
|
+
// Continuation iterator from the sniffer — picks up just past
|
|
303
|
+
// bufferedRows so we don't re-iterate (which would drop data on
|
|
304
|
+
// generators or duplicate rows from fresh-iterator iterables).
|
|
305
|
+
let next = remainingSync.next();
|
|
306
|
+
while (!next.done) {
|
|
307
|
+
options?.signal?.throwIfAborted();
|
|
308
|
+
appendOne(next.value);
|
|
309
|
+
next = remainingSync.next();
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
else {
|
|
313
|
+
for (const row of rows) {
|
|
314
|
+
options?.signal?.throwIfAborted();
|
|
315
|
+
appendOne(row);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
173
318
|
}
|
|
319
|
+
finally {
|
|
320
|
+
appender.closeSync();
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
catch (err) {
|
|
324
|
+
// A spill that cannot start fails here; the caller's own row source
|
|
325
|
+
// runs in the same block, so only errors naming a host path change.
|
|
326
|
+
throw this.redactHostPaths(err, record);
|
|
174
327
|
}
|
|
328
|
+
return {
|
|
329
|
+
tableName: name,
|
|
330
|
+
rowCount: count,
|
|
331
|
+
columns: schema.map((c) => c.name),
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
catch (e_1) {
|
|
335
|
+
env_1.error = e_1;
|
|
336
|
+
env_1.hasError = true;
|
|
175
337
|
}
|
|
176
338
|
finally {
|
|
177
|
-
|
|
339
|
+
__disposeResources(env_1);
|
|
178
340
|
}
|
|
179
|
-
return {
|
|
180
|
-
tableName: name,
|
|
181
|
-
rowCount: count,
|
|
182
|
-
columns: schema.map((c) => c.name),
|
|
183
|
-
};
|
|
184
341
|
}
|
|
185
342
|
async query(canvasId, sql, _context, options) {
|
|
186
|
-
const
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
rowLimit
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
if (!Number.isSafeInteger(preview) || preview < 0 || preview > rowLimit) {
|
|
196
|
-
throw validationError('preview must be a non-negative safe integer no greater than rowLimit.', { reason: 'invalid_query_bounds', field: 'preview' });
|
|
197
|
-
}
|
|
198
|
-
const duck = await importDuckDB();
|
|
199
|
-
await this.assertReadOnlySql(record, sql, duck, options);
|
|
200
|
-
return await this.withConnection(record, options?.signal, 'Canvas query was cancelled.', async (conn) => {
|
|
201
|
-
let registeredAs;
|
|
202
|
-
let rowsToReturn = [];
|
|
203
|
-
let columns = [];
|
|
204
|
-
let totalRowCount = 0;
|
|
205
|
-
let truncated;
|
|
206
|
-
if (options?.registerAs) {
|
|
207
|
-
assertValidIdentifier(options.registerAs, 'table');
|
|
208
|
-
await ensureTableMissing(record.controlConnection, options.registerAs);
|
|
209
|
-
const ctas = `CREATE TABLE ${quoteIdentifier(options.registerAs)} AS ${sql}`;
|
|
210
|
-
await conn.run(ctas);
|
|
211
|
-
registeredAs = options.registerAs;
|
|
212
|
-
const reader = await conn.runAndReadUntil(`SELECT * FROM ${quoteIdentifier(options.registerAs)} LIMIT ${preview}`, preview);
|
|
213
|
-
rowsToReturn = reader.getRowObjectsJson();
|
|
214
|
-
columns = reader.columnNames();
|
|
215
|
-
totalRowCount = await this.countRows(conn, options.registerAs);
|
|
216
|
-
// truncated is false on the registerAs path — rowCount is exact.
|
|
343
|
+
const env_2 = { stack: [], error: void 0, hasError: false };
|
|
344
|
+
try {
|
|
345
|
+
const record = __addDisposableResource(env_2, this.holdCanvas(canvasId), false);
|
|
346
|
+
options?.signal?.throwIfAborted();
|
|
347
|
+
const rowLimit = options?.rowLimit ?? this.options.defaultRowLimit;
|
|
348
|
+
if (!Number.isSafeInteger(rowLimit) ||
|
|
349
|
+
rowLimit < 1 ||
|
|
350
|
+
rowLimit > this.options.defaultRowLimit) {
|
|
351
|
+
throw validationError(`rowLimit must be a positive safe integer no greater than ${this.options.defaultRowLimit}.`, { reason: 'invalid_query_bounds', field: 'rowLimit' });
|
|
217
352
|
}
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
353
|
+
const preview = options?.preview ?? rowLimit;
|
|
354
|
+
if (!Number.isSafeInteger(preview) || preview < 0 || preview > rowLimit) {
|
|
355
|
+
throw validationError('preview must be a non-negative safe integer no greater than rowLimit.', { reason: 'invalid_query_bounds', field: 'preview' });
|
|
356
|
+
}
|
|
357
|
+
const duck = await importDuckDB();
|
|
358
|
+
await this.assertReadOnlySql(record, sql, duck, options);
|
|
359
|
+
return await this.withConnection(record, options?.signal, 'Canvas query was cancelled.', async (conn) => {
|
|
360
|
+
let registeredAs;
|
|
361
|
+
let rowsToReturn = [];
|
|
362
|
+
let columns = [];
|
|
363
|
+
let totalRowCount = 0;
|
|
364
|
+
let truncated;
|
|
365
|
+
if (options?.registerAs) {
|
|
366
|
+
assertValidIdentifier(options.registerAs, 'table');
|
|
367
|
+
await ensureTableMissing(record.controlConnection, options.registerAs);
|
|
368
|
+
const ctas = `CREATE TABLE ${quoteIdentifier(options.registerAs)} AS ${sql}`;
|
|
369
|
+
await conn.run(ctas);
|
|
370
|
+
registeredAs = options.registerAs;
|
|
371
|
+
const reader = await conn.runAndReadUntil(`SELECT * FROM ${quoteIdentifier(options.registerAs)} LIMIT ${preview}`, preview);
|
|
372
|
+
rowsToReturn = reader.getRowObjectsJson();
|
|
373
|
+
columns = reader.columnNames();
|
|
374
|
+
totalRowCount = await this.countRows(conn, options.registerAs);
|
|
375
|
+
// truncated is false on the registerAs path — rowCount is exact.
|
|
229
376
|
}
|
|
230
377
|
else {
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
378
|
+
// Use streamAndReadUntil to keep the engine pipeline lazy: reads at
|
|
379
|
+
// most rowLimit+1 rows without materializing the full result in JS.
|
|
380
|
+
const reader = await conn.streamAndReadUntil(sql, rowLimit + 1);
|
|
381
|
+
const fetched = reader.getRowObjectsJson();
|
|
382
|
+
columns = reader.columnNames();
|
|
383
|
+
if (fetched.length > rowLimit) {
|
|
384
|
+
// More rows exist beyond the cap.
|
|
385
|
+
rowsToReturn = fetched.slice(0, rowLimit);
|
|
386
|
+
totalRowCount = rowLimit;
|
|
387
|
+
truncated = true;
|
|
388
|
+
}
|
|
389
|
+
else {
|
|
390
|
+
rowsToReturn = fetched;
|
|
391
|
+
totalRowCount = fetched.length;
|
|
392
|
+
}
|
|
393
|
+
// Apply preview cap (may be smaller than rowLimit when caller requests a smaller slice).
|
|
394
|
+
if (rowsToReturn.length > preview) {
|
|
395
|
+
rowsToReturn = rowsToReturn.slice(0, preview);
|
|
396
|
+
}
|
|
237
397
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
};
|
|
246
|
-
}
|
|
398
|
+
return {
|
|
399
|
+
rows: rowsToReturn,
|
|
400
|
+
columns,
|
|
401
|
+
rowCount: totalRowCount,
|
|
402
|
+
...(truncated && { truncated }),
|
|
403
|
+
...(registeredAs && { tableName: registeredAs }),
|
|
404
|
+
};
|
|
405
|
+
});
|
|
406
|
+
}
|
|
407
|
+
catch (e_2) {
|
|
408
|
+
env_2.error = e_2;
|
|
409
|
+
env_2.hasError = true;
|
|
410
|
+
}
|
|
411
|
+
finally {
|
|
412
|
+
__disposeResources(env_2);
|
|
413
|
+
}
|
|
247
414
|
}
|
|
248
415
|
async export(canvasId, tableName, target, _context, options) {
|
|
249
|
-
const
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
const
|
|
255
|
-
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
|
|
416
|
+
const env_3 = { stack: [], error: void 0, hasError: false };
|
|
417
|
+
try {
|
|
418
|
+
const record = __addDisposableResource(env_3, this.holdCanvas(canvasId), false);
|
|
419
|
+
assertValidIdentifier(tableName, 'table');
|
|
420
|
+
options?.signal?.throwIfAborted();
|
|
421
|
+
const formatClause = copyFormatClause(target.format);
|
|
422
|
+
return await this.withConnection(record, options?.signal, 'Canvas export was cancelled.', async (conn) => {
|
|
423
|
+
const rowCount = await this.countRows(conn, tableName);
|
|
424
|
+
if (isPathTarget(target)) {
|
|
425
|
+
const absolutePath = await resolveExportPath(this.options.exportRootPath, target.path);
|
|
426
|
+
await conn.run(`COPY ${quoteIdentifier(tableName)} TO '${escapeSqlString(absolutePath)}' ${formatClause}`);
|
|
427
|
+
const sizeBytes = await safeSizeBytes(absolutePath);
|
|
428
|
+
return {
|
|
429
|
+
format: target.format,
|
|
430
|
+
path: absolutePath,
|
|
431
|
+
sizeBytes,
|
|
432
|
+
rowCount,
|
|
433
|
+
};
|
|
434
|
+
}
|
|
435
|
+
// Stream branch: COPY to a scratch file, pipe to the caller's stream,
|
|
436
|
+
// then unlink. pipeFileToStream owns cleanup once invoked; if the COPY
|
|
437
|
+
// itself fails we must unlink here before re-throwing.
|
|
438
|
+
const tempPath = tempFilePathFor(record.scratchDir, target.format);
|
|
439
|
+
try {
|
|
440
|
+
await conn.run(`COPY ${quoteIdentifier(tableName)} TO '${escapeSqlString(tempPath)}' ${formatClause}`);
|
|
441
|
+
}
|
|
442
|
+
catch (copyErr) {
|
|
443
|
+
await unlink(tempPath).catch(() => { });
|
|
444
|
+
throw copyErr;
|
|
445
|
+
}
|
|
446
|
+
const { sizeBytes } = await pipeFileToStream(tempPath, target.stream);
|
|
259
447
|
return {
|
|
260
448
|
format: target.format,
|
|
261
|
-
path: absolutePath,
|
|
262
449
|
sizeBytes,
|
|
263
450
|
rowCount,
|
|
264
451
|
};
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
catch (e_3) {
|
|
455
|
+
env_3.error = e_3;
|
|
456
|
+
env_3.hasError = true;
|
|
457
|
+
}
|
|
458
|
+
finally {
|
|
459
|
+
__disposeResources(env_3);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
async registerView(canvasId, name, selectSql, _context, options) {
|
|
463
|
+
const env_4 = { stack: [], error: void 0, hasError: false };
|
|
464
|
+
try {
|
|
465
|
+
const record = __addDisposableResource(env_4, this.holdCanvas(canvasId), false);
|
|
466
|
+
const duck = await importDuckDB();
|
|
467
|
+
assertValidIdentifier(name, 'table');
|
|
468
|
+
options?.signal?.throwIfAborted();
|
|
469
|
+
// Same four-layer gate `query()` enforces. View definitions inherit the
|
|
470
|
+
// operator allowlist transitively at query time, but we also gate the
|
|
471
|
+
// SELECT at registration so a malicious definition fails loud here, not
|
|
472
|
+
// later when the view is referenced.
|
|
473
|
+
await this.assertReadOnlySql(record, selectSql, duck, options);
|
|
474
|
+
options?.signal?.throwIfAborted();
|
|
475
|
+
// Block view-on-table-name collisions explicitly so the failure carries a
|
|
476
|
+
// structured `reason` rather than a raw DuckDB catalog message.
|
|
477
|
+
const existing = await this.lookupKind(record.controlConnection, name);
|
|
478
|
+
if (existing === 'table') {
|
|
479
|
+
throw validationError(`Canvas already contains a base table named "${name}". Drop the table or choose a different name.`, { reason: 'view_table_clash', name });
|
|
265
480
|
}
|
|
266
|
-
// Stream branch: COPY to a scratch file, pipe to the caller's stream,
|
|
267
|
-
// then unlink. pipeFileToStream owns cleanup once invoked; if the COPY
|
|
268
|
-
// itself fails we must unlink here before re-throwing.
|
|
269
|
-
const tempPath = await tempFilePathFor(await this.ensureTempRoot(), target.format);
|
|
270
481
|
try {
|
|
271
|
-
await
|
|
482
|
+
await record.controlConnection.run(`CREATE OR REPLACE VIEW ${quoteIdentifier(name)} AS ${selectSql}`);
|
|
272
483
|
}
|
|
273
|
-
catch (
|
|
274
|
-
|
|
275
|
-
throw copyErr;
|
|
484
|
+
catch (err) {
|
|
485
|
+
throw this.classify(err, record);
|
|
276
486
|
}
|
|
277
|
-
|
|
278
|
-
return {
|
|
279
|
-
format: target.format,
|
|
280
|
-
sizeBytes,
|
|
281
|
-
rowCount,
|
|
282
|
-
};
|
|
283
|
-
});
|
|
284
|
-
}
|
|
285
|
-
async registerView(canvasId, name, selectSql, _context, options) {
|
|
286
|
-
const record = this.requireCanvas(canvasId);
|
|
287
|
-
const duck = await importDuckDB();
|
|
288
|
-
assertValidIdentifier(name, 'table');
|
|
289
|
-
options?.signal?.throwIfAborted();
|
|
290
|
-
// Same four-layer gate `query()` enforces. View definitions inherit the
|
|
291
|
-
// operator allowlist transitively at query time, but we also gate the
|
|
292
|
-
// SELECT at registration so a malicious definition fails loud here, not
|
|
293
|
-
// later when the view is referenced.
|
|
294
|
-
await this.assertReadOnlySql(record, selectSql, duck, options);
|
|
295
|
-
options?.signal?.throwIfAborted();
|
|
296
|
-
// Block view-on-table-name collisions explicitly so the failure carries a
|
|
297
|
-
// structured `reason` rather than a raw DuckDB catalog message.
|
|
298
|
-
const existing = await this.lookupKind(record.controlConnection, name);
|
|
299
|
-
if (existing === 'table') {
|
|
300
|
-
throw validationError(`Canvas already contains a base table named "${name}". Drop the table or choose a different name.`, { reason: 'view_table_clash', name });
|
|
487
|
+
return { viewName: name, columns: await this.columnNames(record.controlConnection, name) };
|
|
301
488
|
}
|
|
302
|
-
|
|
303
|
-
|
|
489
|
+
catch (e_4) {
|
|
490
|
+
env_4.error = e_4;
|
|
491
|
+
env_4.hasError = true;
|
|
304
492
|
}
|
|
305
|
-
|
|
306
|
-
|
|
493
|
+
finally {
|
|
494
|
+
__disposeResources(env_4);
|
|
307
495
|
}
|
|
308
|
-
return { viewName: name, columns: await this.columnNames(record.controlConnection, name) };
|
|
309
496
|
}
|
|
310
497
|
async importFrom(targetCanvasId, sourceCanvasId, sourceTableName, asName, _context, options) {
|
|
311
|
-
|
|
312
|
-
throw validationError('Source and target canvases must differ. Importing copies a table between two canvases.', {
|
|
313
|
-
reason: 'import_same_canvas',
|
|
314
|
-
recovery: {
|
|
315
|
-
hint: 'Pass the canvas id of a different canvas as the source — the table is already on this one.',
|
|
316
|
-
},
|
|
317
|
-
});
|
|
318
|
-
}
|
|
319
|
-
const target = this.requireCanvas(targetCanvasId);
|
|
320
|
-
const source = this.requireCanvas(sourceCanvasId);
|
|
321
|
-
assertValidIdentifier(sourceTableName, 'table');
|
|
322
|
-
assertValidIdentifier(asName, 'table');
|
|
323
|
-
options?.signal?.throwIfAborted();
|
|
324
|
-
const sourceKind = await this.lookupKind(source.controlConnection, sourceTableName);
|
|
325
|
-
if (sourceKind === undefined) {
|
|
326
|
-
throw notFound(`Source canvas does not contain a table or view named "${sourceTableName}".`, {
|
|
327
|
-
reason: 'missing_table',
|
|
328
|
-
sourceCanvasId,
|
|
329
|
-
sourceTableName,
|
|
330
|
-
recovery: {
|
|
331
|
-
hint: "Re-check the source table name, or list the tables staged on the source canvas with this server's dataframe-describe tool.",
|
|
332
|
-
},
|
|
333
|
-
});
|
|
334
|
-
}
|
|
335
|
-
const targetExisting = await this.lookupKind(target.controlConnection, asName);
|
|
336
|
-
if (targetExisting === 'view') {
|
|
337
|
-
throw validationError(`Target canvas already contains a view named "${asName}". Drop the view or choose a different name.`, {
|
|
338
|
-
reason: 'import_view_clash',
|
|
339
|
-
asName,
|
|
340
|
-
recovery: {
|
|
341
|
-
hint: 'Drop the existing view first, or choose a different destination name for the imported table.',
|
|
342
|
-
},
|
|
343
|
-
});
|
|
344
|
-
}
|
|
345
|
-
// Drop+create makes import idempotent under re-imports of the same name,
|
|
346
|
-
// matching registerTable's behavior.
|
|
347
|
-
await target.controlConnection.run(`DROP TABLE IF EXISTS ${quoteIdentifier(asName)}`);
|
|
348
|
-
// Round-trip through a scratch Parquet file. Parquet is
|
|
349
|
-
// built into DuckDB's core (no extension load needed even with
|
|
350
|
-
// autoload disabled). All column types — including TIMESTAMP/DATE/BLOB
|
|
351
|
-
// — round-trip losslessly, which an in-memory appender path can't
|
|
352
|
-
// guarantee for native engine value types.
|
|
353
|
-
const tempPath = await tempFilePathFor(await this.ensureTempRoot(), 'parquet');
|
|
498
|
+
const env_5 = { stack: [], error: void 0, hasError: false };
|
|
354
499
|
try {
|
|
355
|
-
|
|
500
|
+
if (sourceCanvasId === targetCanvasId) {
|
|
501
|
+
throw validationError('Source and target canvases must differ. Importing copies a table between two canvases.', {
|
|
502
|
+
reason: 'import_same_canvas',
|
|
503
|
+
recovery: {
|
|
504
|
+
hint: 'Pass the canvas id of a different canvas as the source — the table is already on this one.',
|
|
505
|
+
},
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
const target = __addDisposableResource(env_5, this.holdCanvas(targetCanvasId), false);
|
|
509
|
+
const source = __addDisposableResource(env_5, this.holdCanvas(sourceCanvasId), false);
|
|
510
|
+
assertValidIdentifier(sourceTableName, 'table');
|
|
511
|
+
assertValidIdentifier(asName, 'table');
|
|
356
512
|
options?.signal?.throwIfAborted();
|
|
357
|
-
await
|
|
513
|
+
const sourceKind = await this.lookupKind(source.controlConnection, sourceTableName);
|
|
514
|
+
if (sourceKind === undefined) {
|
|
515
|
+
throw notFound(`Source canvas does not contain a table or view named "${sourceTableName}".`, {
|
|
516
|
+
reason: 'missing_table',
|
|
517
|
+
sourceCanvasId,
|
|
518
|
+
sourceTableName,
|
|
519
|
+
recovery: {
|
|
520
|
+
hint: "Re-check the source table name, or list the tables staged on the source canvas with this server's dataframe-describe tool.",
|
|
521
|
+
},
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
const targetExisting = await this.lookupKind(target.controlConnection, asName);
|
|
525
|
+
if (targetExisting === 'view') {
|
|
526
|
+
throw validationError(`Target canvas already contains a view named "${asName}". Drop the view or choose a different name.`, {
|
|
527
|
+
reason: 'import_view_clash',
|
|
528
|
+
asName,
|
|
529
|
+
recovery: {
|
|
530
|
+
hint: 'Drop the existing view first, or choose a different destination name for the imported table.',
|
|
531
|
+
},
|
|
532
|
+
});
|
|
533
|
+
}
|
|
534
|
+
// Drop+create makes import idempotent under re-imports of the same name,
|
|
535
|
+
// matching registerTable's behavior.
|
|
536
|
+
await target.controlConnection.run(`DROP TABLE IF EXISTS ${quoteIdentifier(asName)}`);
|
|
537
|
+
// Round-trip through a scratch Parquet file. Parquet is
|
|
538
|
+
// built into DuckDB's core (no extension load needed even with
|
|
539
|
+
// autoload disabled). All column types — including TIMESTAMP/DATE/BLOB
|
|
540
|
+
// — round-trip losslessly, which an in-memory appender path can't
|
|
541
|
+
// guarantee for native engine value types.
|
|
542
|
+
const tempPath = tempFilePathFor(target.scratchDir, 'parquet');
|
|
543
|
+
try {
|
|
544
|
+
await source.controlConnection.run(`COPY ${quoteIdentifier(sourceTableName)} TO '${escapeSqlString(tempPath)}' (FORMAT 'parquet')`);
|
|
545
|
+
options?.signal?.throwIfAborted();
|
|
546
|
+
await target.controlConnection.run(`CREATE TABLE ${quoteIdentifier(asName)} AS SELECT * FROM read_parquet('${escapeSqlString(tempPath)}')`);
|
|
547
|
+
}
|
|
548
|
+
catch (err) {
|
|
549
|
+
// Best-effort cleanup of a half-written target before surfacing.
|
|
550
|
+
await target.controlConnection
|
|
551
|
+
.run(`DROP TABLE IF EXISTS ${quoteIdentifier(asName)}`)
|
|
552
|
+
.catch(() => { });
|
|
553
|
+
throw this.classify(err, target);
|
|
554
|
+
}
|
|
555
|
+
finally {
|
|
556
|
+
await unlink(tempPath).catch(() => { });
|
|
557
|
+
}
|
|
558
|
+
const [columns, rowCount] = await Promise.all([
|
|
559
|
+
this.columnNames(target.controlConnection, asName),
|
|
560
|
+
this.countRows(target.controlConnection, asName),
|
|
561
|
+
]);
|
|
562
|
+
return { tableName: asName, rowCount, columns };
|
|
358
563
|
}
|
|
359
|
-
catch (
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
.run(`DROP TABLE IF EXISTS ${quoteIdentifier(asName)}`)
|
|
363
|
-
.catch(() => { });
|
|
364
|
-
throw classifyDuckdbError(err);
|
|
564
|
+
catch (e_5) {
|
|
565
|
+
env_5.error = e_5;
|
|
566
|
+
env_5.hasError = true;
|
|
365
567
|
}
|
|
366
568
|
finally {
|
|
367
|
-
|
|
569
|
+
__disposeResources(env_5);
|
|
368
570
|
}
|
|
369
|
-
const [columns, rowCount] = await Promise.all([
|
|
370
|
-
this.columnNames(target.controlConnection, asName),
|
|
371
|
-
this.countRows(target.controlConnection, asName),
|
|
372
|
-
]);
|
|
373
|
-
return { tableName: asName, rowCount, columns };
|
|
374
571
|
}
|
|
375
572
|
async describe(canvasId, _context, options) {
|
|
376
|
-
const
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
filters
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
573
|
+
const env_6 = { stack: [], error: void 0, hasError: false };
|
|
574
|
+
try {
|
|
575
|
+
const record = __addDisposableResource(env_6, this.holdCanvas(canvasId), false);
|
|
576
|
+
if (options?.tableName !== undefined) {
|
|
577
|
+
assertValidIdentifier(options.tableName, 'table');
|
|
578
|
+
}
|
|
579
|
+
// Every pushed filter stays qualified with the `t` alias. The join that
|
|
580
|
+
// made an unqualified `table_name` ambiguous is gone (#324), but keeping
|
|
581
|
+
// the qualification means re-introducing a second source here cannot
|
|
582
|
+
// silently resurrect the Binder Error of #235.
|
|
583
|
+
const filters = [`t.table_schema = 'main'`];
|
|
584
|
+
if (options?.tableName) {
|
|
585
|
+
filters.push(`t.table_name = '${escapeSqlString(options.tableName)}'`);
|
|
586
|
+
}
|
|
587
|
+
if (options?.kind === 'view') {
|
|
588
|
+
filters.push(`t.table_type = 'VIEW'`);
|
|
589
|
+
}
|
|
590
|
+
else if (options?.kind === 'table') {
|
|
591
|
+
filters.push(`t.table_type <> 'VIEW'`);
|
|
592
|
+
}
|
|
593
|
+
// No join against duckdb_tables(): its `estimated_size` is an estimated row
|
|
594
|
+
// count, not a byte footprint, so reporting it as `approxSizeBytes` sent a
|
|
595
|
+
// caller choosing an eviction victim after the wrong table. DuckDB exposes
|
|
596
|
+
// no per-table byte column at all (#324), so describe() reports none.
|
|
597
|
+
const reader = await record.controlConnection.runAndReadAll(`SELECT t.table_name, t.table_type
|
|
399
598
|
FROM information_schema.tables t
|
|
400
599
|
WHERE ${filters.join(' AND ')}
|
|
401
600
|
ORDER BY t.table_name`);
|
|
402
|
-
|
|
403
|
-
|
|
601
|
+
const tableRows = reader.getRowObjectsJson();
|
|
602
|
+
try {
|
|
603
|
+
return await Promise.all(tableRows.map((row) => this.describeOne(record.controlConnection, row.table_name, row.table_type === 'VIEW' ? 'view' : 'table')));
|
|
604
|
+
}
|
|
605
|
+
catch (err) {
|
|
606
|
+
// Counting a view runs it, and a view past the memory limit spills.
|
|
607
|
+
throw this.redactHostPaths(err, record);
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
catch (e_6) {
|
|
611
|
+
env_6.error = e_6;
|
|
612
|
+
env_6.hasError = true;
|
|
613
|
+
}
|
|
614
|
+
finally {
|
|
615
|
+
__disposeResources(env_6);
|
|
616
|
+
}
|
|
404
617
|
}
|
|
405
618
|
async describeOne(connection, tableName, kind) {
|
|
406
619
|
const [colRows, rowCount] = await Promise.all([
|
|
@@ -420,43 +633,103 @@ export class DuckdbProvider {
|
|
|
420
633
|
};
|
|
421
634
|
}
|
|
422
635
|
async drop(canvasId, name, _context) {
|
|
423
|
-
const
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
636
|
+
const env_7 = { stack: [], error: void 0, hasError: false };
|
|
637
|
+
try {
|
|
638
|
+
const record = __addDisposableResource(env_7, this.holdCanvas(canvasId), false);
|
|
639
|
+
assertValidIdentifier(name, 'table');
|
|
640
|
+
const kind = await this.lookupKind(record.controlConnection, name);
|
|
641
|
+
if (kind === undefined)
|
|
642
|
+
return false;
|
|
643
|
+
const dropKeyword = kind === 'view' ? 'VIEW' : 'TABLE';
|
|
644
|
+
await record.controlConnection.run(`DROP ${dropKeyword} ${quoteIdentifier(name)}`);
|
|
645
|
+
return true;
|
|
646
|
+
}
|
|
647
|
+
catch (e_7) {
|
|
648
|
+
env_7.error = e_7;
|
|
649
|
+
env_7.hasError = true;
|
|
650
|
+
}
|
|
651
|
+
finally {
|
|
652
|
+
__disposeResources(env_7);
|
|
653
|
+
}
|
|
431
654
|
}
|
|
432
655
|
async clear(canvasId, _context) {
|
|
433
|
-
const
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
const
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
656
|
+
const env_8 = { stack: [], error: void 0, hasError: false };
|
|
657
|
+
try {
|
|
658
|
+
const record = __addDisposableResource(env_8, this.holdCanvas(canvasId), false);
|
|
659
|
+
const reader = await record.controlConnection.runAndReadAll(`SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = 'main'`);
|
|
660
|
+
const rows = reader.getRowObjectsJson();
|
|
661
|
+
// Drop views before tables so a dependent view doesn't block its base table.
|
|
662
|
+
const ordered = [...rows].sort((a, b) => {
|
|
663
|
+
const aView = a.table_type === 'VIEW';
|
|
664
|
+
const bView = b.table_type === 'VIEW';
|
|
665
|
+
if (aView !== bView)
|
|
666
|
+
return aView ? -1 : 1;
|
|
667
|
+
return a.table_name.localeCompare(b.table_name);
|
|
668
|
+
});
|
|
669
|
+
for (const row of ordered) {
|
|
670
|
+
const dropKeyword = row.table_type === 'VIEW' ? 'VIEW' : 'TABLE';
|
|
671
|
+
await record.controlConnection.run(`DROP ${dropKeyword} ${quoteIdentifier(row.table_name)}`);
|
|
672
|
+
}
|
|
673
|
+
return rows.length;
|
|
674
|
+
}
|
|
675
|
+
catch (e_8) {
|
|
676
|
+
env_8.error = e_8;
|
|
677
|
+
env_8.hasError = true;
|
|
678
|
+
}
|
|
679
|
+
finally {
|
|
680
|
+
__disposeResources(env_8);
|
|
447
681
|
}
|
|
448
|
-
return rows.length;
|
|
449
682
|
}
|
|
450
683
|
// ---------------------------------------------------------------------
|
|
451
684
|
// Internals
|
|
452
685
|
// ---------------------------------------------------------------------
|
|
453
|
-
|
|
686
|
+
/**
|
|
687
|
+
* `canvasId`'s record, held until the caller's `using` scope exits, on
|
|
688
|
+
* return or throw. {@link destroyCanvas} removes the canvas's spill
|
|
689
|
+
* directory only once every hold on that canvas is released, so a call
|
|
690
|
+
* already running can still read back the blocks it spilled. {@link shutdown}
|
|
691
|
+
* removes the scratch directory only once every hold taken before it began
|
|
692
|
+
* is released, so a call already running finishes inside the private
|
|
693
|
+
* directory, never in one another local user re-created under its name.
|
|
694
|
+
* Every data-plane call reaches its record through a hold.
|
|
695
|
+
*/
|
|
696
|
+
holdCanvas(canvasId) {
|
|
454
697
|
const record = this.canvases.get(canvasId);
|
|
455
698
|
// Defensive — CanvasInstance touches the registry first, which throws the
|
|
456
699
|
// same structured canvas_not_found before the provider is reached (#261).
|
|
457
700
|
if (!record)
|
|
458
701
|
throw canvasNotFound(canvasId);
|
|
459
|
-
|
|
702
|
+
const { promise, resolve } = Promise.withResolvers();
|
|
703
|
+
this.holds.add(promise);
|
|
704
|
+
record.holds.add(promise);
|
|
705
|
+
return {
|
|
706
|
+
...record,
|
|
707
|
+
[Symbol.dispose]: () => {
|
|
708
|
+
this.holds.delete(promise);
|
|
709
|
+
record.holds.delete(promise);
|
|
710
|
+
resolve();
|
|
711
|
+
},
|
|
712
|
+
};
|
|
713
|
+
}
|
|
714
|
+
/** Host directories an engine message can quote and a caller must not see. */
|
|
715
|
+
hostPaths(record) {
|
|
716
|
+
return [resolve(this.options.exportRootPath), record.scratchDir];
|
|
717
|
+
}
|
|
718
|
+
/** {@link classifyDuckdbError} with this canvas's host directories redacted. */
|
|
719
|
+
classify(err, record) {
|
|
720
|
+
return classifyDuckdbError(err, this.hostPaths(record));
|
|
721
|
+
}
|
|
722
|
+
/**
|
|
723
|
+
* `err` classified and redacted when it is an engine error naming a host
|
|
724
|
+
* directory; anything else as it came. For blocks that also run code other
|
|
725
|
+
* than the engine's — a caller's row source, a cancellation check — whose
|
|
726
|
+
* failures must reach the caller as thrown.
|
|
727
|
+
*/
|
|
728
|
+
redactHostPaths(err, record) {
|
|
729
|
+
if (!(err instanceof Error) || err instanceof McpError)
|
|
730
|
+
return err;
|
|
731
|
+
const paths = this.hostPaths(record);
|
|
732
|
+
return redactPaths(err.message, paths) === err.message ? err : classifyDuckdbError(err, paths);
|
|
460
733
|
}
|
|
461
734
|
/**
|
|
462
735
|
* Runs `fn` on a connection of its own, so an abort interrupts only this
|
|
@@ -483,7 +756,7 @@ export class DuckdbProvider {
|
|
|
483
756
|
catch (err) {
|
|
484
757
|
if (cancelled)
|
|
485
758
|
throw timeout(cancelledMessage, { reason: 'cancelled' }, { cause: err });
|
|
486
|
-
throw
|
|
759
|
+
throw this.classify(err, record);
|
|
487
760
|
}
|
|
488
761
|
finally {
|
|
489
762
|
signal?.removeEventListener('abort', onAbort);
|
|
@@ -507,18 +780,19 @@ export class DuckdbProvider {
|
|
|
507
780
|
}
|
|
508
781
|
/**
|
|
509
782
|
* The rejection for a statement that failed to parse or prepare: a
|
|
510
|
-
* SELECT-shaped statement surfaces the
|
|
511
|
-
* itself is probably wrong), anything else is rejected as
|
|
783
|
+
* SELECT-shaped statement surfaces the binder message, host directories
|
|
784
|
+
* redacted (the SQL itself is probably wrong), anything else is rejected as
|
|
785
|
+
* non-SELECT.
|
|
512
786
|
*/
|
|
513
|
-
prepareFailure(sql, err) {
|
|
787
|
+
prepareFailure(sql, err, record) {
|
|
514
788
|
if (isSelectShaped(sql) && err instanceof Error) {
|
|
515
|
-
const binderMessage =
|
|
789
|
+
const binderMessage = redactPaths(err.message.trim(), this.hostPaths(record));
|
|
516
790
|
return validationError(`Canvas query failed to prepare: ${binderMessage}`, {
|
|
517
791
|
reason: SQL_GATE_REASONS.invalidSql,
|
|
518
792
|
statementType: 'UNKNOWN',
|
|
519
793
|
binderMessage,
|
|
520
794
|
...gateRecovery(SQL_GATE_REASONS.invalidSql),
|
|
521
|
-
});
|
|
795
|
+
}, { cause: err });
|
|
522
796
|
}
|
|
523
797
|
return validationError('Canvas query must be SELECT; the statement could not be parsed or prepared.', {
|
|
524
798
|
reason: SQL_GATE_REASONS.nonSelectStatement,
|
|
@@ -580,7 +854,7 @@ export class DuckdbProvider {
|
|
|
580
854
|
});
|
|
581
855
|
}
|
|
582
856
|
}
|
|
583
|
-
throw this.prepareFailure(sql, prepErr);
|
|
857
|
+
throw this.prepareFailure(sql, prepErr, record);
|
|
584
858
|
}
|
|
585
859
|
finally {
|
|
586
860
|
prepared?.destroySync();
|
|
@@ -597,7 +871,7 @@ export class DuckdbProvider {
|
|
|
597
871
|
// can carry errno-style `code` props and must not escape the gate raw.
|
|
598
872
|
if (err instanceof McpError)
|
|
599
873
|
throw err;
|
|
600
|
-
throw this.prepareFailure(sql, err);
|
|
874
|
+
throw this.prepareFailure(sql, err, record);
|
|
601
875
|
}
|
|
602
876
|
assertSelectOnly({ statementCount, statementType });
|
|
603
877
|
// Layer 4: walk the plan with the allowlist + denied-function rescan.
|
|
@@ -857,19 +1131,18 @@ function isSelectShaped(sql) {
|
|
|
857
1131
|
return /^\s*(?:select|with|from)\b/i.test(sql);
|
|
858
1132
|
}
|
|
859
1133
|
/**
|
|
860
|
-
*
|
|
861
|
-
*
|
|
862
|
-
*
|
|
863
|
-
*
|
|
864
|
-
*
|
|
1134
|
+
* Replace every occurrence of each host directory in an engine message with
|
|
1135
|
+
* `[path]`, longest first so a directory nested inside another is redacted
|
|
1136
|
+
* whole. DuckDB quotes the absolute export and scratch paths it failed on;
|
|
1137
|
+
* the part below the directory — the caller's own export name — stays. A
|
|
1138
|
+
* one-character path (the filesystem root) is skipped, since replacing it
|
|
1139
|
+
* would mangle every path in the message.
|
|
865
1140
|
*/
|
|
866
|
-
function
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
}
|
|
872
|
-
return trimmed;
|
|
1141
|
+
function redactPaths(message, paths) {
|
|
1142
|
+
return paths
|
|
1143
|
+
.filter((path) => path.length > 1)
|
|
1144
|
+
.sort((a, b) => b.length - a.length)
|
|
1145
|
+
.reduce((text, path) => text.replaceAll(path, '[path]'), message);
|
|
873
1146
|
}
|
|
874
1147
|
async function ensureTableMissing(connection, tableName) {
|
|
875
1148
|
const reader = await connection.runAndReadAll(`SELECT 1 FROM information_schema.tables WHERE table_schema = 'main' AND table_name = '${escapeSqlString(tableName)}' LIMIT 1`);
|
|
@@ -885,7 +1158,10 @@ async function ensureTableMissing(connection, tableName) {
|
|
|
885
1158
|
export const DUCKDB_ERROR_REASONS = {
|
|
886
1159
|
/** SQL the engine could not parse. */
|
|
887
1160
|
sqlParseError: 'sql_parse_error',
|
|
888
|
-
/**
|
|
1161
|
+
/**
|
|
1162
|
+
* A write the engine refused, or file access its configuration denies —
|
|
1163
|
+
* never the operating system refusing a file, which is an I/O fault.
|
|
1164
|
+
*/
|
|
889
1165
|
sqlReadOnly: 'sql_read_only',
|
|
890
1166
|
/** A gated SELECT that prepared cleanly and then failed on the staged data. */
|
|
891
1167
|
sqlExecutionError: 'sql_execution_error',
|
|
@@ -911,6 +1187,18 @@ const DUCKDB_ERROR_RECOVERY = {
|
|
|
911
1187
|
* merely quotes a data-error class mid-sentence is still an engine fault.
|
|
912
1188
|
*/
|
|
913
1189
|
const DUCKDB_EXECUTION_ERROR_PREFIX = /^(?:Conversion|Invalid Input|Out of Range) Error\b/;
|
|
1190
|
+
/**
|
|
1191
|
+
* DuckDB's refusals, anchored on the class prefix the engine gives them:
|
|
1192
|
+
* `Permission Error` (file access with `enable_external_access` off), and a
|
|
1193
|
+
* write attempted in a read-only transaction (`TransactionContext Error: …
|
|
1194
|
+
* read-only mode`) or against a read-only database (`Invalid Input Error: …
|
|
1195
|
+
* attached in read-only mode!`). An `IO Error` never matches, whatever its
|
|
1196
|
+
* text: `Permission denied` and `Read-only file system` there are the
|
|
1197
|
+
* operating system refusing a file, an engine fault the caller cannot act on.
|
|
1198
|
+
*/
|
|
1199
|
+
const DUCKDB_REFUSAL = /^(?:Permission Error\b|(?:TransactionContext|Invalid Input) Error: .*\bread-only mode\b)/;
|
|
1200
|
+
/** DuckDB's parser class; a quoted file name mentioning "syntax" is not one. */
|
|
1201
|
+
const DUCKDB_PARSER_ERROR_PREFIX = /^Parser Error\b/;
|
|
914
1202
|
/**
|
|
915
1203
|
* Map a DuckDB-thrown error to a framework error class. Classification is for
|
|
916
1204
|
* raw engine errors only — an already-structured `McpError` passes through
|
|
@@ -919,31 +1207,35 @@ const DUCKDB_EXECUTION_ERROR_PREFIX = /^(?:Conversion|Invalid Input|Out of Range
|
|
|
919
1207
|
* validations) must keep their code and `data.reason` instead of being
|
|
920
1208
|
* reclassified as `DatabaseError`.
|
|
921
1209
|
*
|
|
922
|
-
* The three caller-side classes are `ValidationError
|
|
923
|
-
* internal, out-of-memory,
|
|
924
|
-
*
|
|
925
|
-
* must remain.
|
|
1210
|
+
* The three caller-side classes are `ValidationError`, each matched on the
|
|
1211
|
+
* engine's own class prefix; everything else — I/O, internal, out-of-memory,
|
|
1212
|
+
* transaction conflict, interrupt — stays `DatabaseError`, which is what an
|
|
1213
|
+
* export or Parquet round-trip failing on the filesystem must remain (#565).
|
|
1214
|
+
* Every `redact` path in the message becomes `[path]`; the raw engine error,
|
|
1215
|
+
* paths included, stays on `cause`.
|
|
926
1216
|
* @internal Exported for unit testing.
|
|
927
1217
|
*/
|
|
928
|
-
export function classifyDuckdbError(err) {
|
|
1218
|
+
export function classifyDuckdbError(err, redact = []) {
|
|
929
1219
|
if (err instanceof McpError)
|
|
930
1220
|
return err;
|
|
931
1221
|
if (err instanceof Error) {
|
|
932
|
-
const msg = err.message;
|
|
933
|
-
// Checked
|
|
934
|
-
//
|
|
1222
|
+
const msg = redactPaths(err.message, redact);
|
|
1223
|
+
// Checked before the execution prefix: a write refused by a read-only
|
|
1224
|
+
// database is phrased as an `Invalid Input Error`.
|
|
1225
|
+
if (DUCKDB_REFUSAL.test(msg)) {
|
|
1226
|
+
return engineFailure(DUCKDB_ERROR_REASONS.sqlReadOnly, `Canvas SQL rejected: ${msg}`, err);
|
|
1227
|
+
}
|
|
935
1228
|
if (DUCKDB_EXECUTION_ERROR_PREFIX.test(msg)) {
|
|
936
1229
|
return engineFailure(DUCKDB_ERROR_REASONS.sqlExecutionError, `Canvas query failed: ${msg}`, err);
|
|
937
1230
|
}
|
|
938
|
-
if (
|
|
1231
|
+
if (DUCKDB_PARSER_ERROR_PREFIX.test(msg)) {
|
|
939
1232
|
return engineFailure(DUCKDB_ERROR_REASONS.sqlParseError, `Canvas SQL rejected: ${msg}`, err);
|
|
940
1233
|
}
|
|
941
|
-
if (/permission|read.?only/i.test(msg)) {
|
|
942
|
-
return engineFailure(DUCKDB_ERROR_REASONS.sqlReadOnly, `Canvas SQL rejected: ${msg}`, err);
|
|
943
|
-
}
|
|
944
1234
|
return databaseError(msg, undefined, { cause: err });
|
|
945
1235
|
}
|
|
946
|
-
return databaseError('DuckDB threw a non-Error value.', {
|
|
1236
|
+
return databaseError('DuckDB threw a non-Error value.', {
|
|
1237
|
+
value: redactPaths(String(err), redact),
|
|
1238
|
+
});
|
|
947
1239
|
}
|
|
948
1240
|
/** A caller-side engine failure: reason, contract recovery, and the engine error chained. */
|
|
949
1241
|
function engineFailure(reason, message, cause) {
|