@malloy-publisher/server 0.0.195 → 0.0.196-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server.mjs +40 -22
- package/package.json +2 -2
- package/src/service/model.ts +2 -2
- package/src/service/project_store.spec.ts +3 -1
- package/src/service/project_store.ts +8 -4
- package/src/storage/StorageManager.ts +45 -10
- package/src/storage/ducklake/DuckLakeManifestStore.ts +20 -11
- package/tests/harness/mcp_test_setup.ts +11 -23
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +1 -1
- package/tests/integration/mcp/mcp_resource.integration.spec.ts +13 -12
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +1 -1
package/dist/server.mjs
CHANGED
|
@@ -225011,7 +225011,7 @@ class Mutex {
|
|
|
225011
225011
|
}
|
|
225012
225012
|
|
|
225013
225013
|
// src/service/project_store.ts
|
|
225014
|
-
import
|
|
225014
|
+
import crypto4 from "crypto";
|
|
225015
225015
|
import * as fs7 from "fs";
|
|
225016
225016
|
import * as path9 from "path";
|
|
225017
225017
|
|
|
@@ -229253,6 +229253,9 @@ function registerHealthEndpoints(app) {
|
|
|
229253
229253
|
});
|
|
229254
229254
|
}
|
|
229255
229255
|
|
|
229256
|
+
// src/storage/StorageManager.ts
|
|
229257
|
+
import * as crypto3 from "crypto";
|
|
229258
|
+
|
|
229256
229259
|
// src/storage/duckdb/DuckDBConnection.ts
|
|
229257
229260
|
import duckdb from "duckdb";
|
|
229258
229261
|
import * as path5 from "path";
|
|
@@ -230168,15 +230171,17 @@ async function dropAllTables(db) {
|
|
|
230168
230171
|
class DuckLakeManifestStore {
|
|
230169
230172
|
db;
|
|
230170
230173
|
table;
|
|
230171
|
-
|
|
230174
|
+
projectName;
|
|
230175
|
+
constructor(db, catalogName, projectName) {
|
|
230172
230176
|
this.db = db;
|
|
230173
230177
|
this.table = `${catalogName}.build_manifests`;
|
|
230178
|
+
this.projectName = projectName;
|
|
230174
230179
|
}
|
|
230175
230180
|
async bootstrapSchema() {
|
|
230176
230181
|
await this.db.run(`
|
|
230177
230182
|
CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
230178
230183
|
id VARCHAR,
|
|
230179
|
-
|
|
230184
|
+
project_name VARCHAR NOT NULL,
|
|
230180
230185
|
package_name VARCHAR NOT NULL,
|
|
230181
230186
|
build_id VARCHAR NOT NULL,
|
|
230182
230187
|
table_name VARCHAR NOT NULL,
|
|
@@ -230188,8 +230193,8 @@ class DuckLakeManifestStore {
|
|
|
230188
230193
|
`);
|
|
230189
230194
|
logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
|
|
230190
230195
|
}
|
|
230191
|
-
async getManifest(
|
|
230192
|
-
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE
|
|
230196
|
+
async getManifest(_projectId, packageName) {
|
|
230197
|
+
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.projectName, packageName]);
|
|
230193
230198
|
const manifest = { entries: {}, strict: false };
|
|
230194
230199
|
for (const row of rows) {
|
|
230195
230200
|
const buildId = row.build_id;
|
|
@@ -230201,13 +230206,13 @@ class DuckLakeManifestStore {
|
|
|
230201
230206
|
}
|
|
230202
230207
|
return manifest;
|
|
230203
230208
|
}
|
|
230204
|
-
async writeEntry(
|
|
230209
|
+
async writeEntry(_projectId, packageName, buildId, tableName, sourceName, connectionName) {
|
|
230205
230210
|
const now = new Date().toISOString();
|
|
230206
230211
|
const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
230207
|
-
await this.db.run(`INSERT INTO ${this.table} (id,
|
|
230212
|
+
await this.db.run(`INSERT INTO ${this.table} (id, project_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
230208
230213
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
|
|
230209
230214
|
id,
|
|
230210
|
-
|
|
230215
|
+
this.projectName,
|
|
230211
230216
|
packageName,
|
|
230212
230217
|
buildId,
|
|
230213
230218
|
tableName,
|
|
@@ -230220,14 +230225,14 @@ class DuckLakeManifestStore {
|
|
|
230220
230225
|
async deleteEntry(id) {
|
|
230221
230226
|
await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
|
|
230222
230227
|
}
|
|
230223
|
-
async listEntries(
|
|
230224
|
-
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE
|
|
230228
|
+
async listEntries(_projectId, packageName) {
|
|
230229
|
+
const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.projectName, packageName]);
|
|
230225
230230
|
return rows.map(this.mapToEntry);
|
|
230226
230231
|
}
|
|
230227
230232
|
mapToEntry(row) {
|
|
230228
230233
|
return {
|
|
230229
230234
|
id: row.id,
|
|
230230
|
-
projectId: row.
|
|
230235
|
+
projectId: row.project_name,
|
|
230231
230236
|
packageName: row.package_name,
|
|
230232
230237
|
buildId: row.build_id,
|
|
230233
230238
|
tableName: row.table_name,
|
|
@@ -230243,6 +230248,13 @@ class DuckLakeManifestStore {
|
|
|
230243
230248
|
function escapeSQL2(value) {
|
|
230244
230249
|
return value.replace(/'/g, "''");
|
|
230245
230250
|
}
|
|
230251
|
+
function configKey(c) {
|
|
230252
|
+
return `${c.catalogUrl}|${c.dataPath}`;
|
|
230253
|
+
}
|
|
230254
|
+
function catalogNameForConfig(c) {
|
|
230255
|
+
const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
|
|
230256
|
+
return `manifest_lake_${hash}`;
|
|
230257
|
+
}
|
|
230246
230258
|
|
|
230247
230259
|
class StorageManager {
|
|
230248
230260
|
connection = null;
|
|
@@ -230250,7 +230262,7 @@ class StorageManager {
|
|
|
230250
230262
|
repository = null;
|
|
230251
230263
|
defaultManifestStore = null;
|
|
230252
230264
|
projectManifestStores = new Map;
|
|
230253
|
-
attachedCatalogs = new
|
|
230265
|
+
attachedCatalogs = new Map;
|
|
230254
230266
|
config;
|
|
230255
230267
|
constructor(config) {
|
|
230256
230268
|
this.config = config;
|
|
@@ -230283,19 +230295,23 @@ class StorageManager {
|
|
|
230283
230295
|
this.repository = new DuckDBRepository(connection);
|
|
230284
230296
|
this.defaultManifestStore = new DuckDBManifestStore(this.repository);
|
|
230285
230297
|
}
|
|
230286
|
-
async initializeDuckLakeForProject(projectId, config) {
|
|
230298
|
+
async initializeDuckLakeForProject(projectId, projectName, config) {
|
|
230287
230299
|
if (!this.duckDbConnection) {
|
|
230288
230300
|
throw new Error("Storage not initialized. Call initialize() first.");
|
|
230289
230301
|
}
|
|
230290
|
-
const
|
|
230291
|
-
|
|
230302
|
+
const key = configKey(config);
|
|
230303
|
+
let catalogName = this.attachedCatalogs.get(key);
|
|
230304
|
+
if (!catalogName) {
|
|
230305
|
+
catalogName = catalogNameForConfig(config);
|
|
230292
230306
|
await this.attachDuckLakeCatalog(config, catalogName);
|
|
230307
|
+
this.attachedCatalogs.set(key, catalogName);
|
|
230293
230308
|
}
|
|
230294
|
-
const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName);
|
|
230309
|
+
const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, projectName);
|
|
230295
230310
|
await store.bootstrapSchema();
|
|
230296
230311
|
this.projectManifestStores.set(projectId, store);
|
|
230297
230312
|
logger.info("DuckLake manifest store initialized for project", {
|
|
230298
230313
|
projectId,
|
|
230314
|
+
projectName,
|
|
230299
230315
|
catalogName
|
|
230300
230316
|
});
|
|
230301
230317
|
}
|
|
@@ -230310,14 +230326,16 @@ class StorageManager {
|
|
|
230310
230326
|
const escapedDataPath = escapeSQL2(config.dataPath);
|
|
230311
230327
|
const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
|
|
230312
230328
|
let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
|
|
230313
|
-
const attachOpts = [
|
|
230329
|
+
const attachOpts = [
|
|
230330
|
+
`DATA_PATH '${escapedDataPath}'`,
|
|
230331
|
+
"DATA_INLINING_ROW_LIMIT 100000"
|
|
230332
|
+
];
|
|
230314
230333
|
if (isCloudStorage) {
|
|
230315
230334
|
attachOpts.push("OVERRIDE_DATA_PATH true");
|
|
230316
230335
|
}
|
|
230317
230336
|
attachCmd += ` (${attachOpts.join(", ")});`;
|
|
230318
230337
|
logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
|
|
230319
230338
|
await connection.run(attachCmd);
|
|
230320
|
-
this.attachedCatalogs.add(catalogName);
|
|
230321
230339
|
}
|
|
230322
230340
|
getRepository() {
|
|
230323
230341
|
if (!this.repository) {
|
|
@@ -230401,8 +230419,8 @@ import {
|
|
|
230401
230419
|
MalloySQLParser,
|
|
230402
230420
|
MalloySQLStatementType
|
|
230403
230421
|
} from "@malloydata/malloy-sql";
|
|
230404
|
-
import { createRequire as createRequire2 } from "module";
|
|
230405
230422
|
import * as fs4 from "fs/promises";
|
|
230423
|
+
import { createRequire as createRequire2 } from "module";
|
|
230406
230424
|
import * as path6 from "path";
|
|
230407
230425
|
|
|
230408
230426
|
// src/data_styles.ts
|
|
@@ -231032,7 +231050,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
|
|
|
231032
231050
|
}
|
|
231033
231051
|
const modelURL = new URL(`file://${fullModelPath}`);
|
|
231034
231052
|
const baseUrl = new URL(".", modelURL);
|
|
231035
|
-
const importBaseURL =
|
|
231053
|
+
const importBaseURL = baseUrl;
|
|
231036
231054
|
const urlReader = new HackyDataStylesAccumulator(URL_READER);
|
|
231037
231055
|
const runtime = new Runtime({
|
|
231038
231056
|
urlReader,
|
|
@@ -232156,7 +232174,7 @@ class ProjectStore {
|
|
|
232156
232174
|
}
|
|
232157
232175
|
const materializationStorage = project.metadata?.materializationStorage;
|
|
232158
232176
|
if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
|
|
232159
|
-
await this.storageManager.initializeDuckLakeForProject(dbProject.id, {
|
|
232177
|
+
await this.storageManager.initializeDuckLakeForProject(dbProject.id, dbProject.name, {
|
|
232160
232178
|
catalogUrl: materializationStorage.catalogUrl,
|
|
232161
232179
|
dataPath: materializationStorage.dataPath
|
|
232162
232180
|
});
|
|
@@ -232589,7 +232607,7 @@ class ProjectStore {
|
|
|
232589
232607
|
});
|
|
232590
232608
|
}
|
|
232591
232609
|
for (const [groupedLocation, packagesForLocation] of locationGroups) {
|
|
232592
|
-
const locationHash =
|
|
232610
|
+
const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
|
|
232593
232611
|
const tempDownloadPath = `${absoluteTargetPath}/.temp_${locationHash}`;
|
|
232594
232612
|
await fs7.promises.mkdir(tempDownloadPath, { recursive: true });
|
|
232595
232613
|
logger.info(`Created temporary directory: ${tempDownloadPath}`);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@malloy-publisher/server",
|
|
3
3
|
"description": "Malloy Publisher Server",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.196-dev",
|
|
5
5
|
"main": "dist/server.mjs",
|
|
6
6
|
"bin": {
|
|
7
7
|
"malloy-publisher": "dist/server.mjs"
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"scripts": {
|
|
17
17
|
"test": "bun run test:unit && bun run test:integration",
|
|
18
18
|
"test:unit": "bun test --timeout 100000 src",
|
|
19
|
-
"test:integration": "bun test --timeout
|
|
19
|
+
"test:integration": "bun test --timeout 200000 tests --max-workers=1",
|
|
20
20
|
"build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
|
|
21
21
|
"build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
|
|
22
22
|
"start": "NODE_ENV=production bun run ./dist/server.mjs",
|
package/src/service/model.ts
CHANGED
|
@@ -22,10 +22,10 @@ import {
|
|
|
22
22
|
MalloySQLParser,
|
|
23
23
|
MalloySQLStatementType,
|
|
24
24
|
} from "@malloydata/malloy-sql";
|
|
25
|
-
import { createRequire } from "module";
|
|
26
25
|
import { DataStyles } from "@malloydata/render";
|
|
27
26
|
import { metrics } from "@opentelemetry/api";
|
|
28
27
|
import * as fs from "fs/promises";
|
|
28
|
+
import { createRequire } from "module";
|
|
29
29
|
import * as path from "path";
|
|
30
30
|
import { components } from "../api";
|
|
31
31
|
import {
|
|
@@ -703,7 +703,7 @@ export class Model {
|
|
|
703
703
|
|
|
704
704
|
const modelURL = new URL(`file://${fullModelPath}`);
|
|
705
705
|
const baseUrl = new URL(".", modelURL);
|
|
706
|
-
const importBaseURL =
|
|
706
|
+
const importBaseURL = baseUrl;
|
|
707
707
|
const urlReader = new HackyDataStylesAccumulator(URL_READER);
|
|
708
708
|
|
|
709
709
|
// Request runtimes borrow the cached package MalloyConfig. The package
|
|
@@ -12,6 +12,7 @@ type MockData = Record<string, unknown>;
|
|
|
12
12
|
|
|
13
13
|
const initializeDuckLakeCalls: Array<{
|
|
14
14
|
projectId: string;
|
|
15
|
+
projectName: string;
|
|
15
16
|
config: { catalogUrl: string; dataPath: string };
|
|
16
17
|
}> = [];
|
|
17
18
|
|
|
@@ -24,9 +25,10 @@ mock.module("../storage/StorageManager", () => {
|
|
|
24
25
|
|
|
25
26
|
async initializeDuckLakeForProject(
|
|
26
27
|
projectId: string,
|
|
28
|
+
projectName: string,
|
|
27
29
|
config: { catalogUrl: string; dataPath: string },
|
|
28
30
|
): Promise<void> {
|
|
29
|
-
initializeDuckLakeCalls.push({ projectId, config });
|
|
31
|
+
initializeDuckLakeCalls.push({ projectId, projectName, config });
|
|
30
32
|
}
|
|
31
33
|
|
|
32
34
|
getRepository() {
|
|
@@ -359,10 +359,14 @@ export class ProjectStore {
|
|
|
359
359
|
materializationStorage?.catalogUrl &&
|
|
360
360
|
materializationStorage?.dataPath
|
|
361
361
|
) {
|
|
362
|
-
await this.storageManager.initializeDuckLakeForProject(
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
362
|
+
await this.storageManager.initializeDuckLakeForProject(
|
|
363
|
+
dbProject.id,
|
|
364
|
+
dbProject.name,
|
|
365
|
+
{
|
|
366
|
+
catalogUrl: materializationStorage.catalogUrl,
|
|
367
|
+
dataPath: materializationStorage.dataPath,
|
|
368
|
+
},
|
|
369
|
+
);
|
|
366
370
|
}
|
|
367
371
|
|
|
368
372
|
return dbProject;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import * as crypto from "crypto";
|
|
1
2
|
import { logger } from "../logger";
|
|
2
3
|
import {
|
|
3
4
|
DatabaseConnection,
|
|
@@ -42,6 +43,19 @@ function escapeSQL(value: string): string {
|
|
|
42
43
|
return value.replace(/'/g, "''");
|
|
43
44
|
}
|
|
44
45
|
|
|
46
|
+
function configKey(c: DuckLakeManifestConfig): string {
|
|
47
|
+
return `${c.catalogUrl}|${c.dataPath}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function catalogNameForConfig(c: DuckLakeManifestConfig): string {
|
|
51
|
+
const hash = crypto
|
|
52
|
+
.createHash("sha256")
|
|
53
|
+
.update(configKey(c))
|
|
54
|
+
.digest("hex")
|
|
55
|
+
.slice(0, 8);
|
|
56
|
+
return `manifest_lake_${hash}`;
|
|
57
|
+
}
|
|
58
|
+
|
|
45
59
|
/**
|
|
46
60
|
* Manages the storage backend (DuckDB, Postgres, etc.) and per-project
|
|
47
61
|
* manifest stores. Projects without `materializationStorage` config use
|
|
@@ -57,8 +71,12 @@ export class StorageManager {
|
|
|
57
71
|
/** Per-project DuckLake manifest stores, keyed by projectId. */
|
|
58
72
|
private projectManifestStores = new Map<string, ManifestStore>();
|
|
59
73
|
|
|
60
|
-
/**
|
|
61
|
-
|
|
74
|
+
/**
|
|
75
|
+
* Tracks attached DuckLake catalogs as `configKey -> catalogName`. Each
|
|
76
|
+
* unique materializationStorage config gets its own ATTACHment under a
|
|
77
|
+
* deterministic catalog name, so multiple configs can coexist on one worker.
|
|
78
|
+
*/
|
|
79
|
+
private attachedCatalogs = new Map<string, string>();
|
|
62
80
|
|
|
63
81
|
private config: StorageConfig;
|
|
64
82
|
|
|
@@ -105,32 +123,44 @@ export class StorageManager {
|
|
|
105
123
|
|
|
106
124
|
/**
|
|
107
125
|
* Lazily initializes a DuckLake manifest store for a project.
|
|
108
|
-
*
|
|
109
|
-
*
|
|
126
|
+
*
|
|
127
|
+
* One shared catalog per materializationStorage config: every project
|
|
128
|
+
* pointing at the same (catalogUrl, dataPath) shares one `build_manifests`
|
|
129
|
+
* table inside it, partitioned by `project_id` (set to the project's name
|
|
130
|
+
* so it's stable across worker replicas — required for cross-pod manifest
|
|
131
|
+
* visibility in orchestrated mode). Different configs (e.g. different
|
|
132
|
+
* orgs) attach as separate catalogs under distinct deterministic aliases.
|
|
110
133
|
*/
|
|
111
134
|
async initializeDuckLakeForProject(
|
|
112
135
|
projectId: string,
|
|
136
|
+
projectName: string,
|
|
113
137
|
config: DuckLakeManifestConfig,
|
|
114
138
|
): Promise<void> {
|
|
115
139
|
if (!this.duckDbConnection) {
|
|
116
140
|
throw new Error("Storage not initialized. Call initialize() first.");
|
|
117
141
|
}
|
|
118
142
|
|
|
119
|
-
const
|
|
120
|
-
|
|
121
|
-
if (!
|
|
143
|
+
const key = configKey(config);
|
|
144
|
+
let catalogName = this.attachedCatalogs.get(key);
|
|
145
|
+
if (!catalogName) {
|
|
146
|
+
// Catalog name derived from the config so multiple configs can coexist as
|
|
147
|
+
// separate ATTACHments without colliding on the name.
|
|
148
|
+
catalogName = catalogNameForConfig(config);
|
|
122
149
|
await this.attachDuckLakeCatalog(config, catalogName);
|
|
150
|
+
this.attachedCatalogs.set(key, catalogName);
|
|
123
151
|
}
|
|
124
152
|
|
|
125
153
|
const store = new DuckLakeManifestStore(
|
|
126
154
|
this.duckDbConnection,
|
|
127
155
|
catalogName,
|
|
156
|
+
projectName,
|
|
128
157
|
);
|
|
129
158
|
await store.bootstrapSchema();
|
|
130
159
|
|
|
131
160
|
this.projectManifestStores.set(projectId, store);
|
|
132
161
|
logger.info("DuckLake manifest store initialized for project", {
|
|
133
162
|
projectId,
|
|
163
|
+
projectName,
|
|
134
164
|
catalogName,
|
|
135
165
|
});
|
|
136
166
|
}
|
|
@@ -155,7 +185,14 @@ export class StorageManager {
|
|
|
155
185
|
config.dataPath.startsWith("s3://");
|
|
156
186
|
|
|
157
187
|
let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
|
|
158
|
-
const attachOpts: string[] = [
|
|
188
|
+
const attachOpts: string[] = [
|
|
189
|
+
`DATA_PATH '${escapedDataPath}'`,
|
|
190
|
+
// The manifest table is small relational metadata (one row per build).
|
|
191
|
+
// Set a high inlining limit so writes always land transactionally in
|
|
192
|
+
// the postgres catalog rather than as parquet files in object storage,
|
|
193
|
+
// sidestepping object-storage auth issues entirely for this path.
|
|
194
|
+
"DATA_INLINING_ROW_LIMIT 100000",
|
|
195
|
+
];
|
|
159
196
|
if (isCloudStorage) {
|
|
160
197
|
attachOpts.push("OVERRIDE_DATA_PATH true");
|
|
161
198
|
}
|
|
@@ -163,8 +200,6 @@ export class StorageManager {
|
|
|
163
200
|
|
|
164
201
|
logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
|
|
165
202
|
await connection.run(attachCmd);
|
|
166
|
-
|
|
167
|
-
this.attachedCatalogs.add(catalogName);
|
|
168
203
|
}
|
|
169
204
|
|
|
170
205
|
getRepository(): ResourceRepository {
|
|
@@ -31,23 +31,32 @@ import { DuckDBConnection } from "../duckdb/DuckDBConnection";
|
|
|
31
31
|
*/
|
|
32
32
|
export class DuckLakeManifestStore implements ManifestStore {
|
|
33
33
|
private readonly table: string;
|
|
34
|
+
private readonly projectName: string;
|
|
34
35
|
|
|
35
36
|
constructor(
|
|
36
37
|
private db: DuckDBConnection,
|
|
37
38
|
catalogName: string,
|
|
39
|
+
projectName: string,
|
|
38
40
|
) {
|
|
39
41
|
this.table = `${catalogName}.build_manifests`;
|
|
42
|
+
this.projectName = projectName;
|
|
40
43
|
}
|
|
41
44
|
|
|
42
45
|
/**
|
|
43
46
|
* Idempotently creates the `build_manifests` table and indices in the
|
|
44
47
|
* DuckLake catalog. Safe to call from every worker on startup.
|
|
48
|
+
*
|
|
49
|
+
* Note: this table uses `project_name` for the partition column, unlike
|
|
50
|
+
* the local DuckDB `build_manifests` table (in `storage/duckdb/schema.ts`)
|
|
51
|
+
* which uses `project_id` and FK-references `projects.id`. The two stores
|
|
52
|
+
* partition by different identifiers — local random id vs cross-pod-stable
|
|
53
|
+
* project name — so they intentionally diverge here.
|
|
45
54
|
*/
|
|
46
55
|
async bootstrapSchema(): Promise<void> {
|
|
47
56
|
await this.db.run(`
|
|
48
57
|
CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
49
58
|
id VARCHAR,
|
|
50
|
-
|
|
59
|
+
project_name VARCHAR NOT NULL,
|
|
51
60
|
package_name VARCHAR NOT NULL,
|
|
52
61
|
build_id VARCHAR NOT NULL,
|
|
53
62
|
table_name VARCHAR NOT NULL,
|
|
@@ -61,12 +70,12 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
61
70
|
}
|
|
62
71
|
|
|
63
72
|
async getManifest(
|
|
64
|
-
|
|
73
|
+
_projectId: string,
|
|
65
74
|
packageName: string,
|
|
66
75
|
): Promise<BuildManifest> {
|
|
67
76
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
68
|
-
`SELECT * FROM ${this.table} WHERE
|
|
69
|
-
[
|
|
77
|
+
`SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`,
|
|
78
|
+
[this.projectName, packageName],
|
|
70
79
|
);
|
|
71
80
|
const manifest: BuildManifest = { entries: {}, strict: false };
|
|
72
81
|
for (const row of rows) {
|
|
@@ -88,7 +97,7 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
88
97
|
* {@link getManifest} deduplicates by build_id keeping the newest row.
|
|
89
98
|
*/
|
|
90
99
|
async writeEntry(
|
|
91
|
-
|
|
100
|
+
_projectId: string,
|
|
92
101
|
packageName: string,
|
|
93
102
|
buildId: string,
|
|
94
103
|
tableName: string,
|
|
@@ -99,11 +108,11 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
99
108
|
const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
100
109
|
|
|
101
110
|
await this.db.run(
|
|
102
|
-
`INSERT INTO ${this.table} (id,
|
|
111
|
+
`INSERT INTO ${this.table} (id, project_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
103
112
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
104
113
|
[
|
|
105
114
|
id,
|
|
106
|
-
|
|
115
|
+
this.projectName,
|
|
107
116
|
packageName,
|
|
108
117
|
buildId,
|
|
109
118
|
tableName,
|
|
@@ -120,12 +129,12 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
120
129
|
}
|
|
121
130
|
|
|
122
131
|
async listEntries(
|
|
123
|
-
|
|
132
|
+
_projectId: string,
|
|
124
133
|
packageName: string,
|
|
125
134
|
): Promise<ManifestEntry[]> {
|
|
126
135
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
127
|
-
`SELECT * FROM ${this.table} WHERE
|
|
128
|
-
[
|
|
136
|
+
`SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`,
|
|
137
|
+
[this.projectName, packageName],
|
|
129
138
|
);
|
|
130
139
|
return rows.map(this.mapToEntry);
|
|
131
140
|
}
|
|
@@ -133,7 +142,7 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
133
142
|
private mapToEntry(row: Record<string, unknown>): ManifestEntry {
|
|
134
143
|
return {
|
|
135
144
|
id: row.id as string,
|
|
136
|
-
projectId: row.
|
|
145
|
+
projectId: row.project_name as string,
|
|
137
146
|
packageName: row.package_name as string,
|
|
138
147
|
buildId: row.build_id as string,
|
|
139
148
|
tableName: row.table_name as string,
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
Request,
|
|
6
6
|
Result,
|
|
7
7
|
} from "@modelcontextprotocol/sdk/types.js";
|
|
8
|
+
import { Mutex } from "async-mutex";
|
|
8
9
|
import http from "http";
|
|
9
10
|
import { AddressInfo } from "net";
|
|
10
11
|
import path from "path";
|
|
@@ -24,37 +25,24 @@ export interface McpE2ETestEnvironment {
|
|
|
24
25
|
originalInitializeStorage: string | undefined;
|
|
25
26
|
}
|
|
26
27
|
|
|
27
|
-
// Counter for unique port assignment per test suite
|
|
28
|
+
// Counter for unique port assignment per test suite (incremented only under e2eSetupMutex)
|
|
28
29
|
let portCounter = 0;
|
|
29
30
|
|
|
30
|
-
//
|
|
31
|
-
|
|
31
|
+
// True mutex: the previous Promise-based lock had a TOCTOU race where two beforeAll hooks
|
|
32
|
+
// could both pass the `if (initializationLock)` check and run setup concurrently, sharing
|
|
33
|
+
// one publisher.db / server singleton and causing flaky listResources and port collisions.
|
|
34
|
+
const e2eSetupMutex = new Mutex();
|
|
32
35
|
|
|
33
36
|
/**
|
|
34
37
|
* Starts the real application server and connects a real MCP client.
|
|
35
38
|
*/
|
|
36
39
|
export async function setupE2ETestEnvironment(): Promise<McpE2ETestEnvironment> {
|
|
37
|
-
|
|
38
|
-
if (initializationLock) {
|
|
40
|
+
return e2eSetupMutex.runExclusive(async () => {
|
|
39
41
|
console.log(
|
|
40
|
-
"[E2E Test Setup]
|
|
42
|
+
"[E2E Test Setup] Acquired setup mutex; starting environment...",
|
|
41
43
|
);
|
|
42
|
-
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
// Create a new lock for this initialization
|
|
46
|
-
let resolveLock: () => void;
|
|
47
|
-
initializationLock = new Promise((resolve) => {
|
|
48
|
-
resolveLock = resolve;
|
|
44
|
+
return setupE2ETestEnvironmentInternal();
|
|
49
45
|
});
|
|
50
|
-
|
|
51
|
-
try {
|
|
52
|
-
return await setupE2ETestEnvironmentInternal();
|
|
53
|
-
} finally {
|
|
54
|
-
// Release the lock
|
|
55
|
-
resolveLock!();
|
|
56
|
-
initializationLock = null;
|
|
57
|
-
}
|
|
58
46
|
}
|
|
59
47
|
|
|
60
48
|
async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment> {
|
|
@@ -129,9 +117,9 @@ async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment>
|
|
|
129
117
|
|
|
130
118
|
// --- Wait for server to be ready (packages downloaded) ---
|
|
131
119
|
// Poll the readiness endpoint to ensure initialization completes before tests run
|
|
132
|
-
//
|
|
120
|
+
// Allow >90s: cold CI can spend most of the budget on clone + package load before markReady().
|
|
133
121
|
console.log("[E2E Test Setup] Waiting for server to be ready...");
|
|
134
|
-
const maxWaitTime =
|
|
122
|
+
const maxWaitTime = 150000; // 150 seconds (INITIALIZE_STORAGE + large samples can be slow)
|
|
135
123
|
const pollInterval = 1000; // Check every second
|
|
136
124
|
const startTime = Date.now();
|
|
137
125
|
let isReady = false;
|
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
} from "../../harness/mcp_test_setup";
|
|
20
20
|
|
|
21
21
|
// --- Test Suite ---
|
|
22
|
-
describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
22
|
+
describe.serial("MCP Tool Handlers (E2E Integration)", () => {
|
|
23
23
|
let env: McpE2ETestEnvironment | null = null;
|
|
24
24
|
let mcpClient: Client;
|
|
25
25
|
|
|
@@ -20,7 +20,8 @@ interface PackageContentEntry {
|
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
// --- Test Suite ---
|
|
23
|
-
|
|
23
|
+
// Serial: one shared MCP client; concurrent it() blocks can interleave StreamableHTTP requests.
|
|
24
|
+
describe.serial("MCP Resource Handlers (E2E Integration)", () => {
|
|
24
25
|
let env: McpE2ETestEnvironment | null = null;
|
|
25
26
|
let mcpClient: Client;
|
|
26
27
|
|
|
@@ -85,17 +86,17 @@ describe("MCP Resource Handlers (E2E Integration)", () => {
|
|
|
85
86
|
expect(faaPackageEntry).toBeDefined();
|
|
86
87
|
expect(faaPackageEntry?.name).toBe("faa");
|
|
87
88
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
expect(
|
|
91
|
-
expect(typeof
|
|
92
|
-
expect(
|
|
93
|
-
expect(
|
|
94
|
-
expect(typeof
|
|
95
|
-
expect(
|
|
96
|
-
expect(typeof
|
|
97
|
-
if (
|
|
98
|
-
expect(typeof
|
|
89
|
+
// Assert shape on a known row (order of listResources is not guaranteed).
|
|
90
|
+
const sample = faaPackageEntry!;
|
|
91
|
+
expect(sample.uri).toBeDefined();
|
|
92
|
+
expect(typeof sample.uri).toBe("string");
|
|
93
|
+
expect(sample.uri).toMatch(/^malloy:\/\//);
|
|
94
|
+
expect(sample.name).toBeDefined();
|
|
95
|
+
expect(typeof sample.name).toBe("string");
|
|
96
|
+
expect(sample.description).toBeDefined();
|
|
97
|
+
expect(typeof sample.description).toBe("string");
|
|
98
|
+
if (sample.mimeType) {
|
|
99
|
+
expect(typeof sample.mimeType).toBe("string");
|
|
99
100
|
}
|
|
100
101
|
},
|
|
101
102
|
{ timeout: 30000 },
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
// --- Test Suite ---
|
|
19
19
|
// Note: These tests assume interaction via a standard MCP client.
|
|
20
20
|
// Tests for raw HTTP edge cases (non-JSON, non-RPC) are omitted based on this assumption.
|
|
21
|
-
describe("MCP Transport Tests (E2E Integration)", () => {
|
|
21
|
+
describe.serial("MCP Transport Tests (E2E Integration)", () => {
|
|
22
22
|
let env: McpE2ETestEnvironment | null = null;
|
|
23
23
|
let mcpClient: Client<Request, Notification, Result>; // Convenience variable
|
|
24
24
|
|