@malloy-publisher/server 0.0.195 → 0.0.196-dev

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/server.mjs CHANGED
@@ -225011,7 +225011,7 @@ class Mutex {
225011
225011
  }
225012
225012
 
225013
225013
  // src/service/project_store.ts
225014
- import crypto3 from "crypto";
225014
+ import crypto4 from "crypto";
225015
225015
  import * as fs7 from "fs";
225016
225016
  import * as path9 from "path";
225017
225017
 
@@ -229253,6 +229253,9 @@ function registerHealthEndpoints(app) {
229253
229253
  });
229254
229254
  }
229255
229255
 
229256
+ // src/storage/StorageManager.ts
229257
+ import * as crypto3 from "crypto";
229258
+
229256
229259
  // src/storage/duckdb/DuckDBConnection.ts
229257
229260
  import duckdb from "duckdb";
229258
229261
  import * as path5 from "path";
@@ -230168,15 +230171,17 @@ async function dropAllTables(db) {
230168
230171
  class DuckLakeManifestStore {
230169
230172
  db;
230170
230173
  table;
230171
- constructor(db, catalogName) {
230174
+ projectName;
230175
+ constructor(db, catalogName, projectName) {
230172
230176
  this.db = db;
230173
230177
  this.table = `${catalogName}.build_manifests`;
230178
+ this.projectName = projectName;
230174
230179
  }
230175
230180
  async bootstrapSchema() {
230176
230181
  await this.db.run(`
230177
230182
  CREATE TABLE IF NOT EXISTS ${this.table} (
230178
230183
  id VARCHAR,
230179
- project_id VARCHAR NOT NULL,
230184
+ project_name VARCHAR NOT NULL,
230180
230185
  package_name VARCHAR NOT NULL,
230181
230186
  build_id VARCHAR NOT NULL,
230182
230187
  table_name VARCHAR NOT NULL,
@@ -230188,8 +230193,8 @@ class DuckLakeManifestStore {
230188
230193
  `);
230189
230194
  logger.info(`DuckLake manifest table bootstrapped: ${this.table}`);
230190
230195
  }
230191
- async getManifest(projectId, packageName) {
230192
- const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`, [projectId, packageName]);
230196
+ async getManifest(_projectId, packageName) {
230197
+ const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.projectName, packageName]);
230193
230198
  const manifest = { entries: {}, strict: false };
230194
230199
  for (const row of rows) {
230195
230200
  const buildId = row.build_id;
@@ -230201,13 +230206,13 @@ class DuckLakeManifestStore {
230201
230206
  }
230202
230207
  return manifest;
230203
230208
  }
230204
- async writeEntry(projectId, packageName, buildId, tableName, sourceName, connectionName) {
230209
+ async writeEntry(_projectId, packageName, buildId, tableName, sourceName, connectionName) {
230205
230210
  const now = new Date().toISOString();
230206
230211
  const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
230207
- await this.db.run(`INSERT INTO ${this.table} (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
230212
+ await this.db.run(`INSERT INTO ${this.table} (id, project_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
230208
230213
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, [
230209
230214
  id,
230210
- projectId,
230215
+ this.projectName,
230211
230216
  packageName,
230212
230217
  buildId,
230213
230218
  tableName,
@@ -230220,14 +230225,14 @@ class DuckLakeManifestStore {
230220
230225
  async deleteEntry(id) {
230221
230226
  await this.db.run(`DELETE FROM ${this.table} WHERE id = ?`, [id]);
230222
230227
  }
230223
- async listEntries(projectId, packageName) {
230224
- const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`, [projectId, packageName]);
230228
+ async listEntries(_projectId, packageName) {
230229
+ const rows = await this.db.all(`SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`, [this.projectName, packageName]);
230225
230230
  return rows.map(this.mapToEntry);
230226
230231
  }
230227
230232
  mapToEntry(row) {
230228
230233
  return {
230229
230234
  id: row.id,
230230
- projectId: row.project_id,
230235
+ projectId: row.project_name,
230231
230236
  packageName: row.package_name,
230232
230237
  buildId: row.build_id,
230233
230238
  tableName: row.table_name,
@@ -230243,6 +230248,13 @@ class DuckLakeManifestStore {
230243
230248
  function escapeSQL2(value) {
230244
230249
  return value.replace(/'/g, "''");
230245
230250
  }
230251
+ function configKey(c) {
230252
+ return `${c.catalogUrl}|${c.dataPath}`;
230253
+ }
230254
+ function catalogNameForConfig(c) {
230255
+ const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
230256
+ return `manifest_lake_${hash}`;
230257
+ }
230246
230258
 
230247
230259
  class StorageManager {
230248
230260
  connection = null;
@@ -230250,7 +230262,7 @@ class StorageManager {
230250
230262
  repository = null;
230251
230263
  defaultManifestStore = null;
230252
230264
  projectManifestStores = new Map;
230253
- attachedCatalogs = new Set;
230265
+ attachedCatalogs = new Map;
230254
230266
  config;
230255
230267
  constructor(config) {
230256
230268
  this.config = config;
@@ -230283,19 +230295,23 @@ class StorageManager {
230283
230295
  this.repository = new DuckDBRepository(connection);
230284
230296
  this.defaultManifestStore = new DuckDBManifestStore(this.repository);
230285
230297
  }
230286
- async initializeDuckLakeForProject(projectId, config) {
230298
+ async initializeDuckLakeForProject(projectId, projectName, config) {
230287
230299
  if (!this.duckDbConnection) {
230288
230300
  throw new Error("Storage not initialized. Call initialize() first.");
230289
230301
  }
230290
- const catalogName = `manifest_lake_${projectId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
230291
- if (!this.attachedCatalogs.has(catalogName)) {
230302
+ const key = configKey(config);
230303
+ let catalogName = this.attachedCatalogs.get(key);
230304
+ if (!catalogName) {
230305
+ catalogName = catalogNameForConfig(config);
230292
230306
  await this.attachDuckLakeCatalog(config, catalogName);
230307
+ this.attachedCatalogs.set(key, catalogName);
230293
230308
  }
230294
- const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName);
230309
+ const store = new DuckLakeManifestStore(this.duckDbConnection, catalogName, projectName);
230295
230310
  await store.bootstrapSchema();
230296
230311
  this.projectManifestStores.set(projectId, store);
230297
230312
  logger.info("DuckLake manifest store initialized for project", {
230298
230313
  projectId,
230314
+ projectName,
230299
230315
  catalogName
230300
230316
  });
230301
230317
  }
@@ -230310,14 +230326,16 @@ class StorageManager {
230310
230326
  const escapedDataPath = escapeSQL2(config.dataPath);
230311
230327
  const isCloudStorage = config.dataPath.startsWith("gs://") || config.dataPath.startsWith("s3://");
230312
230328
  let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
230313
- const attachOpts = [`DATA_PATH '${escapedDataPath}'`];
230329
+ const attachOpts = [
230330
+ `DATA_PATH '${escapedDataPath}'`,
230331
+ "DATA_INLINING_ROW_LIMIT 100000"
230332
+ ];
230314
230333
  if (isCloudStorage) {
230315
230334
  attachOpts.push("OVERRIDE_DATA_PATH true");
230316
230335
  }
230317
230336
  attachCmd += ` (${attachOpts.join(", ")});`;
230318
230337
  logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
230319
230338
  await connection.run(attachCmd);
230320
- this.attachedCatalogs.add(catalogName);
230321
230339
  }
230322
230340
  getRepository() {
230323
230341
  if (!this.repository) {
@@ -230401,8 +230419,8 @@ import {
230401
230419
  MalloySQLParser,
230402
230420
  MalloySQLStatementType
230403
230421
  } from "@malloydata/malloy-sql";
230404
- import { createRequire as createRequire2 } from "module";
230405
230422
  import * as fs4 from "fs/promises";
230423
+ import { createRequire as createRequire2 } from "module";
230406
230424
  import * as path6 from "path";
230407
230425
 
230408
230426
  // src/data_styles.ts
@@ -231032,7 +231050,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
231032
231050
  }
231033
231051
  const modelURL = new URL(`file://${fullModelPath}`);
231034
231052
  const baseUrl = new URL(".", modelURL);
231035
- const importBaseURL = new URL(baseUrl.pathname + "/", "file:");
231053
+ const importBaseURL = baseUrl;
231036
231054
  const urlReader = new HackyDataStylesAccumulator(URL_READER);
231037
231055
  const runtime = new Runtime({
231038
231056
  urlReader,
@@ -232156,7 +232174,7 @@ class ProjectStore {
232156
232174
  }
232157
232175
  const materializationStorage = project.metadata?.materializationStorage;
232158
232176
  if (materializationStorage?.catalogUrl && materializationStorage?.dataPath) {
232159
- await this.storageManager.initializeDuckLakeForProject(dbProject.id, {
232177
+ await this.storageManager.initializeDuckLakeForProject(dbProject.id, dbProject.name, {
232160
232178
  catalogUrl: materializationStorage.catalogUrl,
232161
232179
  dataPath: materializationStorage.dataPath
232162
232180
  });
@@ -232589,7 +232607,7 @@ class ProjectStore {
232589
232607
  });
232590
232608
  }
232591
232609
  for (const [groupedLocation, packagesForLocation] of locationGroups) {
232592
- const locationHash = crypto3.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
232610
+ const locationHash = crypto4.createHash("sha256").update(groupedLocation).digest("hex").substring(0, 16);
232593
232611
  const tempDownloadPath = `${absoluteTargetPath}/.temp_${locationHash}`;
232594
232612
  await fs7.promises.mkdir(tempDownloadPath, { recursive: true });
232595
232613
  logger.info(`Created temporary directory: ${tempDownloadPath}`);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@malloy-publisher/server",
3
3
  "description": "Malloy Publisher Server",
4
- "version": "0.0.195",
4
+ "version": "0.0.196-dev",
5
5
  "main": "dist/server.mjs",
6
6
  "bin": {
7
7
  "malloy-publisher": "dist/server.mjs"
@@ -16,7 +16,7 @@
16
16
  "scripts": {
17
17
  "test": "bun run test:unit && bun run test:integration",
18
18
  "test:unit": "bun test --timeout 100000 src",
19
- "test:integration": "bun test --timeout 100000 tests",
19
+ "test:integration": "bun test --timeout 200000 tests --max-workers=1",
20
20
  "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
21
21
  "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
22
22
  "start": "NODE_ENV=production bun run ./dist/server.mjs",
@@ -22,10 +22,10 @@ import {
22
22
  MalloySQLParser,
23
23
  MalloySQLStatementType,
24
24
  } from "@malloydata/malloy-sql";
25
- import { createRequire } from "module";
26
25
  import { DataStyles } from "@malloydata/render";
27
26
  import { metrics } from "@opentelemetry/api";
28
27
  import * as fs from "fs/promises";
28
+ import { createRequire } from "module";
29
29
  import * as path from "path";
30
30
  import { components } from "../api";
31
31
  import {
@@ -703,7 +703,7 @@ export class Model {
703
703
 
704
704
  const modelURL = new URL(`file://${fullModelPath}`);
705
705
  const baseUrl = new URL(".", modelURL);
706
- const importBaseURL = new URL(baseUrl.pathname + "/", "file:");
706
+ const importBaseURL = baseUrl;
707
707
  const urlReader = new HackyDataStylesAccumulator(URL_READER);
708
708
 
709
709
  // Request runtimes borrow the cached package MalloyConfig. The package
@@ -12,6 +12,7 @@ type MockData = Record<string, unknown>;
12
12
 
13
13
  const initializeDuckLakeCalls: Array<{
14
14
  projectId: string;
15
+ projectName: string;
15
16
  config: { catalogUrl: string; dataPath: string };
16
17
  }> = [];
17
18
 
@@ -24,9 +25,10 @@ mock.module("../storage/StorageManager", () => {
24
25
 
25
26
  async initializeDuckLakeForProject(
26
27
  projectId: string,
28
+ projectName: string,
27
29
  config: { catalogUrl: string; dataPath: string },
28
30
  ): Promise<void> {
29
- initializeDuckLakeCalls.push({ projectId, config });
31
+ initializeDuckLakeCalls.push({ projectId, projectName, config });
30
32
  }
31
33
 
32
34
  getRepository() {
@@ -359,10 +359,14 @@ export class ProjectStore {
359
359
  materializationStorage?.catalogUrl &&
360
360
  materializationStorage?.dataPath
361
361
  ) {
362
- await this.storageManager.initializeDuckLakeForProject(dbProject.id, {
363
- catalogUrl: materializationStorage.catalogUrl,
364
- dataPath: materializationStorage.dataPath,
365
- });
362
+ await this.storageManager.initializeDuckLakeForProject(
363
+ dbProject.id,
364
+ dbProject.name,
365
+ {
366
+ catalogUrl: materializationStorage.catalogUrl,
367
+ dataPath: materializationStorage.dataPath,
368
+ },
369
+ );
366
370
  }
367
371
 
368
372
  return dbProject;
@@ -1,3 +1,4 @@
1
+ import * as crypto from "crypto";
1
2
  import { logger } from "../logger";
2
3
  import {
3
4
  DatabaseConnection,
@@ -42,6 +43,19 @@ function escapeSQL(value: string): string {
42
43
  return value.replace(/'/g, "''");
43
44
  }
44
45
 
46
+ function configKey(c: DuckLakeManifestConfig): string {
47
+ return `${c.catalogUrl}|${c.dataPath}`;
48
+ }
49
+
50
+ function catalogNameForConfig(c: DuckLakeManifestConfig): string {
51
+ const hash = crypto
52
+ .createHash("sha256")
53
+ .update(configKey(c))
54
+ .digest("hex")
55
+ .slice(0, 8);
56
+ return `manifest_lake_${hash}`;
57
+ }
58
+
45
59
  /**
46
60
  * Manages the storage backend (DuckDB, Postgres, etc.) and per-project
47
61
  * manifest stores. Projects without `materializationStorage` config use
@@ -57,8 +71,12 @@ export class StorageManager {
57
71
  /** Per-project DuckLake manifest stores, keyed by projectId. */
58
72
  private projectManifestStores = new Map<string, ManifestStore>();
59
73
 
60
- /** Tracks which catalogs have been attached to avoid duplicate ATTACHes. */
61
- private attachedCatalogs = new Set<string>();
74
+ /**
75
+ * Tracks attached DuckLake catalogs as `configKey -> catalogName`. Each
76
+ * unique materializationStorage config gets its own ATTACHment under a
77
+ * deterministic catalog name, so multiple configs can coexist on one worker.
78
+ */
79
+ private attachedCatalogs = new Map<string, string>();
62
80
 
63
81
  private config: StorageConfig;
64
82
 
@@ -105,32 +123,44 @@ export class StorageManager {
105
123
 
106
124
  /**
107
125
  * Lazily initializes a DuckLake manifest store for a project.
108
- * The DuckLake catalog is attached to the shared DuckDB connection
109
- * and persists for the lifetime of the process.
126
+ *
127
+ * One shared catalog per materializationStorage config: every project
128
+ * pointing at the same (catalogUrl, dataPath) shares one `build_manifests`
129
+ * table inside it, partitioned by `project_id` (set to the project's name
130
+ * so it's stable across worker replicas — required for cross-pod manifest
131
+ * visibility in orchestrated mode). Different configs (e.g. different
132
+ * orgs) attach as separate catalogs under distinct deterministic aliases.
110
133
  */
111
134
  async initializeDuckLakeForProject(
112
135
  projectId: string,
136
+ projectName: string,
113
137
  config: DuckLakeManifestConfig,
114
138
  ): Promise<void> {
115
139
  if (!this.duckDbConnection) {
116
140
  throw new Error("Storage not initialized. Call initialize() first.");
117
141
  }
118
142
 
119
- const catalogName = `manifest_lake_${projectId.replace(/[^a-zA-Z0-9_]/g, "_")}`;
120
-
121
- if (!this.attachedCatalogs.has(catalogName)) {
143
+ const key = configKey(config);
144
+ let catalogName = this.attachedCatalogs.get(key);
145
+ if (!catalogName) {
146
+ // Catalog name derived from the config so multiple configs can coexist as
147
+ // separate ATTACHments without colliding on the name.
148
+ catalogName = catalogNameForConfig(config);
122
149
  await this.attachDuckLakeCatalog(config, catalogName);
150
+ this.attachedCatalogs.set(key, catalogName);
123
151
  }
124
152
 
125
153
  const store = new DuckLakeManifestStore(
126
154
  this.duckDbConnection,
127
155
  catalogName,
156
+ projectName,
128
157
  );
129
158
  await store.bootstrapSchema();
130
159
 
131
160
  this.projectManifestStores.set(projectId, store);
132
161
  logger.info("DuckLake manifest store initialized for project", {
133
162
  projectId,
163
+ projectName,
134
164
  catalogName,
135
165
  });
136
166
  }
@@ -155,7 +185,14 @@ export class StorageManager {
155
185
  config.dataPath.startsWith("s3://");
156
186
 
157
187
  let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
158
- const attachOpts: string[] = [`DATA_PATH '${escapedDataPath}'`];
188
+ const attachOpts: string[] = [
189
+ `DATA_PATH '${escapedDataPath}'`,
190
+ // The manifest table is small relational metadata (one row per build).
191
+ // Set a high inlining limit so writes always land transactionally in
192
+ // the postgres catalog rather than as parquet files in object storage,
193
+ // sidestepping object-storage auth issues entirely for this path.
194
+ "DATA_INLINING_ROW_LIMIT 100000",
195
+ ];
159
196
  if (isCloudStorage) {
160
197
  attachOpts.push("OVERRIDE_DATA_PATH true");
161
198
  }
@@ -163,8 +200,6 @@ export class StorageManager {
163
200
 
164
201
  logger.info(`Attaching DuckLake manifest catalog: ${attachCmd}`);
165
202
  await connection.run(attachCmd);
166
-
167
- this.attachedCatalogs.add(catalogName);
168
203
  }
169
204
 
170
205
  getRepository(): ResourceRepository {
@@ -31,23 +31,32 @@ import { DuckDBConnection } from "../duckdb/DuckDBConnection";
31
31
  */
32
32
  export class DuckLakeManifestStore implements ManifestStore {
33
33
  private readonly table: string;
34
+ private readonly projectName: string;
34
35
 
35
36
  constructor(
36
37
  private db: DuckDBConnection,
37
38
  catalogName: string,
39
+ projectName: string,
38
40
  ) {
39
41
  this.table = `${catalogName}.build_manifests`;
42
+ this.projectName = projectName;
40
43
  }
41
44
 
42
45
  /**
43
46
  * Idempotently creates the `build_manifests` table and indices in the
44
47
  * DuckLake catalog. Safe to call from every worker on startup.
48
+ *
49
+ * Note: this table uses `project_name` for the partition column, unlike
50
+ * the local DuckDB `build_manifests` table (in `storage/duckdb/schema.ts`)
51
+ * which uses `project_id` and FK-references `projects.id`. The two stores
52
+ * partition by different identifiers — local random id vs cross-pod-stable
53
+ * project name — so they intentionally diverge here.
45
54
  */
46
55
  async bootstrapSchema(): Promise<void> {
47
56
  await this.db.run(`
48
57
  CREATE TABLE IF NOT EXISTS ${this.table} (
49
58
  id VARCHAR,
50
- project_id VARCHAR NOT NULL,
59
+ project_name VARCHAR NOT NULL,
51
60
  package_name VARCHAR NOT NULL,
52
61
  build_id VARCHAR NOT NULL,
53
62
  table_name VARCHAR NOT NULL,
@@ -61,12 +70,12 @@ export class DuckLakeManifestStore implements ManifestStore {
61
70
  }
62
71
 
63
72
  async getManifest(
64
- projectId: string,
73
+ _projectId: string,
65
74
  packageName: string,
66
75
  ): Promise<BuildManifest> {
67
76
  const rows = await this.db.all<Record<string, unknown>>(
68
- `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
69
- [projectId, packageName],
77
+ `SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`,
78
+ [this.projectName, packageName],
70
79
  );
71
80
  const manifest: BuildManifest = { entries: {}, strict: false };
72
81
  for (const row of rows) {
@@ -88,7 +97,7 @@ export class DuckLakeManifestStore implements ManifestStore {
88
97
  * {@link getManifest} deduplicates by build_id keeping the newest row.
89
98
  */
90
99
  async writeEntry(
91
- projectId: string,
100
+ _projectId: string,
92
101
  packageName: string,
93
102
  buildId: string,
94
103
  tableName: string,
@@ -99,11 +108,11 @@ export class DuckLakeManifestStore implements ManifestStore {
99
108
  const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
100
109
 
101
110
  await this.db.run(
102
- `INSERT INTO ${this.table} (id, project_id, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
111
+ `INSERT INTO ${this.table} (id, project_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
103
112
  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
104
113
  [
105
114
  id,
106
- projectId,
115
+ this.projectName,
107
116
  packageName,
108
117
  buildId,
109
118
  tableName,
@@ -120,12 +129,12 @@ export class DuckLakeManifestStore implements ManifestStore {
120
129
  }
121
130
 
122
131
  async listEntries(
123
- projectId: string,
132
+ _projectId: string,
124
133
  packageName: string,
125
134
  ): Promise<ManifestEntry[]> {
126
135
  const rows = await this.db.all<Record<string, unknown>>(
127
- `SELECT * FROM ${this.table} WHERE project_id = ? AND package_name = ? ORDER BY created_at DESC`,
128
- [projectId, packageName],
136
+ `SELECT * FROM ${this.table} WHERE project_name = ? AND package_name = ? ORDER BY created_at DESC`,
137
+ [this.projectName, packageName],
129
138
  );
130
139
  return rows.map(this.mapToEntry);
131
140
  }
@@ -133,7 +142,7 @@ export class DuckLakeManifestStore implements ManifestStore {
133
142
  private mapToEntry(row: Record<string, unknown>): ManifestEntry {
134
143
  return {
135
144
  id: row.id as string,
136
- projectId: row.project_id as string,
145
+ projectId: row.project_name as string,
137
146
  packageName: row.package_name as string,
138
147
  buildId: row.build_id as string,
139
148
  tableName: row.table_name as string,
@@ -5,6 +5,7 @@ import {
5
5
  Request,
6
6
  Result,
7
7
  } from "@modelcontextprotocol/sdk/types.js";
8
+ import { Mutex } from "async-mutex";
8
9
  import http from "http";
9
10
  import { AddressInfo } from "net";
10
11
  import path from "path";
@@ -24,37 +25,24 @@ export interface McpE2ETestEnvironment {
24
25
  originalInitializeStorage: string | undefined;
25
26
  }
26
27
 
27
- // Counter for unique port assignment per test suite
28
+ // Counter for unique port assignment per test suite (incremented only under e2eSetupMutex)
28
29
  let portCounter = 0;
29
30
 
30
- // Mutex to prevent concurrent initialization (since all tests share the same database)
31
- let initializationLock: Promise<void> | null = null;
31
+ // True mutex: the previous Promise-based lock had a TOCTOU race where two beforeAll hooks
32
+ // could both pass the `if (initializationLock)` check and run setup concurrently, sharing
33
+ // one publisher.db / server singleton and causing flaky listResources and port collisions.
34
+ const e2eSetupMutex = new Mutex();
32
35
 
33
36
  /**
34
37
  * Starts the real application server and connects a real MCP client.
35
38
  */
36
39
  export async function setupE2ETestEnvironment(): Promise<McpE2ETestEnvironment> {
37
- // Wait for any ongoing initialization to complete (prevents concurrent DB initialization)
38
- if (initializationLock) {
40
+ return e2eSetupMutex.runExclusive(async () => {
39
41
  console.log(
40
- "[E2E Test Setup] Waiting for previous initialization to complete...",
42
+ "[E2E Test Setup] Acquired setup mutex; starting environment...",
41
43
  );
42
- await initializationLock;
43
- }
44
-
45
- // Create a new lock for this initialization
46
- let resolveLock: () => void;
47
- initializationLock = new Promise((resolve) => {
48
- resolveLock = resolve;
44
+ return setupE2ETestEnvironmentInternal();
49
45
  });
50
-
51
- try {
52
- return await setupE2ETestEnvironmentInternal();
53
- } finally {
54
- // Release the lock
55
- resolveLock!();
56
- initializationLock = null;
57
- }
58
46
  }
59
47
 
60
48
  async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment> {
@@ -129,9 +117,9 @@ async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment>
129
117
 
130
118
  // --- Wait for server to be ready (packages downloaded) ---
131
119
  // Poll the readiness endpoint to ensure initialization completes before tests run
132
- // Note: Reduced timeout to 90s to be under the 100s test timeout
120
+ // Allow >90s: cold CI can spend most of the budget on clone + package load before markReady().
133
121
  console.log("[E2E Test Setup] Waiting for server to be ready...");
134
- const maxWaitTime = 90000; // 90 seconds max wait (under 100s test timeout)
122
+ const maxWaitTime = 150000; // 150 seconds (INITIALIZE_STORAGE + large samples can be slow)
135
123
  const pollInterval = 1000; // Check every second
136
124
  const startTime = Date.now();
137
125
  let isReady = false;
@@ -19,7 +19,7 @@ import {
19
19
  } from "../../harness/mcp_test_setup";
20
20
 
21
21
  // --- Test Suite ---
22
- describe("MCP Tool Handlers (E2E Integration)", () => {
22
+ describe.serial("MCP Tool Handlers (E2E Integration)", () => {
23
23
  let env: McpE2ETestEnvironment | null = null;
24
24
  let mcpClient: Client;
25
25
 
@@ -20,7 +20,8 @@ interface PackageContentEntry {
20
20
  }
21
21
 
22
22
  // --- Test Suite ---
23
- describe("MCP Resource Handlers (E2E Integration)", () => {
23
+ // Serial: one shared MCP client; concurrent it() blocks can interleave StreamableHTTP requests.
24
+ describe.serial("MCP Resource Handlers (E2E Integration)", () => {
24
25
  let env: McpE2ETestEnvironment | null = null;
25
26
  let mcpClient: Client;
26
27
 
@@ -85,17 +86,17 @@ describe("MCP Resource Handlers (E2E Integration)", () => {
85
86
  expect(faaPackageEntry).toBeDefined();
86
87
  expect(faaPackageEntry?.name).toBe("faa");
87
88
 
88
- const firstResource = result.resources[0];
89
- expect(firstResource).toBeDefined();
90
- expect(firstResource.uri).toBeDefined();
91
- expect(typeof firstResource.uri).toBe("string");
92
- expect(firstResource.uri).toMatch(/^malloy:\/\//);
93
- expect(firstResource.name).toBeDefined();
94
- expect(typeof firstResource.name).toBe("string");
95
- expect(firstResource.description).toBeDefined();
96
- expect(typeof firstResource.description).toBe("string");
97
- if (firstResource.mimeType) {
98
- expect(typeof firstResource.mimeType).toBe("string");
89
+ // Assert shape on a known row (order of listResources is not guaranteed).
90
+ const sample = faaPackageEntry!;
91
+ expect(sample.uri).toBeDefined();
92
+ expect(typeof sample.uri).toBe("string");
93
+ expect(sample.uri).toMatch(/^malloy:\/\//);
94
+ expect(sample.name).toBeDefined();
95
+ expect(typeof sample.name).toBe("string");
96
+ expect(sample.description).toBeDefined();
97
+ expect(typeof sample.description).toBe("string");
98
+ if (sample.mimeType) {
99
+ expect(typeof sample.mimeType).toBe("string");
99
100
  }
100
101
  },
101
102
  { timeout: 30000 },
@@ -18,7 +18,7 @@ import {
18
18
  // --- Test Suite ---
19
19
  // Note: These tests assume interaction via a standard MCP client.
20
20
  // Tests for raw HTTP edge cases (non-JSON, non-RPC) are omitted based on this assumption.
21
- describe("MCP Transport Tests (E2E Integration)", () => {
21
+ describe.serial("MCP Transport Tests (E2E Integration)", () => {
22
22
  let env: McpE2ETestEnvironment | null = null;
23
23
  let mcpClient: Client<Request, Notification, Result>; // Convenience variable
24
24