@malloy-publisher/server 0.0.195 → 0.0.197-dev
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +213 -214
- package/dist/app/assets/EnvironmentPage-1j6QDWAy.js +1 -0
- package/dist/app/assets/HomePage-DMop21VG.js +1 -0
- package/dist/app/assets/MainPage-BbE8ETz1.js +2 -0
- package/dist/app/assets/ModelPage-D2jvfe3t.js +1 -0
- package/dist/app/assets/PackagePage-BbnhGoD3.js +1 -0
- package/dist/app/assets/{RouteError-DefbDO7F.js → RouteError-D3LGEZ3i.js} +1 -1
- package/dist/app/assets/WorkbookPage-DttVIj4u.js +1 -0
- package/dist/app/assets/{core-BrfQApxh.es-DnvCX4oH.js → core-w79IMXAG.es-Bd0UlzOL.js} +1 -1
- package/dist/app/assets/{index-Bu0ub036.js → index-5K9YjIxF.js} +117 -117
- package/dist/app/assets/{index-CkzK3JIl.js → index-C513UodQ.js} +1 -1
- package/dist/app/assets/{index-CoA6HIGS.js → index-DIgzgp69.js} +1 -1
- package/dist/app/assets/{index.umd-B6Ms2PpL.js → index.umd-BMeMPq_9.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/server.mjs +1352 -1310
- package/package.json +2 -2
- package/publisher.config.json +2 -2
- package/src/config.spec.ts +74 -66
- package/src/config.ts +50 -47
- package/src/controller/compile.controller.ts +10 -7
- package/src/controller/connection.controller.ts +79 -58
- package/src/controller/database.controller.ts +10 -7
- package/src/controller/manifest.controller.ts +23 -14
- package/src/controller/materialization.controller.ts +14 -14
- package/src/controller/model.controller.ts +35 -20
- package/src/controller/package.controller.ts +83 -49
- package/src/controller/query.controller.ts +11 -8
- package/src/controller/watch-mode.controller.ts +35 -29
- package/src/errors.ts +2 -2
- package/src/mcp/error_messages.ts +2 -2
- package/src/mcp/handler_utils.ts +23 -20
- package/src/mcp/mcp_constants.ts +1 -1
- package/src/mcp/prompts/handlers.ts +3 -3
- package/src/mcp/prompts/prompt_service.ts +5 -5
- package/src/mcp/prompts/utils.ts +12 -12
- package/src/mcp/resource_metadata.ts +3 -3
- package/src/mcp/resources/environment_resource.ts +187 -0
- package/src/mcp/resources/model_resource.ts +19 -17
- package/src/mcp/resources/notebook_resource.ts +13 -13
- package/src/mcp/resources/package_resource.ts +30 -27
- package/src/mcp/resources/query_resource.ts +15 -10
- package/src/mcp/resources/source_resource.ts +10 -10
- package/src/mcp/resources/view_resource.ts +11 -11
- package/src/mcp/server.ts +16 -14
- package/src/mcp/tools/discovery_tools.ts +67 -49
- package/src/mcp/tools/execute_query_tool.ts +14 -14
- package/src/server.ts +175 -159
- package/src/service/connection.spec.ts +158 -133
- package/src/service/connection.ts +42 -39
- package/src/service/connection_config.spec.ts +13 -11
- package/src/service/connection_config.ts +28 -19
- package/src/service/connection_service.spec.ts +63 -43
- package/src/service/connection_service.ts +106 -89
- package/src/service/{project.ts → environment.ts} +92 -77
- package/src/service/{project_compile.spec.ts → environment_compile.spec.ts} +1 -1
- package/src/service/{project_store.spec.ts → environment_store.spec.ts} +99 -83
- package/src/service/{project_store.ts → environment_store.ts} +373 -327
- package/src/service/manifest_service.spec.ts +15 -15
- package/src/service/manifest_service.ts +26 -21
- package/src/service/materialization_service.spec.ts +93 -59
- package/src/service/materialization_service.ts +71 -62
- package/src/service/materialized_table_gc.spec.ts +15 -15
- package/src/service/materialized_table_gc.ts +3 -3
- package/src/service/model.ts +4 -4
- package/src/service/package.spec.ts +2 -2
- package/src/service/package.ts +23 -21
- package/src/service/resolve_environment.ts +15 -0
- package/src/storage/DatabaseInterface.ts +34 -25
- package/src/storage/StorageManager.mock.ts +3 -3
- package/src/storage/StorageManager.ts +64 -28
- package/src/storage/duckdb/ConnectionRepository.ts +13 -11
- package/src/storage/duckdb/DuckDBConnection.ts +1 -1
- package/src/storage/duckdb/DuckDBManifestStore.ts +6 -6
- package/src/storage/duckdb/DuckDBRepository.ts +47 -47
- package/src/storage/duckdb/{ProjectRepository.ts → EnvironmentRepository.ts} +35 -35
- package/src/storage/duckdb/ManifestRepository.ts +21 -20
- package/src/storage/duckdb/MaterializationRepository.ts +31 -28
- package/src/storage/duckdb/PackageRepository.ts +11 -11
- package/src/storage/duckdb/manifest_store.spec.ts +2 -2
- package/src/storage/duckdb/schema.ts +20 -20
- package/src/storage/ducklake/DuckLakeManifestStore.ts +20 -11
- package/tests/fixtures/publisher.config.json +1 -1
- package/tests/harness/e2e.ts +1 -1
- package/tests/harness/mcp_test_setup.ts +12 -24
- package/tests/harness/mocks.ts +10 -8
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +4 -4
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +28 -49
- package/tests/integration/mcp/mcp_resource.integration.spec.ts +39 -47
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +1 -1
- package/tests/unit/duckdb/attached_databases.test.ts +51 -33
- package/tests/unit/ducklake/ducklake.test.ts +24 -22
- package/tests/unit/mcp/prompt_happy.test.ts +8 -8
- package/dist/app/assets/HomePage-DbZS0N7G.js +0 -1
- package/dist/app/assets/MainPage-CBuWkbmr.js +0 -2
- package/dist/app/assets/ModelPage-Bt37smot.js +0 -1
- package/dist/app/assets/PackagePage-DLZe50WG.js +0 -1
- package/dist/app/assets/ProjectPage-FQTEPXP4.js +0 -1
- package/dist/app/assets/WorkbookPage-CkAo16ar.js +0 -1
- package/src/mcp/resources/project_resource.ts +0 -184
- package/src/service/resolve_project.ts +0 -13
|
@@ -12,10 +12,10 @@ export class PackageRepository {
|
|
|
12
12
|
return new Date();
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
async listPackages(
|
|
15
|
+
async listPackages(environmentId: string): Promise<Package[]> {
|
|
16
16
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
17
|
-
"SELECT * FROM packages WHERE
|
|
18
|
-
[
|
|
17
|
+
"SELECT * FROM packages WHERE environment_id = ? ORDER BY name",
|
|
18
|
+
[environmentId],
|
|
19
19
|
);
|
|
20
20
|
return rows.map(this.mapToPackage);
|
|
21
21
|
}
|
|
@@ -29,12 +29,12 @@ export class PackageRepository {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
async getPackageByName(
|
|
32
|
-
|
|
32
|
+
environmentId: string,
|
|
33
33
|
name: string,
|
|
34
34
|
): Promise<Package | null> {
|
|
35
35
|
const row = await this.db.get<Record<string, unknown>>(
|
|
36
|
-
"SELECT * FROM packages WHERE
|
|
37
|
-
[
|
|
36
|
+
"SELECT * FROM packages WHERE environment_id = ? AND name = ?",
|
|
37
|
+
[environmentId, name],
|
|
38
38
|
);
|
|
39
39
|
return row ? this.mapToPackage(row) : null;
|
|
40
40
|
}
|
|
@@ -46,11 +46,11 @@ export class PackageRepository {
|
|
|
46
46
|
const now = this.now();
|
|
47
47
|
|
|
48
48
|
await this.db.run(
|
|
49
|
-
`INSERT INTO packages (id,
|
|
49
|
+
`INSERT INTO packages (id, environment_id, name, description, manifest_path, metadata, created_at, updated_at)
|
|
50
50
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
51
51
|
[
|
|
52
52
|
id,
|
|
53
|
-
pkg.
|
|
53
|
+
pkg.environmentId,
|
|
54
54
|
pkg.name,
|
|
55
55
|
pkg.description || null,
|
|
56
56
|
pkg.manifestPath,
|
|
@@ -114,14 +114,14 @@ export class PackageRepository {
|
|
|
114
114
|
await this.db.run("DELETE FROM packages WHERE id = ?", [id]);
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
async
|
|
118
|
-
await this.db.run("DELETE FROM packages WHERE
|
|
117
|
+
async deletePackagesByEnvironmentId(id: string): Promise<void> {
|
|
118
|
+
await this.db.run("DELETE FROM packages WHERE environment_id = ?", [id]);
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
private mapToPackage(row: Record<string, unknown>): Package {
|
|
122
122
|
return {
|
|
123
123
|
id: row.id as string,
|
|
124
|
-
|
|
124
|
+
environmentId: row.environment_id as string,
|
|
125
125
|
name: row.name as string,
|
|
126
126
|
description: row.description as string | undefined,
|
|
127
127
|
manifestPath: row.manifest_path as string,
|
|
@@ -6,7 +6,7 @@ import { DuckDBManifestStore } from "./DuckDBManifestStore";
|
|
|
6
6
|
function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
|
|
7
7
|
return {
|
|
8
8
|
id: "entry-1",
|
|
9
|
-
|
|
9
|
+
environmentId: "proj-1",
|
|
10
10
|
packageName: "pkg",
|
|
11
11
|
buildId: "build-abc",
|
|
12
12
|
tableName: "my_table",
|
|
@@ -83,7 +83,7 @@ describe("DuckDBManifestStore", () => {
|
|
|
83
83
|
expect(ctx.repository.upsertManifestEntry.calledOnce).toBe(true);
|
|
84
84
|
const arg = ctx.repository.upsertManifestEntry.firstCall.args[0];
|
|
85
85
|
expect(arg).toEqual({
|
|
86
|
-
|
|
86
|
+
environmentId: "proj-1",
|
|
87
87
|
packageName: "pkg",
|
|
88
88
|
buildId: "build-abc",
|
|
89
89
|
tableName: "tbl",
|
|
@@ -20,9 +20,9 @@ export async function initializeSchema(
|
|
|
20
20
|
logger.info("Creating database schema for the first time...");
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
-
//
|
|
23
|
+
// Environments table
|
|
24
24
|
await db.run(`
|
|
25
|
-
CREATE TABLE IF NOT EXISTS
|
|
25
|
+
CREATE TABLE IF NOT EXISTS environments (
|
|
26
26
|
id VARCHAR PRIMARY KEY,
|
|
27
27
|
name VARCHAR NOT NULL UNIQUE,
|
|
28
28
|
path VARCHAR NOT NULL,
|
|
@@ -37,15 +37,15 @@ export async function initializeSchema(
|
|
|
37
37
|
await db.run(`
|
|
38
38
|
CREATE TABLE IF NOT EXISTS packages (
|
|
39
39
|
id VARCHAR PRIMARY KEY,
|
|
40
|
-
|
|
40
|
+
environment_id VARCHAR NOT NULL,
|
|
41
41
|
name VARCHAR NOT NULL,
|
|
42
42
|
description VARCHAR,
|
|
43
43
|
manifest_path VARCHAR NOT NULL,
|
|
44
44
|
metadata JSON,
|
|
45
45
|
created_at TIMESTAMP NOT NULL,
|
|
46
46
|
updated_at TIMESTAMP NOT NULL,
|
|
47
|
-
FOREIGN KEY (
|
|
48
|
-
UNIQUE (
|
|
47
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
48
|
+
UNIQUE (environment_id, name)
|
|
49
49
|
)
|
|
50
50
|
`);
|
|
51
51
|
|
|
@@ -53,22 +53,22 @@ export async function initializeSchema(
|
|
|
53
53
|
await db.run(`
|
|
54
54
|
CREATE TABLE IF NOT EXISTS connections (
|
|
55
55
|
id VARCHAR PRIMARY KEY,
|
|
56
|
-
|
|
56
|
+
environment_id VARCHAR NOT NULL,
|
|
57
57
|
name VARCHAR NOT NULL,
|
|
58
58
|
type VARCHAR NOT NULL,
|
|
59
59
|
config JSON NOT NULL,
|
|
60
60
|
created_at TIMESTAMP NOT NULL,
|
|
61
61
|
updated_at TIMESTAMP NOT NULL,
|
|
62
|
-
FOREIGN KEY (
|
|
63
|
-
UNIQUE (
|
|
62
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
63
|
+
UNIQUE (environment_id, name)
|
|
64
64
|
)
|
|
65
65
|
`);
|
|
66
66
|
|
|
67
67
|
// Materializations table.
|
|
68
68
|
//
|
|
69
69
|
// `active_key` enforces at-most-one active (PENDING or RUNNING)
|
|
70
|
-
// materialization per (
|
|
71
|
-
// `{
|
|
70
|
+
// materialization per (environment, package) at the DB layer. It is set to
|
|
71
|
+
// `{environment_id}|{package_name}` while the row is active and cleared
|
|
72
72
|
// to NULL on transition to any terminal state. A unique index on
|
|
73
73
|
// `active_key` (see below) makes the insert-then-check race impossible —
|
|
74
74
|
// a second concurrent create fails with a constraint violation, which the
|
|
@@ -76,7 +76,7 @@ export async function initializeSchema(
|
|
|
76
76
|
await db.run(`
|
|
77
77
|
CREATE TABLE IF NOT EXISTS materializations (
|
|
78
78
|
id VARCHAR PRIMARY KEY,
|
|
79
|
-
|
|
79
|
+
environment_id VARCHAR NOT NULL,
|
|
80
80
|
package_name VARCHAR NOT NULL,
|
|
81
81
|
status VARCHAR NOT NULL,
|
|
82
82
|
active_key VARCHAR,
|
|
@@ -86,7 +86,7 @@ export async function initializeSchema(
|
|
|
86
86
|
metadata JSON,
|
|
87
87
|
created_at TIMESTAMP NOT NULL,
|
|
88
88
|
updated_at TIMESTAMP NOT NULL,
|
|
89
|
-
FOREIGN KEY (
|
|
89
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id)
|
|
90
90
|
)
|
|
91
91
|
`);
|
|
92
92
|
|
|
@@ -94,7 +94,7 @@ export async function initializeSchema(
|
|
|
94
94
|
await db.run(`
|
|
95
95
|
CREATE TABLE IF NOT EXISTS build_manifests (
|
|
96
96
|
id VARCHAR PRIMARY KEY,
|
|
97
|
-
|
|
97
|
+
environment_id VARCHAR NOT NULL,
|
|
98
98
|
package_name VARCHAR NOT NULL,
|
|
99
99
|
build_id VARCHAR NOT NULL,
|
|
100
100
|
table_name VARCHAR NOT NULL,
|
|
@@ -102,26 +102,26 @@ export async function initializeSchema(
|
|
|
102
102
|
connection_name VARCHAR NOT NULL,
|
|
103
103
|
created_at TIMESTAMP NOT NULL,
|
|
104
104
|
updated_at TIMESTAMP NOT NULL,
|
|
105
|
-
FOREIGN KEY (
|
|
106
|
-
UNIQUE (
|
|
105
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
106
|
+
UNIQUE (environment_id, package_name, build_id)
|
|
107
107
|
)
|
|
108
108
|
`);
|
|
109
109
|
|
|
110
110
|
// Create indexes for better query performance
|
|
111
111
|
await db.run(
|
|
112
|
-
"CREATE INDEX IF NOT EXISTS
|
|
112
|
+
"CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)",
|
|
113
113
|
);
|
|
114
114
|
await db.run(
|
|
115
|
-
"CREATE INDEX IF NOT EXISTS
|
|
115
|
+
"CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)",
|
|
116
116
|
);
|
|
117
117
|
await db.run(
|
|
118
|
-
"CREATE INDEX IF NOT EXISTS
|
|
118
|
+
"CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)",
|
|
119
119
|
);
|
|
120
120
|
await db.run(
|
|
121
121
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
|
|
122
122
|
);
|
|
123
123
|
await db.run(
|
|
124
|
-
"CREATE INDEX IF NOT EXISTS
|
|
124
|
+
"CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)",
|
|
125
125
|
);
|
|
126
126
|
}
|
|
127
127
|
|
|
@@ -131,7 +131,7 @@ async function dropAllTables(db: DuckDBConnection): Promise<void> {
|
|
|
131
131
|
"materializations",
|
|
132
132
|
"packages",
|
|
133
133
|
"connections",
|
|
134
|
-
"
|
|
134
|
+
"environments",
|
|
135
135
|
];
|
|
136
136
|
|
|
137
137
|
logger.info("Dropping tables:", tables.join(", "));
|
|
@@ -31,23 +31,32 @@ import { DuckDBConnection } from "../duckdb/DuckDBConnection";
|
|
|
31
31
|
*/
|
|
32
32
|
export class DuckLakeManifestStore implements ManifestStore {
|
|
33
33
|
private readonly table: string;
|
|
34
|
+
private readonly environmentName: string;
|
|
34
35
|
|
|
35
36
|
constructor(
|
|
36
37
|
private db: DuckDBConnection,
|
|
37
38
|
catalogName: string,
|
|
39
|
+
environmentName: string,
|
|
38
40
|
) {
|
|
39
41
|
this.table = `${catalogName}.build_manifests`;
|
|
42
|
+
this.environmentName = environmentName;
|
|
40
43
|
}
|
|
41
44
|
|
|
42
45
|
/**
|
|
43
46
|
* Idempotently creates the `build_manifests` table and indices in the
|
|
44
47
|
* DuckLake catalog. Safe to call from every worker on startup.
|
|
48
|
+
*
|
|
49
|
+
* Note: this table uses `project_name` for the partition column, unlike
|
|
50
|
+
* the local DuckDB `build_manifests` table (in `storage/duckdb/schema.ts`)
|
|
51
|
+
* which uses `project_id` and FK-references `projects.id`. The two stores
|
|
52
|
+
* partition by different identifiers — local random id vs cross-pod-stable
|
|
53
|
+
* project name — so they intentionally diverge here.
|
|
45
54
|
*/
|
|
46
55
|
async bootstrapSchema(): Promise<void> {
|
|
47
56
|
await this.db.run(`
|
|
48
57
|
CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
49
58
|
id VARCHAR,
|
|
50
|
-
|
|
59
|
+
environment_name VARCHAR NOT NULL,
|
|
51
60
|
package_name VARCHAR NOT NULL,
|
|
52
61
|
build_id VARCHAR NOT NULL,
|
|
53
62
|
table_name VARCHAR NOT NULL,
|
|
@@ -61,12 +70,12 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
61
70
|
}
|
|
62
71
|
|
|
63
72
|
async getManifest(
|
|
64
|
-
|
|
73
|
+
_environmentId: string,
|
|
65
74
|
packageName: string,
|
|
66
75
|
): Promise<BuildManifest> {
|
|
67
76
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
68
|
-
`SELECT * FROM ${this.table} WHERE
|
|
69
|
-
[
|
|
77
|
+
`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`,
|
|
78
|
+
[this.environmentName, packageName],
|
|
70
79
|
);
|
|
71
80
|
const manifest: BuildManifest = { entries: {}, strict: false };
|
|
72
81
|
for (const row of rows) {
|
|
@@ -88,7 +97,7 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
88
97
|
* {@link getManifest} deduplicates by build_id keeping the newest row.
|
|
89
98
|
*/
|
|
90
99
|
async writeEntry(
|
|
91
|
-
|
|
100
|
+
_environmentId: string,
|
|
92
101
|
packageName: string,
|
|
93
102
|
buildId: string,
|
|
94
103
|
tableName: string,
|
|
@@ -99,11 +108,11 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
99
108
|
const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
100
109
|
|
|
101
110
|
await this.db.run(
|
|
102
|
-
`INSERT INTO ${this.table} (id,
|
|
111
|
+
`INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
103
112
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
104
113
|
[
|
|
105
114
|
id,
|
|
106
|
-
|
|
115
|
+
this.environmentName,
|
|
107
116
|
packageName,
|
|
108
117
|
buildId,
|
|
109
118
|
tableName,
|
|
@@ -120,12 +129,12 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
120
129
|
}
|
|
121
130
|
|
|
122
131
|
async listEntries(
|
|
123
|
-
|
|
132
|
+
_environmentId: string,
|
|
124
133
|
packageName: string,
|
|
125
134
|
): Promise<ManifestEntry[]> {
|
|
126
135
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
127
|
-
`SELECT * FROM ${this.table} WHERE
|
|
128
|
-
[
|
|
136
|
+
`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`,
|
|
137
|
+
[this.environmentName, packageName],
|
|
129
138
|
);
|
|
130
139
|
return rows.map(this.mapToEntry);
|
|
131
140
|
}
|
|
@@ -133,7 +142,7 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
133
142
|
private mapToEntry(row: Record<string, unknown>): ManifestEntry {
|
|
134
143
|
return {
|
|
135
144
|
id: row.id as string,
|
|
136
|
-
|
|
145
|
+
environmentId: row.environment_name as string,
|
|
137
146
|
packageName: row.package_name as string,
|
|
138
147
|
buildId: row.build_id as string,
|
|
139
148
|
tableName: row.table_name as string,
|
package/tests/harness/e2e.ts
CHANGED
|
@@ -28,7 +28,7 @@ let originalServerRoot: string | undefined;
|
|
|
28
28
|
*/
|
|
29
29
|
export async function startE2E(): Promise<E2EEnv & { stop(): Promise<void> }> {
|
|
30
30
|
//--------------------------------------------------------------------------
|
|
31
|
-
// 1. Set SERVER_ROOT so
|
|
31
|
+
// 1. Set SERVER_ROOT so EnvironmentStore loader finds publisher.config.json
|
|
32
32
|
//--------------------------------------------------------------------------
|
|
33
33
|
originalServerRoot = process.env.SERVER_ROOT;
|
|
34
34
|
// Use import.meta.url for cross-platform compatibility (works on Windows)
|
|
@@ -5,6 +5,7 @@ import {
|
|
|
5
5
|
Request,
|
|
6
6
|
Result,
|
|
7
7
|
} from "@modelcontextprotocol/sdk/types.js";
|
|
8
|
+
import { Mutex } from "async-mutex";
|
|
8
9
|
import http from "http";
|
|
9
10
|
import { AddressInfo } from "net";
|
|
10
11
|
import path from "path";
|
|
@@ -24,42 +25,29 @@ export interface McpE2ETestEnvironment {
|
|
|
24
25
|
originalInitializeStorage: string | undefined;
|
|
25
26
|
}
|
|
26
27
|
|
|
27
|
-
// Counter for unique port assignment per test suite
|
|
28
|
+
// Counter for unique port assignment per test suite (incremented only under e2eSetupMutex)
|
|
28
29
|
let portCounter = 0;
|
|
29
30
|
|
|
30
|
-
//
|
|
31
|
-
|
|
31
|
+
// True mutex: the previous Promise-based lock had a TOCTOU race where two beforeAll hooks
|
|
32
|
+
// could both pass the `if (initializationLock)` check and run setup concurrently, sharing
|
|
33
|
+
// one publisher.db / server singleton and causing flaky listResources and port collisions.
|
|
34
|
+
const e2eSetupMutex = new Mutex();
|
|
32
35
|
|
|
33
36
|
/**
|
|
34
37
|
* Starts the real application server and connects a real MCP client.
|
|
35
38
|
*/
|
|
36
39
|
export async function setupE2ETestEnvironment(): Promise<McpE2ETestEnvironment> {
|
|
37
|
-
|
|
38
|
-
if (initializationLock) {
|
|
40
|
+
return e2eSetupMutex.runExclusive(async () => {
|
|
39
41
|
console.log(
|
|
40
|
-
"[E2E Test Setup]
|
|
42
|
+
"[E2E Test Setup] Acquired setup mutex; starting environment...",
|
|
41
43
|
);
|
|
42
|
-
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
// Create a new lock for this initialization
|
|
46
|
-
let resolveLock: () => void;
|
|
47
|
-
initializationLock = new Promise((resolve) => {
|
|
48
|
-
resolveLock = resolve;
|
|
44
|
+
return setupE2ETestEnvironmentInternal();
|
|
49
45
|
});
|
|
50
|
-
|
|
51
|
-
try {
|
|
52
|
-
return await setupE2ETestEnvironmentInternal();
|
|
53
|
-
} finally {
|
|
54
|
-
// Release the lock
|
|
55
|
-
resolveLock!();
|
|
56
|
-
initializationLock = null;
|
|
57
|
-
}
|
|
58
46
|
}
|
|
59
47
|
|
|
60
48
|
async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment> {
|
|
61
49
|
// --- Store and Set SERVER_ROOT Env Var ---
|
|
62
|
-
// The
|
|
50
|
+
// The EnvironmentStore relies on SERVER_ROOT to find publisher.config.json.
|
|
63
51
|
const originalServerRoot = process.env.SERVER_ROOT; // Store original value
|
|
64
52
|
// Resolve the path to 'packages/server' based on the location of this file
|
|
65
53
|
// Use import.meta.url for cross-platform compatibility (works on Windows)
|
|
@@ -129,9 +117,9 @@ async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment>
|
|
|
129
117
|
|
|
130
118
|
// --- Wait for server to be ready (packages downloaded) ---
|
|
131
119
|
// Poll the readiness endpoint to ensure initialization completes before tests run
|
|
132
|
-
//
|
|
120
|
+
// Allow >90s: cold CI can spend most of the budget on clone + package load before markReady().
|
|
133
121
|
console.log("[E2E Test Setup] Waiting for server to be ready...");
|
|
134
|
-
const maxWaitTime =
|
|
122
|
+
const maxWaitTime = 150000; // 150 seconds (INITIALIZE_STORAGE + large samples can be slow)
|
|
135
123
|
const pollInterval = 1000; // Check every second
|
|
136
124
|
const startTime = Date.now();
|
|
137
125
|
let isReady = false;
|
package/tests/harness/mocks.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import sinon from "sinon";
|
|
2
|
-
import {
|
|
2
|
+
import { EnvironmentStore } from "../../src/service/environment_store";
|
|
3
3
|
|
|
4
|
-
/** Return a stubbed
|
|
5
|
-
export function
|
|
6
|
-
const
|
|
7
|
-
// For now just have
|
|
8
|
-
|
|
9
|
-
|
|
4
|
+
/** Return a stubbed EnvironmentStore where every lookup throws or returns minimal objects. */
|
|
5
|
+
export function fakeEnvironmentStore(): sinon.SinonStubbedInstance<EnvironmentStore> {
|
|
6
|
+
const es = sinon.createStubInstance(EnvironmentStore);
|
|
7
|
+
// For now just have getEnvironment reject; suites can stub more.
|
|
8
|
+
es.getEnvironment.rejects(
|
|
9
|
+
new Error("fakeEnvironmentStore: getEnvironment not stubbed"),
|
|
10
|
+
);
|
|
11
|
+
return es;
|
|
10
12
|
}
|
|
11
13
|
|
|
12
14
|
// The runtime implementation of `McpServer` lives in the MCP SDK package, which
|
|
@@ -27,7 +29,7 @@ export type McpServer = DummyMcpServer;
|
|
|
27
29
|
/** Convenience helper mimicking the old mocks used in integration specs. */
|
|
28
30
|
export function createMalloyServiceMocks() {
|
|
29
31
|
return {
|
|
30
|
-
|
|
32
|
+
environmentStore: fakeEnvironmentStore(),
|
|
31
33
|
} as const;
|
|
32
34
|
}
|
|
33
35
|
|
|
@@ -10,7 +10,7 @@ const __dirname = path.dirname(__filename);
|
|
|
10
10
|
|
|
11
11
|
const PROJECT_NAME = "test-project";
|
|
12
12
|
const PACKAGE_NAME = "persist-test";
|
|
13
|
-
const API = `/api/v0/
|
|
13
|
+
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
14
14
|
|
|
15
15
|
describe("Materialization & Manifest REST API (E2E)", () => {
|
|
16
16
|
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
@@ -23,7 +23,7 @@ describe("Materialization & Manifest REST API (E2E)", () => {
|
|
|
23
23
|
// Create the test project via the REST API using an absolute
|
|
24
24
|
// path to the fixture so it works regardless of SERVER_ROOT.
|
|
25
25
|
const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
|
|
26
|
-
const createRes = await fetch(`${baseUrl}/api/v0/
|
|
26
|
+
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
27
27
|
method: "POST",
|
|
28
28
|
headers: { "Content-Type": "application/json" },
|
|
29
29
|
body: JSON.stringify({
|
|
@@ -45,7 +45,7 @@ describe("Materialization & Manifest REST API (E2E)", () => {
|
|
|
45
45
|
while (!pkgReady && Date.now() < deadline) {
|
|
46
46
|
try {
|
|
47
47
|
const res = await fetch(
|
|
48
|
-
`${baseUrl}/api/v0/
|
|
48
|
+
`${baseUrl}/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
|
|
49
49
|
);
|
|
50
50
|
if (res.ok) {
|
|
51
51
|
pkgReady = true;
|
|
@@ -65,7 +65,7 @@ describe("Materialization & Manifest REST API (E2E)", () => {
|
|
|
65
65
|
// Tear down the test project, then the HTTP server.
|
|
66
66
|
if (baseUrl) {
|
|
67
67
|
try {
|
|
68
|
-
await fetch(`${baseUrl}/api/v0/
|
|
68
|
+
await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
|
|
69
69
|
method: "DELETE",
|
|
70
70
|
});
|
|
71
71
|
} catch {
|
|
@@ -19,11 +19,11 @@ import {
|
|
|
19
19
|
} from "../../harness/mcp_test_setup";
|
|
20
20
|
|
|
21
21
|
// --- Test Suite ---
|
|
22
|
-
describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
22
|
+
describe.serial("MCP Tool Handlers (E2E Integration)", () => {
|
|
23
23
|
let env: McpE2ETestEnvironment | null = null;
|
|
24
24
|
let mcpClient: Client;
|
|
25
25
|
|
|
26
|
-
const
|
|
26
|
+
const ENVIRONMENT_NAME = "malloy-samples";
|
|
27
27
|
const PACKAGE_NAME = "faa";
|
|
28
28
|
|
|
29
29
|
beforeAll(async () => {
|
|
@@ -47,7 +47,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
47
47
|
const result = await mcpClient.callTool({
|
|
48
48
|
name: "malloy_executeQuery",
|
|
49
49
|
arguments: {
|
|
50
|
-
|
|
50
|
+
environmentName: ENVIRONMENT_NAME,
|
|
51
51
|
packageName: PACKAGE_NAME,
|
|
52
52
|
modelPath: "flights.malloy",
|
|
53
53
|
query: "run: flights->{ aggregate: c is count() }",
|
|
@@ -95,7 +95,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
95
95
|
async () => {
|
|
96
96
|
if (!env) throw new Error("Test environment not initialized");
|
|
97
97
|
const params = {
|
|
98
|
-
|
|
98
|
+
environmentName: ENVIRONMENT_NAME,
|
|
99
99
|
packageName: PACKAGE_NAME,
|
|
100
100
|
modelPath: "flights.malloy",
|
|
101
101
|
sourceName: "flights", // Added sourceName
|
|
@@ -140,7 +140,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
140
140
|
async () => {
|
|
141
141
|
if (!env) throw new Error("Test environment not initialized");
|
|
142
142
|
const params = {
|
|
143
|
-
|
|
143
|
+
environmentName: ENVIRONMENT_NAME,
|
|
144
144
|
packageName: PACKAGE_NAME,
|
|
145
145
|
modelPath: "flights.malloy",
|
|
146
146
|
query: "run: flights->{BAD SYNTAX aggregate: flight_count is count()}",
|
|
@@ -179,7 +179,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
179
179
|
it("should RESOLVE with InvalidParams for conflicting parameters (query and queryName)", async () => {
|
|
180
180
|
if (!env) throw new Error("Test environment not initialized");
|
|
181
181
|
const params = {
|
|
182
|
-
|
|
182
|
+
environmentName: ENVIRONMENT_NAME,
|
|
183
183
|
packageName: PACKAGE_NAME,
|
|
184
184
|
modelPath: "flights.malloy",
|
|
185
185
|
query: "run: flights->{aggregate: c is count()}",
|
|
@@ -207,7 +207,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
207
207
|
it("should RESOLVE with InvalidParams if required params are missing (e.g., query or queryName)", async () => {
|
|
208
208
|
if (!env) throw new Error("Test environment not initialized");
|
|
209
209
|
const params = {
|
|
210
|
-
|
|
210
|
+
environmentName: ENVIRONMENT_NAME,
|
|
211
211
|
packageName: PACKAGE_NAME,
|
|
212
212
|
modelPath: "flights.malloy",
|
|
213
213
|
// Missing query AND queryName
|
|
@@ -235,7 +235,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
235
235
|
if (!env) throw new Error("Test environment not initialized");
|
|
236
236
|
const params = {
|
|
237
237
|
// Missing modelPath
|
|
238
|
-
|
|
238
|
+
environmentName: ENVIRONMENT_NAME,
|
|
239
239
|
packageName: PACKAGE_NAME,
|
|
240
240
|
query: "run: flights->{aggregate: flight_count is count()}",
|
|
241
241
|
};
|
|
@@ -257,7 +257,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
257
257
|
it("should return application error if package not found", async () => {
|
|
258
258
|
if (!env) throw new Error("Test environment not initialized");
|
|
259
259
|
const params = {
|
|
260
|
-
|
|
260
|
+
environmentName: ENVIRONMENT_NAME,
|
|
261
261
|
packageName: "nonexistent_package", // Use a package that doesn't exist
|
|
262
262
|
modelPath: "flights.malloy",
|
|
263
263
|
query: "run: flights->{aggregate: c is count()}",
|
|
@@ -292,7 +292,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
292
292
|
expect(errorPayloadPkgNotFound.suggestions.length).toBeGreaterThan(0);
|
|
293
293
|
|
|
294
294
|
// Check the specific error message within the parsed object
|
|
295
|
-
const expectedErrorMessageNotFound = `Resource not found: package '${params.packageName}' in
|
|
295
|
+
const expectedErrorMessageNotFound = `Resource not found: package '${params.packageName}' in environment '${params.environmentName}'`;
|
|
296
296
|
expect(errorPayloadPkgNotFound.error).toEqual(
|
|
297
297
|
expectedErrorMessageNotFound,
|
|
298
298
|
);
|
|
@@ -301,7 +301,7 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
301
301
|
it("should return application error if model not found within package", async () => {
|
|
302
302
|
if (!env) throw new Error("Test environment not initialized");
|
|
303
303
|
const params = {
|
|
304
|
-
|
|
304
|
+
environmentName: ENVIRONMENT_NAME,
|
|
305
305
|
packageName: PACKAGE_NAME,
|
|
306
306
|
modelPath: "nonexistent_model.malloy", // Use a model that doesn't exist
|
|
307
307
|
query: "run: flights->{aggregate: c is count()}",
|
|
@@ -335,68 +335,47 @@ describe("MCP Tool Handlers (E2E Integration)", () => {
|
|
|
335
335
|
expect(errorPayloadModel.suggestions.length).toBeGreaterThan(0);
|
|
336
336
|
|
|
337
337
|
// Check the specific error message within the parsed object
|
|
338
|
-
const expectedErrorMessageModel = `Resource not found: model '${params.modelPath}' in package '${params.packageName}' for
|
|
338
|
+
const expectedErrorMessageModel = `Resource not found: model '${params.modelPath}' in package '${params.packageName}' for environment '${params.environmentName}'`;
|
|
339
339
|
expect(errorPayloadModel.error).toEqual(expectedErrorMessageModel);
|
|
340
340
|
|
|
341
341
|
// Check for the specific model name and context in the message
|
|
342
342
|
expect(errorPayloadModel.error).toMatch(/Resource not found/i);
|
|
343
343
|
});
|
|
344
344
|
|
|
345
|
-
//
|
|
346
|
-
|
|
345
|
+
// Stateless HTTP + fast queries make true in-flight cancellation flaky
|
|
346
|
+
// (the response often completes before close wins the race). Assert the
|
|
347
|
+
// transport contract instead: a closed client cannot issue further tools.
|
|
348
|
+
it("should reject malloy_executeQuery after the MCP client is closed", async () => {
|
|
347
349
|
if (!env) throw new Error("Test environment not initialized");
|
|
348
350
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
const cancelClient = new Client<Request, Notification, Result>({
|
|
352
|
-
name: "cancel-test-client",
|
|
351
|
+
const closedClient = new Client<Request, Notification, Result>({
|
|
352
|
+
name: "closed-client-test",
|
|
353
353
|
version: "1.0",
|
|
354
354
|
});
|
|
355
|
-
|
|
356
|
-
const cancelTransport = new StreamableHTTPClientTransport(
|
|
355
|
+
const transport = new StreamableHTTPClientTransport(
|
|
357
356
|
new URL(env.serverUrl + "/mcp"),
|
|
358
357
|
);
|
|
359
|
-
await
|
|
358
|
+
await closedClient.connect(transport);
|
|
359
|
+
await closedClient.close();
|
|
360
360
|
|
|
361
|
-
expect
|
|
362
|
-
|
|
363
|
-
try {
|
|
364
|
-
toolPromise = cancelClient.callTool({
|
|
361
|
+
await expect(
|
|
362
|
+
closedClient.callTool({
|
|
365
363
|
name: "malloy_executeQuery",
|
|
366
364
|
arguments: {
|
|
367
|
-
|
|
365
|
+
environmentName: ENVIRONMENT_NAME,
|
|
368
366
|
packageName: PACKAGE_NAME,
|
|
369
367
|
modelPath: "flights.malloy",
|
|
370
|
-
|
|
371
|
-
query: "run: flights->{aggregate: c is count() for 100}",
|
|
368
|
+
query: "run: flights->{aggregate: c is count()}",
|
|
372
369
|
},
|
|
373
|
-
})
|
|
374
|
-
|
|
375
|
-
// Give the request a moment to start on the server
|
|
376
|
-
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
377
|
-
|
|
378
|
-
// Close the client to trigger cancellation
|
|
379
|
-
await cancelClient.close();
|
|
380
|
-
|
|
381
|
-
// Await the promise - it should reject due to the closure
|
|
382
|
-
await toolPromise;
|
|
383
|
-
|
|
384
|
-
throw new Error("Promise should have rejected due to cancellation");
|
|
385
|
-
} catch (error) {
|
|
386
|
-
// Check that the error is an Error instance and the message indicates closure/cancellation
|
|
387
|
-
expect(error).toBeInstanceOf(Error);
|
|
388
|
-
expect((error as Error).message).toMatch(/cancel|closed/i);
|
|
389
|
-
} finally {
|
|
390
|
-
// Ensure the temporary client is closed even if the test failed unexpectedly
|
|
391
|
-
await cancelClient.close().catch(() => {}); // Ignore errors on final cleanup
|
|
392
|
-
}
|
|
370
|
+
}),
|
|
371
|
+
).rejects.toThrow();
|
|
393
372
|
});
|
|
394
373
|
|
|
395
374
|
// Test invalid usage - nested view called without sourceName
|
|
396
375
|
it("should return application error for nested view without sourceName", async () => {
|
|
397
376
|
if (!env) throw new Error("Test environment not initialized");
|
|
398
377
|
const params = {
|
|
399
|
-
|
|
378
|
+
environmentName: ENVIRONMENT_NAME,
|
|
400
379
|
packageName: PACKAGE_NAME,
|
|
401
380
|
modelPath: "flights.malloy",
|
|
402
381
|
queryName: "top_carriers", // Nested view, but sourceName is missing
|