@malloy-publisher/server 0.0.196-dev → 0.0.196
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +213 -214
- package/dist/app/assets/EnvironmentPage-1j6QDWAy.js +1 -0
- package/dist/app/assets/HomePage-DMop21VG.js +1 -0
- package/dist/app/assets/MainPage-BbE8ETz1.js +2 -0
- package/dist/app/assets/ModelPage-D2jvfe3t.js +1 -0
- package/dist/app/assets/PackagePage-BbnhGoD3.js +1 -0
- package/dist/app/assets/{RouteError-DefbDO7F.js → RouteError-D3LGEZ3i.js} +1 -1
- package/dist/app/assets/WorkbookPage-DttVIj4u.js +1 -0
- package/dist/app/assets/{core-BrfQApxh.es-DnvCX4oH.js → core-w79IMXAG.es-Bd0UlzOL.js} +1 -1
- package/dist/app/assets/{index-Bu0ub036.js → index-5K9YjIxF.js} +117 -117
- package/dist/app/assets/{index-CkzK3JIl.js → index-C513UodQ.js} +1 -1
- package/dist/app/assets/{index-CoA6HIGS.js → index-DIgzgp69.js} +1 -1
- package/dist/app/assets/{index.umd-B6Ms2PpL.js → index.umd-BMeMPq_9.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/server.mjs +1954 -1318
- package/package.json +1 -1
- package/publisher.config.json +2 -2
- package/src/config.spec.ts +181 -66
- package/src/config.ts +68 -47
- package/src/controller/compile.controller.ts +10 -7
- package/src/controller/connection.controller.ts +79 -58
- package/src/controller/database.controller.ts +10 -7
- package/src/controller/manifest.controller.ts +23 -14
- package/src/controller/materialization.controller.ts +14 -14
- package/src/controller/model.controller.ts +35 -20
- package/src/controller/package.controller.ts +83 -49
- package/src/controller/query.controller.ts +11 -8
- package/src/controller/watch-mode.controller.ts +35 -29
- package/src/errors.ts +2 -2
- package/src/mcp/error_messages.ts +2 -2
- package/src/mcp/handler_utils.ts +23 -20
- package/src/mcp/mcp_constants.ts +1 -1
- package/src/mcp/prompts/handlers.ts +3 -3
- package/src/mcp/prompts/prompt_service.ts +5 -5
- package/src/mcp/prompts/utils.ts +12 -12
- package/src/mcp/resource_metadata.ts +3 -3
- package/src/mcp/resources/environment_resource.ts +187 -0
- package/src/mcp/resources/model_resource.ts +19 -17
- package/src/mcp/resources/notebook_resource.ts +13 -13
- package/src/mcp/resources/package_resource.ts +30 -27
- package/src/mcp/resources/query_resource.ts +15 -10
- package/src/mcp/resources/source_resource.ts +10 -10
- package/src/mcp/resources/view_resource.ts +11 -11
- package/src/mcp/server.ts +16 -14
- package/src/mcp/tools/discovery_tools.ts +67 -49
- package/src/mcp/tools/execute_query_tool.ts +14 -14
- package/src/server-old.ts +1119 -0
- package/src/server.ts +191 -159
- package/src/service/connection.spec.ts +158 -133
- package/src/service/connection.ts +42 -39
- package/src/service/connection_config.spec.ts +13 -11
- package/src/service/connection_config.ts +28 -19
- package/src/service/connection_service.spec.ts +63 -43
- package/src/service/connection_service.ts +106 -89
- package/src/service/{project.ts → environment.ts} +92 -77
- package/src/service/{project_compile.spec.ts → environment_compile.spec.ts} +1 -1
- package/src/service/{project_store.spec.ts → environment_store.spec.ts} +99 -85
- package/src/service/{project_store.ts → environment_store.ts} +368 -326
- package/src/service/manifest_service.spec.ts +15 -15
- package/src/service/manifest_service.ts +26 -21
- package/src/service/materialization_service.spec.ts +93 -59
- package/src/service/materialization_service.ts +71 -62
- package/src/service/materialized_table_gc.spec.ts +15 -15
- package/src/service/materialized_table_gc.ts +3 -3
- package/src/service/model.ts +2 -2
- package/src/service/package.spec.ts +2 -2
- package/src/service/package.ts +23 -21
- package/src/service/resolve_environment.ts +15 -0
- package/src/storage/DatabaseInterface.ts +34 -25
- package/src/storage/StorageManager.mock.ts +3 -3
- package/src/storage/StorageManager.ts +24 -23
- package/src/storage/duckdb/ConnectionRepository.ts +13 -11
- package/src/storage/duckdb/DuckDBConnection.ts +1 -1
- package/src/storage/duckdb/DuckDBManifestStore.ts +6 -6
- package/src/storage/duckdb/DuckDBRepository.ts +47 -47
- package/src/storage/duckdb/{ProjectRepository.ts → EnvironmentRepository.ts} +35 -35
- package/src/storage/duckdb/ManifestRepository.ts +21 -20
- package/src/storage/duckdb/MaterializationRepository.ts +31 -28
- package/src/storage/duckdb/PackageRepository.ts +11 -11
- package/src/storage/duckdb/manifest_store.spec.ts +2 -2
- package/src/storage/duckdb/schema.ts +61 -20
- package/src/storage/ducklake/DuckLakeManifestStore.ts +14 -14
- package/tests/fixtures/publisher.config.json +1 -1
- package/tests/harness/e2e.ts +1 -1
- package/tests/harness/mcp_test_setup.ts +1 -1
- package/tests/harness/mocks.ts +10 -8
- package/tests/harness/rest_e2e.ts +2 -2
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +259 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +4 -4
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +27 -48
- package/tests/integration/mcp/mcp_resource.integration.spec.ts +26 -35
- package/tests/unit/duckdb/attached_databases.test.ts +51 -33
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +194 -0
- package/tests/unit/ducklake/ducklake.test.ts +24 -22
- package/tests/unit/mcp/prompt_happy.test.ts +8 -8
- package/dist/app/assets/HomePage-DbZS0N7G.js +0 -1
- package/dist/app/assets/MainPage-CBuWkbmr.js +0 -2
- package/dist/app/assets/ModelPage-Bt37smot.js +0 -1
- package/dist/app/assets/PackagePage-DLZe50WG.js +0 -1
- package/dist/app/assets/ProjectPage-FQTEPXP4.js +0 -1
- package/dist/app/assets/WorkbookPage-CkAo16ar.js +0 -1
- package/src/mcp/resources/project_resource.ts +0 -184
- package/src/service/resolve_project.ts +0 -13
|
@@ -12,10 +12,10 @@ export class PackageRepository {
|
|
|
12
12
|
return new Date();
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
-
async listPackages(
|
|
15
|
+
async listPackages(environmentId: string): Promise<Package[]> {
|
|
16
16
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
17
|
-
"SELECT * FROM packages WHERE
|
|
18
|
-
[
|
|
17
|
+
"SELECT * FROM packages WHERE environment_id = ? ORDER BY name",
|
|
18
|
+
[environmentId],
|
|
19
19
|
);
|
|
20
20
|
return rows.map(this.mapToPackage);
|
|
21
21
|
}
|
|
@@ -29,12 +29,12 @@ export class PackageRepository {
|
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
async getPackageByName(
|
|
32
|
-
|
|
32
|
+
environmentId: string,
|
|
33
33
|
name: string,
|
|
34
34
|
): Promise<Package | null> {
|
|
35
35
|
const row = await this.db.get<Record<string, unknown>>(
|
|
36
|
-
"SELECT * FROM packages WHERE
|
|
37
|
-
[
|
|
36
|
+
"SELECT * FROM packages WHERE environment_id = ? AND name = ?",
|
|
37
|
+
[environmentId, name],
|
|
38
38
|
);
|
|
39
39
|
return row ? this.mapToPackage(row) : null;
|
|
40
40
|
}
|
|
@@ -46,11 +46,11 @@ export class PackageRepository {
|
|
|
46
46
|
const now = this.now();
|
|
47
47
|
|
|
48
48
|
await this.db.run(
|
|
49
|
-
`INSERT INTO packages (id,
|
|
49
|
+
`INSERT INTO packages (id, environment_id, name, description, manifest_path, metadata, created_at, updated_at)
|
|
50
50
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
51
51
|
[
|
|
52
52
|
id,
|
|
53
|
-
pkg.
|
|
53
|
+
pkg.environmentId,
|
|
54
54
|
pkg.name,
|
|
55
55
|
pkg.description || null,
|
|
56
56
|
pkg.manifestPath,
|
|
@@ -114,14 +114,14 @@ export class PackageRepository {
|
|
|
114
114
|
await this.db.run("DELETE FROM packages WHERE id = ?", [id]);
|
|
115
115
|
}
|
|
116
116
|
|
|
117
|
-
async
|
|
118
|
-
await this.db.run("DELETE FROM packages WHERE
|
|
117
|
+
async deletePackagesByEnvironmentId(id: string): Promise<void> {
|
|
118
|
+
await this.db.run("DELETE FROM packages WHERE environment_id = ?", [id]);
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
private mapToPackage(row: Record<string, unknown>): Package {
|
|
122
122
|
return {
|
|
123
123
|
id: row.id as string,
|
|
124
|
-
|
|
124
|
+
environmentId: row.environment_id as string,
|
|
125
125
|
name: row.name as string,
|
|
126
126
|
description: row.description as string | undefined,
|
|
127
127
|
manifestPath: row.manifest_path as string,
|
|
@@ -6,7 +6,7 @@ import { DuckDBManifestStore } from "./DuckDBManifestStore";
|
|
|
6
6
|
function makeEntry(overrides: Partial<ManifestEntry> = {}): ManifestEntry {
|
|
7
7
|
return {
|
|
8
8
|
id: "entry-1",
|
|
9
|
-
|
|
9
|
+
environmentId: "proj-1",
|
|
10
10
|
packageName: "pkg",
|
|
11
11
|
buildId: "build-abc",
|
|
12
12
|
tableName: "my_table",
|
|
@@ -83,7 +83,7 @@ describe("DuckDBManifestStore", () => {
|
|
|
83
83
|
expect(ctx.repository.upsertManifestEntry.calledOnce).toBe(true);
|
|
84
84
|
const arg = ctx.repository.upsertManifestEntry.firstCall.args[0];
|
|
85
85
|
expect(arg).toEqual({
|
|
86
|
-
|
|
86
|
+
environmentId: "proj-1",
|
|
87
87
|
packageName: "pkg",
|
|
88
88
|
buildId: "build-abc",
|
|
89
89
|
tableName: "tbl",
|
|
@@ -17,12 +17,21 @@ export async function initializeSchema(
|
|
|
17
17
|
);
|
|
18
18
|
await dropAllTables(db);
|
|
19
19
|
} else {
|
|
20
|
+
// TODO: Remove this during projects cleanup
|
|
21
|
+
// If a pre-rename `projects` schema is on disk, the new
|
|
22
|
+
// CREATE TABLE IF NOT EXISTS pass below would silently leave child
|
|
23
|
+
// tables on the old `project_id` column and the first query against
|
|
24
|
+
// `environment_id` would crash. Drop the legacy tables (with a loud
|
|
25
|
+
// warning) so the fresh schema can be created cleanly. This is
|
|
26
|
+
// destructive — operators upgrading should re-create their environments
|
|
27
|
+
// and packages via the API after the upgrade.
|
|
28
|
+
await dropLegacyProjectSchema(db);
|
|
20
29
|
logger.info("Creating database schema for the first time...");
|
|
21
30
|
}
|
|
22
31
|
|
|
23
|
-
//
|
|
32
|
+
// Environments table
|
|
24
33
|
await db.run(`
|
|
25
|
-
CREATE TABLE IF NOT EXISTS
|
|
34
|
+
CREATE TABLE IF NOT EXISTS environments (
|
|
26
35
|
id VARCHAR PRIMARY KEY,
|
|
27
36
|
name VARCHAR NOT NULL UNIQUE,
|
|
28
37
|
path VARCHAR NOT NULL,
|
|
@@ -37,15 +46,15 @@ export async function initializeSchema(
|
|
|
37
46
|
await db.run(`
|
|
38
47
|
CREATE TABLE IF NOT EXISTS packages (
|
|
39
48
|
id VARCHAR PRIMARY KEY,
|
|
40
|
-
|
|
49
|
+
environment_id VARCHAR NOT NULL,
|
|
41
50
|
name VARCHAR NOT NULL,
|
|
42
51
|
description VARCHAR,
|
|
43
52
|
manifest_path VARCHAR NOT NULL,
|
|
44
53
|
metadata JSON,
|
|
45
54
|
created_at TIMESTAMP NOT NULL,
|
|
46
55
|
updated_at TIMESTAMP NOT NULL,
|
|
47
|
-
FOREIGN KEY (
|
|
48
|
-
UNIQUE (
|
|
56
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
57
|
+
UNIQUE (environment_id, name)
|
|
49
58
|
)
|
|
50
59
|
`);
|
|
51
60
|
|
|
@@ -53,22 +62,22 @@ export async function initializeSchema(
|
|
|
53
62
|
await db.run(`
|
|
54
63
|
CREATE TABLE IF NOT EXISTS connections (
|
|
55
64
|
id VARCHAR PRIMARY KEY,
|
|
56
|
-
|
|
65
|
+
environment_id VARCHAR NOT NULL,
|
|
57
66
|
name VARCHAR NOT NULL,
|
|
58
67
|
type VARCHAR NOT NULL,
|
|
59
68
|
config JSON NOT NULL,
|
|
60
69
|
created_at TIMESTAMP NOT NULL,
|
|
61
70
|
updated_at TIMESTAMP NOT NULL,
|
|
62
|
-
FOREIGN KEY (
|
|
63
|
-
UNIQUE (
|
|
71
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
72
|
+
UNIQUE (environment_id, name)
|
|
64
73
|
)
|
|
65
74
|
`);
|
|
66
75
|
|
|
67
76
|
// Materializations table.
|
|
68
77
|
//
|
|
69
78
|
// `active_key` enforces at-most-one active (PENDING or RUNNING)
|
|
70
|
-
// materialization per (
|
|
71
|
-
// `{
|
|
79
|
+
// materialization per (environment, package) at the DB layer. It is set to
|
|
80
|
+
// `{environment_id}|{package_name}` while the row is active and cleared
|
|
72
81
|
// to NULL on transition to any terminal state. A unique index on
|
|
73
82
|
// `active_key` (see below) makes the insert-then-check race impossible —
|
|
74
83
|
// a second concurrent create fails with a constraint violation, which the
|
|
@@ -76,7 +85,7 @@ export async function initializeSchema(
|
|
|
76
85
|
await db.run(`
|
|
77
86
|
CREATE TABLE IF NOT EXISTS materializations (
|
|
78
87
|
id VARCHAR PRIMARY KEY,
|
|
79
|
-
|
|
88
|
+
environment_id VARCHAR NOT NULL,
|
|
80
89
|
package_name VARCHAR NOT NULL,
|
|
81
90
|
status VARCHAR NOT NULL,
|
|
82
91
|
active_key VARCHAR,
|
|
@@ -86,7 +95,7 @@ export async function initializeSchema(
|
|
|
86
95
|
metadata JSON,
|
|
87
96
|
created_at TIMESTAMP NOT NULL,
|
|
88
97
|
updated_at TIMESTAMP NOT NULL,
|
|
89
|
-
FOREIGN KEY (
|
|
98
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id)
|
|
90
99
|
)
|
|
91
100
|
`);
|
|
92
101
|
|
|
@@ -94,7 +103,7 @@ export async function initializeSchema(
|
|
|
94
103
|
await db.run(`
|
|
95
104
|
CREATE TABLE IF NOT EXISTS build_manifests (
|
|
96
105
|
id VARCHAR PRIMARY KEY,
|
|
97
|
-
|
|
106
|
+
environment_id VARCHAR NOT NULL,
|
|
98
107
|
package_name VARCHAR NOT NULL,
|
|
99
108
|
build_id VARCHAR NOT NULL,
|
|
100
109
|
table_name VARCHAR NOT NULL,
|
|
@@ -102,36 +111,68 @@ export async function initializeSchema(
|
|
|
102
111
|
connection_name VARCHAR NOT NULL,
|
|
103
112
|
created_at TIMESTAMP NOT NULL,
|
|
104
113
|
updated_at TIMESTAMP NOT NULL,
|
|
105
|
-
FOREIGN KEY (
|
|
106
|
-
UNIQUE (
|
|
114
|
+
FOREIGN KEY (environment_id) REFERENCES environments(id),
|
|
115
|
+
UNIQUE (environment_id, package_name, build_id)
|
|
107
116
|
)
|
|
108
117
|
`);
|
|
109
118
|
|
|
110
119
|
// Create indexes for better query performance
|
|
111
120
|
await db.run(
|
|
112
|
-
"CREATE INDEX IF NOT EXISTS
|
|
121
|
+
"CREATE INDEX IF NOT EXISTS idx_packages_environment_id ON packages(environment_id)",
|
|
113
122
|
);
|
|
114
123
|
await db.run(
|
|
115
|
-
"CREATE INDEX IF NOT EXISTS
|
|
124
|
+
"CREATE INDEX IF NOT EXISTS idx_connections_environment_id ON connections(environment_id)",
|
|
116
125
|
);
|
|
117
126
|
await db.run(
|
|
118
|
-
"CREATE INDEX IF NOT EXISTS
|
|
127
|
+
"CREATE INDEX IF NOT EXISTS idx_materializations_environment_package ON materializations(environment_id, package_name)",
|
|
119
128
|
);
|
|
120
129
|
await db.run(
|
|
121
130
|
"CREATE UNIQUE INDEX IF NOT EXISTS idx_materializations_active_key ON materializations(active_key)",
|
|
122
131
|
);
|
|
123
132
|
await db.run(
|
|
124
|
-
"CREATE INDEX IF NOT EXISTS
|
|
133
|
+
"CREATE INDEX IF NOT EXISTS idx_build_manifests_environment_package ON build_manifests(environment_id, package_name)",
|
|
125
134
|
);
|
|
126
135
|
}
|
|
127
136
|
|
|
137
|
+
// TODO: Remove this during projects cleanup
|
|
138
|
+
// Tables in the pre-rename schema, listed children-first so DROP order
|
|
139
|
+
// satisfies foreign-key dependencies on the legacy `projects` table.
|
|
140
|
+
const LEGACY_TABLES_DROP_ORDER = [
|
|
141
|
+
"build_manifests",
|
|
142
|
+
"materializations",
|
|
143
|
+
"packages",
|
|
144
|
+
"connections",
|
|
145
|
+
"projects",
|
|
146
|
+
] as const;
|
|
147
|
+
|
|
148
|
+
async function dropLegacyProjectSchema(db: DuckDBConnection): Promise<void> {
|
|
149
|
+
const legacy = await db.all<{ name: string }>(
|
|
150
|
+
"SELECT name FROM sqlite_master WHERE type='table' AND name='projects'",
|
|
151
|
+
);
|
|
152
|
+
if (!legacy || legacy.length === 0) {
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
logger.warn(
|
|
157
|
+
"Detected legacy 'projects' schema. Dropping legacy tables; existing environments/packages/connections/materializations data will be lost. Re-create them via the API after upgrade.",
|
|
158
|
+
);
|
|
159
|
+
|
|
160
|
+
for (const table of LEGACY_TABLES_DROP_ORDER) {
|
|
161
|
+
try {
|
|
162
|
+
await db.run(`DROP TABLE IF EXISTS ${table}`);
|
|
163
|
+
} catch (err) {
|
|
164
|
+
logger.warn(`Failed to drop legacy table ${table}:`, err);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
128
169
|
async function dropAllTables(db: DuckDBConnection): Promise<void> {
|
|
129
170
|
const tables = [
|
|
130
171
|
"build_manifests",
|
|
131
172
|
"materializations",
|
|
132
173
|
"packages",
|
|
133
174
|
"connections",
|
|
134
|
-
"
|
|
175
|
+
"environments",
|
|
135
176
|
];
|
|
136
177
|
|
|
137
178
|
logger.info("Dropping tables:", tables.join(", "));
|
|
@@ -31,15 +31,15 @@ import { DuckDBConnection } from "../duckdb/DuckDBConnection";
|
|
|
31
31
|
*/
|
|
32
32
|
export class DuckLakeManifestStore implements ManifestStore {
|
|
33
33
|
private readonly table: string;
|
|
34
|
-
private readonly
|
|
34
|
+
private readonly environmentName: string;
|
|
35
35
|
|
|
36
36
|
constructor(
|
|
37
37
|
private db: DuckDBConnection,
|
|
38
38
|
catalogName: string,
|
|
39
|
-
|
|
39
|
+
environmentName: string,
|
|
40
40
|
) {
|
|
41
41
|
this.table = `${catalogName}.build_manifests`;
|
|
42
|
-
this.
|
|
42
|
+
this.environmentName = environmentName;
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
/**
|
|
@@ -56,7 +56,7 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
56
56
|
await this.db.run(`
|
|
57
57
|
CREATE TABLE IF NOT EXISTS ${this.table} (
|
|
58
58
|
id VARCHAR,
|
|
59
|
-
|
|
59
|
+
environment_name VARCHAR NOT NULL,
|
|
60
60
|
package_name VARCHAR NOT NULL,
|
|
61
61
|
build_id VARCHAR NOT NULL,
|
|
62
62
|
table_name VARCHAR NOT NULL,
|
|
@@ -70,12 +70,12 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
async getManifest(
|
|
73
|
-
|
|
73
|
+
_environmentId: string,
|
|
74
74
|
packageName: string,
|
|
75
75
|
): Promise<BuildManifest> {
|
|
76
76
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
77
|
-
`SELECT * FROM ${this.table} WHERE
|
|
78
|
-
[this.
|
|
77
|
+
`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`,
|
|
78
|
+
[this.environmentName, packageName],
|
|
79
79
|
);
|
|
80
80
|
const manifest: BuildManifest = { entries: {}, strict: false };
|
|
81
81
|
for (const row of rows) {
|
|
@@ -97,7 +97,7 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
97
97
|
* {@link getManifest} deduplicates by build_id keeping the newest row.
|
|
98
98
|
*/
|
|
99
99
|
async writeEntry(
|
|
100
|
-
|
|
100
|
+
_environmentId: string,
|
|
101
101
|
packageName: string,
|
|
102
102
|
buildId: string,
|
|
103
103
|
tableName: string,
|
|
@@ -108,11 +108,11 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
108
108
|
const id = `${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
|
|
109
109
|
|
|
110
110
|
await this.db.run(
|
|
111
|
-
`INSERT INTO ${this.table} (id,
|
|
111
|
+
`INSERT INTO ${this.table} (id, environment_name, package_name, build_id, table_name, source_name, connection_name, created_at, updated_at)
|
|
112
112
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
113
113
|
[
|
|
114
114
|
id,
|
|
115
|
-
this.
|
|
115
|
+
this.environmentName,
|
|
116
116
|
packageName,
|
|
117
117
|
buildId,
|
|
118
118
|
tableName,
|
|
@@ -129,12 +129,12 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
129
129
|
}
|
|
130
130
|
|
|
131
131
|
async listEntries(
|
|
132
|
-
|
|
132
|
+
_environmentId: string,
|
|
133
133
|
packageName: string,
|
|
134
134
|
): Promise<ManifestEntry[]> {
|
|
135
135
|
const rows = await this.db.all<Record<string, unknown>>(
|
|
136
|
-
`SELECT * FROM ${this.table} WHERE
|
|
137
|
-
[this.
|
|
136
|
+
`SELECT * FROM ${this.table} WHERE environment_name = ? AND package_name = ? ORDER BY created_at DESC`,
|
|
137
|
+
[this.environmentName, packageName],
|
|
138
138
|
);
|
|
139
139
|
return rows.map(this.mapToEntry);
|
|
140
140
|
}
|
|
@@ -142,7 +142,7 @@ export class DuckLakeManifestStore implements ManifestStore {
|
|
|
142
142
|
private mapToEntry(row: Record<string, unknown>): ManifestEntry {
|
|
143
143
|
return {
|
|
144
144
|
id: row.id as string,
|
|
145
|
-
|
|
145
|
+
environmentId: row.environment_name as string,
|
|
146
146
|
packageName: row.package_name as string,
|
|
147
147
|
buildId: row.build_id as string,
|
|
148
148
|
tableName: row.table_name as string,
|
package/tests/harness/e2e.ts
CHANGED
|
@@ -28,7 +28,7 @@ let originalServerRoot: string | undefined;
|
|
|
28
28
|
*/
|
|
29
29
|
export async function startE2E(): Promise<E2EEnv & { stop(): Promise<void> }> {
|
|
30
30
|
//--------------------------------------------------------------------------
|
|
31
|
-
// 1. Set SERVER_ROOT so
|
|
31
|
+
// 1. Set SERVER_ROOT so EnvironmentStore loader finds publisher.config.json
|
|
32
32
|
//--------------------------------------------------------------------------
|
|
33
33
|
originalServerRoot = process.env.SERVER_ROOT;
|
|
34
34
|
// Use import.meta.url for cross-platform compatibility (works on Windows)
|
|
@@ -47,7 +47,7 @@ export async function setupE2ETestEnvironment(): Promise<McpE2ETestEnvironment>
|
|
|
47
47
|
|
|
48
48
|
async function setupE2ETestEnvironmentInternal(): Promise<McpE2ETestEnvironment> {
|
|
49
49
|
// --- Store and Set SERVER_ROOT Env Var ---
|
|
50
|
-
// The
|
|
50
|
+
// The EnvironmentStore relies on SERVER_ROOT to find publisher.config.json.
|
|
51
51
|
const originalServerRoot = process.env.SERVER_ROOT; // Store original value
|
|
52
52
|
// Resolve the path to 'packages/server' based on the location of this file
|
|
53
53
|
// Use import.meta.url for cross-platform compatibility (works on Windows)
|
package/tests/harness/mocks.ts
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import sinon from "sinon";
|
|
2
|
-
import {
|
|
2
|
+
import { EnvironmentStore } from "../../src/service/environment_store";
|
|
3
3
|
|
|
4
|
-
/** Return a stubbed
|
|
5
|
-
export function
|
|
6
|
-
const
|
|
7
|
-
// For now just have
|
|
8
|
-
|
|
9
|
-
|
|
4
|
+
/** Return a stubbed EnvironmentStore where every lookup throws or returns minimal objects. */
|
|
5
|
+
export function fakeEnvironmentStore(): sinon.SinonStubbedInstance<EnvironmentStore> {
|
|
6
|
+
const es = sinon.createStubInstance(EnvironmentStore);
|
|
7
|
+
// For now just have getEnvironment reject; suites can stub more.
|
|
8
|
+
es.getEnvironment.rejects(
|
|
9
|
+
new Error("fakeEnvironmentStore: getEnvironment not stubbed"),
|
|
10
|
+
);
|
|
11
|
+
return es;
|
|
10
12
|
}
|
|
11
13
|
|
|
12
14
|
// The runtime implementation of `McpServer` lives in the MCP SDK package, which
|
|
@@ -27,7 +29,7 @@ export type McpServer = DummyMcpServer;
|
|
|
27
29
|
/** Convenience helper mimicking the old mocks used in integration specs. */
|
|
28
30
|
export function createMalloyServiceMocks() {
|
|
29
31
|
return {
|
|
30
|
-
|
|
32
|
+
environmentStore: fakeEnvironmentStore(),
|
|
31
33
|
} as const;
|
|
32
34
|
}
|
|
33
35
|
|
|
@@ -12,8 +12,8 @@ export interface RestE2EEnv {
|
|
|
12
12
|
* reuses the cached Express app and binds on an OS-assigned port
|
|
13
13
|
* to avoid collisions.
|
|
14
14
|
*
|
|
15
|
-
* Callers are responsible for creating any test-specific
|
|
16
|
-
* via the REST API (POST /api/v0/
|
|
15
|
+
* Callers are responsible for creating any test-specific environments
|
|
16
|
+
* via the REST API (POST /api/v0/environments) and cleaning them up.
|
|
17
17
|
*/
|
|
18
18
|
export async function startRestE2E(): Promise<
|
|
19
19
|
RestE2EEnv & { stop(): Promise<void> }
|
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/// <reference types="bun-types" />
|
|
2
|
+
|
|
3
|
+
// TODO: Remove this during projects cleanup
|
|
4
|
+
/**
|
|
5
|
+
* Smoke tests for the legacy `/api/v0/projects/...` REST surface registered
|
|
6
|
+
* by `server-old.ts`. These routes exist purely to keep pre-rename SDK
|
|
7
|
+
* clients (e.g. `@malloydata/db-publisher`) working after the
|
|
8
|
+
* projects→environments rename.
|
|
9
|
+
*
|
|
10
|
+
* One test per route group: projects CRUD, packages, connections, models,
|
|
11
|
+
* notebooks, databases, queries, materializations, manifest. The
|
|
12
|
+
* materialization test additionally asserts the response field rename
|
|
13
|
+
* (`projectId` not `environmentId`). `/status` is no longer in the legacy
|
|
14
|
+
* surface — both old and new clients hit the single `/api/v0/status`
|
|
15
|
+
* handler in server.ts, which returns `environments`.
|
|
16
|
+
*
|
|
17
|
+
* This file is intentionally separate from the regular integration suite so
|
|
18
|
+
* it can be deleted in one motion when legacy support is dropped.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
22
|
+
import path from "path";
|
|
23
|
+
import { fileURLToPath } from "url";
|
|
24
|
+
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
25
|
+
|
|
26
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
27
|
+
const __dirname = path.dirname(__filename);
|
|
28
|
+
|
|
29
|
+
// Use a distinct project name so this suite doesn't collide with the
|
|
30
|
+
// materialization integration suite (which also uses "test-project") if
|
|
31
|
+
// they ever run in the same DB instance.
|
|
32
|
+
const PROJECT_NAME = "legacy-routes-test-project";
|
|
33
|
+
const PACKAGE_NAME = "persist-test";
|
|
34
|
+
|
|
35
|
+
describe("Legacy /api/v0/projects/* REST routes (E2E)", () => {
|
|
36
|
+
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
37
|
+
let baseUrl: string;
|
|
38
|
+
|
|
39
|
+
beforeAll(async () => {
|
|
40
|
+
env = await startRestE2E();
|
|
41
|
+
baseUrl = env.baseUrl;
|
|
42
|
+
|
|
43
|
+
// Create the test environment via the LEGACY route — proves POST
|
|
44
|
+
// /projects works end-to-end.
|
|
45
|
+
const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
|
|
46
|
+
const createRes = await fetch(`${baseUrl}/api/v0/projects`, {
|
|
47
|
+
method: "POST",
|
|
48
|
+
headers: { "Content-Type": "application/json" },
|
|
49
|
+
body: JSON.stringify({
|
|
50
|
+
name: PROJECT_NAME,
|
|
51
|
+
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
52
|
+
connections: [],
|
|
53
|
+
}),
|
|
54
|
+
});
|
|
55
|
+
if (!createRes.ok) {
|
|
56
|
+
const body = await createRes.text();
|
|
57
|
+
throw new Error(
|
|
58
|
+
`Failed to create test project via legacy route (${createRes.status}): ${body}`,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Wait for the package to finish loading via the legacy GET path.
|
|
63
|
+
const deadline = Date.now() + 30_000;
|
|
64
|
+
let pkgReady = false;
|
|
65
|
+
while (!pkgReady && Date.now() < deadline) {
|
|
66
|
+
try {
|
|
67
|
+
const res = await fetch(
|
|
68
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
|
|
69
|
+
);
|
|
70
|
+
if (res.ok) {
|
|
71
|
+
pkgReady = true;
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
} catch {
|
|
75
|
+
// not ready yet
|
|
76
|
+
}
|
|
77
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
78
|
+
}
|
|
79
|
+
if (!pkgReady) {
|
|
80
|
+
throw new Error("Test package did not become available in time");
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
afterAll(async () => {
|
|
85
|
+
if (baseUrl) {
|
|
86
|
+
try {
|
|
87
|
+
// Clean up via the legacy DELETE route.
|
|
88
|
+
await fetch(`${baseUrl}/api/v0/projects/${PROJECT_NAME}`, {
|
|
89
|
+
method: "DELETE",
|
|
90
|
+
});
|
|
91
|
+
} catch {
|
|
92
|
+
// best-effort cleanup
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
await env?.stop();
|
|
96
|
+
env = null;
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
describe("projects CRUD", () => {
|
|
100
|
+
it("GET /projects lists environments under the legacy URL", async () => {
|
|
101
|
+
const res = await fetch(`${baseUrl}/api/v0/projects`);
|
|
102
|
+
expect(res.status).toBe(200);
|
|
103
|
+
const body = (await res.json()) as Array<{ name?: string }>;
|
|
104
|
+
expect(Array.isArray(body)).toBe(true);
|
|
105
|
+
expect(body.some((e) => e.name === PROJECT_NAME)).toBe(true);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it("GET /projects/:projectName returns the project", async () => {
|
|
109
|
+
const res = await fetch(`${baseUrl}/api/v0/projects/${PROJECT_NAME}`);
|
|
110
|
+
expect(res.status).toBe(200);
|
|
111
|
+
const body = (await res.json()) as { name?: string };
|
|
112
|
+
expect(body.name).toBe(PROJECT_NAME);
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe("packages", () => {
|
|
117
|
+
it("GET /projects/:projectName/packages returns the package list", async () => {
|
|
118
|
+
const res = await fetch(
|
|
119
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages`,
|
|
120
|
+
);
|
|
121
|
+
expect(res.status).toBe(200);
|
|
122
|
+
const body = (await res.json()) as Array<{ name?: string }>;
|
|
123
|
+
expect(Array.isArray(body)).toBe(true);
|
|
124
|
+
expect(body.some((p) => p.name === PACKAGE_NAME)).toBe(true);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("GET /projects/:projectName/packages/:packageName returns the package", async () => {
|
|
128
|
+
const res = await fetch(
|
|
129
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}`,
|
|
130
|
+
);
|
|
131
|
+
expect(res.status).toBe(200);
|
|
132
|
+
const body = (await res.json()) as { name?: string };
|
|
133
|
+
expect(body.name).toBe(PACKAGE_NAME);
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe("connections", () => {
|
|
138
|
+
it("GET /projects/:projectName/connections returns 200 (may be empty)", async () => {
|
|
139
|
+
const res = await fetch(
|
|
140
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/connections`,
|
|
141
|
+
);
|
|
142
|
+
expect(res.status).toBe(200);
|
|
143
|
+
const body = await res.json();
|
|
144
|
+
expect(Array.isArray(body)).toBe(true);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
describe("models", () => {
|
|
149
|
+
it("GET /projects/:projectName/packages/:packageName/models returns the model list", async () => {
|
|
150
|
+
const res = await fetch(
|
|
151
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/models`,
|
|
152
|
+
);
|
|
153
|
+
expect(res.status).toBe(200);
|
|
154
|
+
const body = await res.json();
|
|
155
|
+
expect(Array.isArray(body)).toBe(true);
|
|
156
|
+
});
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
describe("notebooks", () => {
|
|
160
|
+
it("GET /projects/:projectName/packages/:packageName/notebooks returns 200", async () => {
|
|
161
|
+
const res = await fetch(
|
|
162
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/notebooks`,
|
|
163
|
+
);
|
|
164
|
+
expect(res.status).toBe(200);
|
|
165
|
+
const body = await res.json();
|
|
166
|
+
expect(Array.isArray(body)).toBe(true);
|
|
167
|
+
});
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
describe("databases", () => {
|
|
171
|
+
it("GET /projects/:projectName/packages/:packageName/databases returns 200", async () => {
|
|
172
|
+
const res = await fetch(
|
|
173
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/databases`,
|
|
174
|
+
);
|
|
175
|
+
expect(res.status).toBe(200);
|
|
176
|
+
const body = await res.json();
|
|
177
|
+
expect(Array.isArray(body)).toBe(true);
|
|
178
|
+
});
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
describe("queries", () => {
|
|
182
|
+
it("POST /projects/:projectName/packages/:packageName/models/.../query reaches the handler", async () => {
|
|
183
|
+
// Hit the route with a bogus model name. We only need to prove the
|
|
184
|
+
// legacy URL is wired up to the controller — a structured JSON
|
|
185
|
+
// error (not Express's HTML fall-through 404) is sufficient signal.
|
|
186
|
+
const res = await fetch(
|
|
187
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/models/does-not-exist.malloy/query`,
|
|
188
|
+
{
|
|
189
|
+
method: "POST",
|
|
190
|
+
headers: { "Content-Type": "application/json" },
|
|
191
|
+
body: JSON.stringify({ query: "run: nothing" }),
|
|
192
|
+
},
|
|
193
|
+
);
|
|
194
|
+
expect(res.status).toBeGreaterThanOrEqual(400);
|
|
195
|
+
expect(res.status).toBeLessThan(600);
|
|
196
|
+
// Controller errors come back as JSON with a `message` field.
|
|
197
|
+
// An unhandled Express 404 returns HTML — that would fail here.
|
|
198
|
+
const body = (await res.json()) as Record<string, unknown>;
|
|
199
|
+
expect(typeof body.message).toBe("string");
|
|
200
|
+
});
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
describe("materializations", () => {
|
|
204
|
+
it("GET list and POST create return 'projectId' (not 'environmentId') under the legacy URL", async () => {
|
|
205
|
+
const listRes = await fetch(
|
|
206
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations`,
|
|
207
|
+
);
|
|
208
|
+
expect(listRes.status).toBe(200);
|
|
209
|
+
const list = (await listRes.json()) as unknown;
|
|
210
|
+
expect(Array.isArray(list)).toBe(true);
|
|
211
|
+
|
|
212
|
+
// Create one so we can assert the field rename on a populated payload.
|
|
213
|
+
const createRes = await fetch(
|
|
214
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations`,
|
|
215
|
+
{
|
|
216
|
+
method: "POST",
|
|
217
|
+
headers: { "Content-Type": "application/json" },
|
|
218
|
+
body: JSON.stringify({ autoLoadManifest: true }),
|
|
219
|
+
},
|
|
220
|
+
);
|
|
221
|
+
expect(createRes.status).toBe(201);
|
|
222
|
+
const created = (await createRes.json()) as Record<string, unknown>;
|
|
223
|
+
|
|
224
|
+
// Legacy contract: materialization payloads expose `projectId`, not
|
|
225
|
+
// `environmentId`. This is the response remapper in server-old.ts.
|
|
226
|
+
expect(created).toHaveProperty("projectId");
|
|
227
|
+
expect(created).not.toHaveProperty("environmentId");
|
|
228
|
+
|
|
229
|
+
const id = created.id as string;
|
|
230
|
+
// Best-effort cleanup so we don't leak a PENDING materialization
|
|
231
|
+
// into other tests. We don't poll-to-terminal; the suite teardown
|
|
232
|
+
// of the project will mop up.
|
|
233
|
+
try {
|
|
234
|
+
await fetch(
|
|
235
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations/${id}?action=stop`,
|
|
236
|
+
{ method: "POST" },
|
|
237
|
+
);
|
|
238
|
+
await fetch(
|
|
239
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/materializations/${id}`,
|
|
240
|
+
{ method: "DELETE" },
|
|
241
|
+
);
|
|
242
|
+
} catch {
|
|
243
|
+
// ignore
|
|
244
|
+
}
|
|
245
|
+
});
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
describe("manifest", () => {
|
|
249
|
+
it("GET /projects/:projectName/packages/:packageName/manifest returns 200 or a structured 4xx", async () => {
|
|
250
|
+
const res = await fetch(
|
|
251
|
+
`${baseUrl}/api/v0/projects/${PROJECT_NAME}/packages/${PACKAGE_NAME}/manifest`,
|
|
252
|
+
);
|
|
253
|
+
// Without a built materialization the manifest may be empty/404 —
|
|
254
|
+
// we only assert the legacy URL reaches the handler, not 404 from
|
|
255
|
+
// Express's catch-all.
|
|
256
|
+
expect([200, 400, 404]).toContain(res.status);
|
|
257
|
+
});
|
|
258
|
+
});
|
|
259
|
+
});
|