@malloy-publisher/server 0.0.195 → 0.0.196
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +213 -214
- package/dist/app/assets/EnvironmentPage-1j6QDWAy.js +1 -0
- package/dist/app/assets/HomePage-DMop21VG.js +1 -0
- package/dist/app/assets/MainPage-BbE8ETz1.js +2 -0
- package/dist/app/assets/ModelPage-D2jvfe3t.js +1 -0
- package/dist/app/assets/PackagePage-BbnhGoD3.js +1 -0
- package/dist/app/assets/{RouteError-DefbDO7F.js → RouteError-D3LGEZ3i.js} +1 -1
- package/dist/app/assets/WorkbookPage-DttVIj4u.js +1 -0
- package/dist/app/assets/{core-BrfQApxh.es-DnvCX4oH.js → core-w79IMXAG.es-Bd0UlzOL.js} +1 -1
- package/dist/app/assets/{index-Bu0ub036.js → index-5K9YjIxF.js} +117 -117
- package/dist/app/assets/{index-CkzK3JIl.js → index-C513UodQ.js} +1 -1
- package/dist/app/assets/{index-CoA6HIGS.js → index-DIgzgp69.js} +1 -1
- package/dist/app/assets/{index.umd-B6Ms2PpL.js → index.umd-BMeMPq_9.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/server.mjs +1976 -1322
- package/package.json +2 -2
- package/publisher.config.json +2 -2
- package/src/config.spec.ts +181 -66
- package/src/config.ts +68 -47
- package/src/controller/compile.controller.ts +10 -7
- package/src/controller/connection.controller.ts +79 -58
- package/src/controller/database.controller.ts +10 -7
- package/src/controller/manifest.controller.ts +23 -14
- package/src/controller/materialization.controller.ts +14 -14
- package/src/controller/model.controller.ts +35 -20
- package/src/controller/package.controller.ts +83 -49
- package/src/controller/query.controller.ts +11 -8
- package/src/controller/watch-mode.controller.ts +35 -29
- package/src/errors.ts +2 -2
- package/src/mcp/error_messages.ts +2 -2
- package/src/mcp/handler_utils.ts +23 -20
- package/src/mcp/mcp_constants.ts +1 -1
- package/src/mcp/prompts/handlers.ts +3 -3
- package/src/mcp/prompts/prompt_service.ts +5 -5
- package/src/mcp/prompts/utils.ts +12 -12
- package/src/mcp/resource_metadata.ts +3 -3
- package/src/mcp/resources/environment_resource.ts +187 -0
- package/src/mcp/resources/model_resource.ts +19 -17
- package/src/mcp/resources/notebook_resource.ts +13 -13
- package/src/mcp/resources/package_resource.ts +30 -27
- package/src/mcp/resources/query_resource.ts +15 -10
- package/src/mcp/resources/source_resource.ts +10 -10
- package/src/mcp/resources/view_resource.ts +11 -11
- package/src/mcp/server.ts +16 -14
- package/src/mcp/tools/discovery_tools.ts +67 -49
- package/src/mcp/tools/execute_query_tool.ts +14 -14
- package/src/server-old.ts +1119 -0
- package/src/server.ts +191 -159
- package/src/service/connection.spec.ts +158 -133
- package/src/service/connection.ts +42 -39
- package/src/service/connection_config.spec.ts +13 -11
- package/src/service/connection_config.ts +28 -19
- package/src/service/connection_service.spec.ts +63 -43
- package/src/service/connection_service.ts +106 -89
- package/src/service/{project.ts → environment.ts} +92 -77
- package/src/service/{project_compile.spec.ts → environment_compile.spec.ts} +1 -1
- package/src/service/{project_store.spec.ts → environment_store.spec.ts} +99 -83
- package/src/service/{project_store.ts → environment_store.ts} +373 -327
- package/src/service/manifest_service.spec.ts +15 -15
- package/src/service/manifest_service.ts +26 -21
- package/src/service/materialization_service.spec.ts +93 -59
- package/src/service/materialization_service.ts +71 -62
- package/src/service/materialized_table_gc.spec.ts +15 -15
- package/src/service/materialized_table_gc.ts +3 -3
- package/src/service/model.ts +4 -4
- package/src/service/package.spec.ts +2 -2
- package/src/service/package.ts +23 -21
- package/src/service/resolve_environment.ts +15 -0
- package/src/storage/DatabaseInterface.ts +34 -25
- package/src/storage/StorageManager.mock.ts +3 -3
- package/src/storage/StorageManager.ts +64 -28
- package/src/storage/duckdb/ConnectionRepository.ts +13 -11
- package/src/storage/duckdb/DuckDBConnection.ts +1 -1
- package/src/storage/duckdb/DuckDBManifestStore.ts +6 -6
- package/src/storage/duckdb/DuckDBRepository.ts +47 -47
- package/src/storage/duckdb/{ProjectRepository.ts → EnvironmentRepository.ts} +35 -35
- package/src/storage/duckdb/ManifestRepository.ts +21 -20
- package/src/storage/duckdb/MaterializationRepository.ts +31 -28
- package/src/storage/duckdb/PackageRepository.ts +11 -11
- package/src/storage/duckdb/manifest_store.spec.ts +2 -2
- package/src/storage/duckdb/schema.ts +61 -20
- package/src/storage/ducklake/DuckLakeManifestStore.ts +20 -11
- package/tests/fixtures/publisher.config.json +1 -1
- package/tests/harness/e2e.ts +1 -1
- package/tests/harness/mcp_test_setup.ts +12 -24
- package/tests/harness/mocks.ts +10 -8
- package/tests/harness/rest_e2e.ts +2 -2
- package/tests/integration/legacy_routes/legacy_routes.integration.spec.ts +259 -0
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +4 -4
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +28 -49
- package/tests/integration/mcp/mcp_resource.integration.spec.ts +39 -47
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +1 -1
- package/tests/unit/duckdb/attached_databases.test.ts +51 -33
- package/tests/unit/duckdb/legacy_schema_migration.test.ts +194 -0
- package/tests/unit/ducklake/ducklake.test.ts +24 -22
- package/tests/unit/mcp/prompt_happy.test.ts +8 -8
- package/dist/app/assets/HomePage-DbZS0N7G.js +0 -1
- package/dist/app/assets/MainPage-CBuWkbmr.js +0 -2
- package/dist/app/assets/ModelPage-Bt37smot.js +0 -1
- package/dist/app/assets/PackagePage-DLZe50WG.js +0 -1
- package/dist/app/assets/ProjectPage-FQTEPXP4.js +0 -1
- package/dist/app/assets/WorkbookPage-CkAo16ar.js +0 -1
- package/src/mcp/resources/project_resource.ts +0 -184
- package/src/service/resolve_project.ts +0 -13
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { components } from "../api";
|
|
2
2
|
import { ConnectionNotFoundError, FrozenConfigError } from "../errors";
|
|
3
3
|
import { logger } from "../logger";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { buildEnvironmentMalloyConfig } from "./connection";
|
|
5
|
+
import { EnvironmentStore } from "./environment_store";
|
|
6
6
|
|
|
7
7
|
type ApiConnection = components["schemas"]["Connection"];
|
|
8
8
|
type ReleaseCallback = () => Promise<void>;
|
|
9
|
-
type
|
|
9
|
+
type ConnectionUpdateEnvironment = {
|
|
10
10
|
runConnectionUpdateExclusive?: <T>(fn: () => Promise<T>) => Promise<T>;
|
|
11
11
|
updateConnections?: (
|
|
12
|
-
nextMalloyConfig: ReturnType<typeof
|
|
12
|
+
nextMalloyConfig: ReturnType<typeof buildEnvironmentMalloyConfig>,
|
|
13
13
|
apiConnections?: ApiConnection[],
|
|
14
14
|
afterPreviousRelease?: ReleaseCallback,
|
|
15
15
|
) => void;
|
|
@@ -18,22 +18,22 @@ type ConnectionUpdateProject = {
|
|
|
18
18
|
deleteDuckLakeConnection?: (connectionName: string) => Promise<void>;
|
|
19
19
|
};
|
|
20
20
|
|
|
21
|
-
async function
|
|
22
|
-
|
|
21
|
+
async function runEnvironmentConnectionUpdate<T>(
|
|
22
|
+
environment: ConnectionUpdateEnvironment,
|
|
23
23
|
fn: () => Promise<T>,
|
|
24
24
|
): Promise<T> {
|
|
25
|
-
if (
|
|
26
|
-
return
|
|
25
|
+
if (environment.runConnectionUpdateExclusive) {
|
|
26
|
+
return environment.runConnectionUpdateExclusive(fn);
|
|
27
27
|
}
|
|
28
28
|
return fn();
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
-
function
|
|
32
|
-
|
|
33
|
-
nextMalloyConfig: ReturnType<typeof
|
|
31
|
+
function updateEnvironmentConnections(
|
|
32
|
+
environment: ConnectionUpdateEnvironment,
|
|
33
|
+
nextMalloyConfig: ReturnType<typeof buildEnvironmentMalloyConfig>,
|
|
34
34
|
afterPreviousRelease?: ReleaseCallback,
|
|
35
35
|
): void {
|
|
36
|
-
|
|
36
|
+
environment.updateConnections?.(
|
|
37
37
|
nextMalloyConfig,
|
|
38
38
|
nextMalloyConfig.apiConnections,
|
|
39
39
|
afterPreviousRelease,
|
|
@@ -41,140 +41,150 @@ function updateProjectConnections(
|
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
function buildDeletedConnectionCleanup(
|
|
44
|
-
|
|
44
|
+
environment: ConnectionUpdateEnvironment,
|
|
45
45
|
deletedConnection: ApiConnection,
|
|
46
46
|
connectionName: string,
|
|
47
47
|
): ReleaseCallback | undefined {
|
|
48
48
|
if (
|
|
49
49
|
deletedConnection.type === "duckdb" &&
|
|
50
|
-
typeof
|
|
50
|
+
typeof environment.deleteDuckDBConnection === "function"
|
|
51
51
|
) {
|
|
52
|
-
return () =>
|
|
52
|
+
return () => environment.deleteDuckDBConnection!(connectionName);
|
|
53
53
|
}
|
|
54
54
|
|
|
55
55
|
if (
|
|
56
56
|
deletedConnection.type === "ducklake" &&
|
|
57
|
-
typeof
|
|
57
|
+
typeof environment.deleteDuckLakeConnection === "function"
|
|
58
58
|
) {
|
|
59
|
-
return () =>
|
|
59
|
+
return () => environment.deleteDuckLakeConnection!(connectionName);
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
return undefined;
|
|
63
63
|
}
|
|
64
64
|
|
|
65
65
|
export class ConnectionService {
|
|
66
|
-
private
|
|
66
|
+
private environmentStore: EnvironmentStore;
|
|
67
67
|
|
|
68
|
-
constructor(
|
|
69
|
-
this.
|
|
68
|
+
constructor(environmentStore: EnvironmentStore) {
|
|
69
|
+
this.environmentStore = environmentStore;
|
|
70
70
|
}
|
|
71
71
|
|
|
72
|
-
public async getConnection(
|
|
73
|
-
await this.
|
|
72
|
+
public async getConnection(environmentName: string, connectionName: string) {
|
|
73
|
+
await this.environmentStore.finishedInitialization;
|
|
74
74
|
|
|
75
|
-
const repository = this.
|
|
76
|
-
const
|
|
75
|
+
const repository = this.environmentStore.storageManager.getRepository();
|
|
76
|
+
const dbEnvironment =
|
|
77
|
+
await repository.getEnvironmentByName(environmentName);
|
|
77
78
|
|
|
78
|
-
if (!
|
|
79
|
-
throw new Error(
|
|
79
|
+
if (!dbEnvironment) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
`Environment "${environmentName}" not found in database`,
|
|
82
|
+
);
|
|
80
83
|
}
|
|
81
84
|
|
|
82
85
|
const dbConnection = await repository.getConnectionByName(
|
|
83
|
-
|
|
86
|
+
dbEnvironment.id,
|
|
84
87
|
connectionName,
|
|
85
88
|
);
|
|
86
89
|
|
|
87
90
|
if (!dbConnection) {
|
|
88
91
|
throw new ConnectionNotFoundError(
|
|
89
|
-
`Connection "${connectionName}" not found in
|
|
92
|
+
`Connection "${connectionName}" not found in environment "${environmentName}"`,
|
|
90
93
|
);
|
|
91
94
|
}
|
|
92
95
|
|
|
93
|
-
return {
|
|
96
|
+
return { dbEnvironment, dbConnection, repository };
|
|
94
97
|
}
|
|
95
98
|
|
|
96
99
|
public async addConnection(
|
|
97
|
-
|
|
100
|
+
environmentName: string,
|
|
98
101
|
connectionName: string,
|
|
99
102
|
connection: ApiConnection,
|
|
100
103
|
): Promise<void> {
|
|
101
|
-
await this.
|
|
104
|
+
await this.environmentStore.finishedInitialization;
|
|
102
105
|
|
|
103
|
-
if (this.
|
|
106
|
+
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
104
107
|
throw new FrozenConfigError();
|
|
105
108
|
}
|
|
106
109
|
|
|
107
110
|
logger.info(
|
|
108
|
-
`Adding connection "${connectionName}" to
|
|
111
|
+
`Adding connection "${connectionName}" to environment "${environmentName}"`,
|
|
109
112
|
);
|
|
110
113
|
|
|
111
|
-
// Get database
|
|
112
|
-
const repository = this.
|
|
113
|
-
const
|
|
114
|
+
// Get database environment record and repository
|
|
115
|
+
const repository = this.environmentStore.storageManager.getRepository();
|
|
116
|
+
const dbEnvironment =
|
|
117
|
+
await repository.getEnvironmentByName(environmentName);
|
|
114
118
|
|
|
115
|
-
if (!
|
|
116
|
-
throw new Error(
|
|
119
|
+
if (!dbEnvironment) {
|
|
120
|
+
throw new Error(
|
|
121
|
+
`Environment "${environmentName}" not found in database`,
|
|
122
|
+
);
|
|
117
123
|
}
|
|
118
124
|
|
|
119
125
|
// Check if connection already exists in database
|
|
120
126
|
const existingDbConn = await repository.getConnectionByName(
|
|
121
|
-
|
|
127
|
+
dbEnvironment.id,
|
|
122
128
|
connectionName!,
|
|
123
129
|
);
|
|
124
130
|
|
|
125
131
|
if (existingDbConn) {
|
|
126
132
|
throw new Error(
|
|
127
|
-
`Connection "${connectionName}" already exists in
|
|
133
|
+
`Connection "${connectionName}" already exists in environment "${environmentName}".`,
|
|
128
134
|
);
|
|
129
135
|
}
|
|
130
136
|
|
|
131
137
|
// Update in-memory connections
|
|
132
|
-
const
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
138
|
+
const environment = await this.environmentStore.getEnvironment(
|
|
139
|
+
environmentName,
|
|
140
|
+
false,
|
|
141
|
+
);
|
|
142
|
+
await runEnvironmentConnectionUpdate(environment, async () => {
|
|
143
|
+
const existingConnections = environment.listApiConnections();
|
|
144
|
+
const nextMalloyConfig = buildEnvironmentMalloyConfig(
|
|
136
145
|
[...existingConnections, connection],
|
|
137
|
-
|
|
146
|
+
environment.metadata.location || "",
|
|
138
147
|
);
|
|
139
148
|
|
|
140
|
-
await this.
|
|
149
|
+
await this.environmentStore.addConnection(
|
|
141
150
|
connection,
|
|
142
|
-
|
|
151
|
+
dbEnvironment.id,
|
|
143
152
|
repository,
|
|
144
153
|
);
|
|
145
154
|
|
|
146
|
-
|
|
155
|
+
updateEnvironmentConnections(environment, nextMalloyConfig);
|
|
147
156
|
});
|
|
148
157
|
|
|
149
158
|
logger.info(
|
|
150
|
-
`Successfully added connection "${connection.name}" to
|
|
159
|
+
`Successfully added connection "${connection.name}" to environment "${environmentName}"`,
|
|
151
160
|
);
|
|
152
161
|
}
|
|
153
162
|
|
|
154
163
|
public async updateConnection(
|
|
155
|
-
|
|
164
|
+
environmentName: string,
|
|
156
165
|
connectionName: string,
|
|
157
166
|
connection: Partial<ApiConnection>,
|
|
158
167
|
): Promise<void> {
|
|
159
|
-
await this.
|
|
168
|
+
await this.environmentStore.finishedInitialization;
|
|
160
169
|
|
|
161
|
-
if (this.
|
|
170
|
+
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
162
171
|
throw new FrozenConfigError();
|
|
163
172
|
}
|
|
164
173
|
|
|
165
174
|
logger.info(
|
|
166
|
-
`Updating connection "${connectionName}" in
|
|
175
|
+
`Updating connection "${connectionName}" in environment "${environmentName}"`,
|
|
167
176
|
);
|
|
168
177
|
|
|
169
|
-
const {
|
|
170
|
-
|
|
171
|
-
connectionName,
|
|
172
|
-
);
|
|
178
|
+
const { dbEnvironment, dbConnection, repository } =
|
|
179
|
+
await this.getConnection(environmentName, connectionName);
|
|
173
180
|
|
|
174
181
|
// Update in-memory connections
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
|
|
182
|
+
const environment = await this.environmentStore.getEnvironment(
|
|
183
|
+
environmentName,
|
|
184
|
+
false,
|
|
185
|
+
);
|
|
186
|
+
await runEnvironmentConnectionUpdate(environment, async () => {
|
|
187
|
+
const existingConnections = environment.listApiConnections();
|
|
178
188
|
|
|
179
189
|
const updatedConnection = {
|
|
180
190
|
...dbConnection.config,
|
|
@@ -182,93 +192,100 @@ export class ConnectionService {
|
|
|
182
192
|
name: connectionName,
|
|
183
193
|
};
|
|
184
194
|
|
|
185
|
-
const updatedConnections = existingConnections.map(
|
|
186
|
-
conn
|
|
195
|
+
const updatedConnections = existingConnections.map(
|
|
196
|
+
(conn: ApiConnection) =>
|
|
197
|
+
conn.name === connectionName ? updatedConnection : conn,
|
|
187
198
|
);
|
|
188
199
|
|
|
189
200
|
// Pass isUpdateConnectionRequest=true so the DuckLake wrapper
|
|
190
201
|
// re-attaches against the updated catalog/storage settings instead
|
|
191
202
|
// of trusting the prior generation's persisted attach state.
|
|
192
|
-
const nextMalloyConfig =
|
|
203
|
+
const nextMalloyConfig = buildEnvironmentMalloyConfig(
|
|
193
204
|
updatedConnections,
|
|
194
|
-
|
|
205
|
+
environment.metadata.location || "",
|
|
195
206
|
true,
|
|
196
207
|
);
|
|
197
208
|
|
|
198
|
-
await this.
|
|
209
|
+
await this.environmentStore.updateConnection(
|
|
199
210
|
updatedConnection,
|
|
200
|
-
|
|
211
|
+
dbEnvironment.id,
|
|
201
212
|
repository,
|
|
202
213
|
);
|
|
203
214
|
|
|
204
|
-
|
|
215
|
+
updateEnvironmentConnections(environment, nextMalloyConfig);
|
|
205
216
|
});
|
|
206
217
|
|
|
207
218
|
logger.info(
|
|
208
|
-
`Successfully updated connection "${connectionName}" in
|
|
219
|
+
`Successfully updated connection "${connectionName}" in environment "${environmentName}"`,
|
|
209
220
|
);
|
|
210
221
|
}
|
|
211
222
|
|
|
212
223
|
public async deleteConnection(
|
|
213
|
-
|
|
224
|
+
environmentName: string,
|
|
214
225
|
connectionName: string,
|
|
215
226
|
): Promise<void> {
|
|
216
|
-
await this.
|
|
227
|
+
await this.environmentStore.finishedInitialization;
|
|
217
228
|
|
|
218
|
-
if (this.
|
|
229
|
+
if (this.environmentStore.publisherConfigIsFrozen) {
|
|
219
230
|
throw new FrozenConfigError();
|
|
220
231
|
}
|
|
221
232
|
|
|
222
233
|
logger.info(
|
|
223
|
-
`Deleting connection "${connectionName}" from
|
|
234
|
+
`Deleting connection "${connectionName}" from environment "${environmentName}"`,
|
|
224
235
|
);
|
|
225
236
|
|
|
226
237
|
const { dbConnection, repository } = await this.getConnection(
|
|
227
|
-
|
|
238
|
+
environmentName,
|
|
228
239
|
connectionName,
|
|
229
240
|
);
|
|
230
241
|
|
|
231
242
|
// Update in-memory connections
|
|
232
|
-
const
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
243
|
+
const environment = await this.environmentStore.getEnvironment(
|
|
244
|
+
environmentName,
|
|
245
|
+
false,
|
|
246
|
+
);
|
|
247
|
+
await runEnvironmentConnectionUpdate(environment, async () => {
|
|
248
|
+
if (typeof environment.listApiConnections !== "function") {
|
|
249
|
+
if (typeof environment.deleteConnection === "function") {
|
|
250
|
+
await environment.deleteConnection(connectionName);
|
|
237
251
|
}
|
|
238
252
|
await repository.deleteConnection(dbConnection.id);
|
|
239
253
|
return;
|
|
240
254
|
}
|
|
241
255
|
|
|
242
256
|
const deletedConnection =
|
|
243
|
-
"getApiConnection" in
|
|
244
|
-
typeof
|
|
245
|
-
?
|
|
257
|
+
"getApiConnection" in environment &&
|
|
258
|
+
typeof environment.getApiConnection === "function"
|
|
259
|
+
? environment.getApiConnection(connectionName)
|
|
246
260
|
: dbConnection.config;
|
|
247
|
-
const updatedConnections =
|
|
261
|
+
const updatedConnections = environment
|
|
248
262
|
.listApiConnections()
|
|
249
|
-
.filter(
|
|
250
|
-
|
|
263
|
+
.filter(
|
|
264
|
+
(connection: ApiConnection) =>
|
|
265
|
+
connection.name !== connectionName,
|
|
266
|
+
);
|
|
267
|
+
const nextMalloyConfig = buildEnvironmentMalloyConfig(
|
|
251
268
|
updatedConnections,
|
|
252
|
-
|
|
269
|
+
environment.metadata.location || "",
|
|
253
270
|
);
|
|
254
271
|
const deleteConnectionFilesAfterRelease =
|
|
255
272
|
buildDeletedConnectionCleanup(
|
|
256
|
-
|
|
273
|
+
environment,
|
|
257
274
|
deletedConnection,
|
|
258
275
|
connectionName,
|
|
259
276
|
);
|
|
260
277
|
|
|
261
278
|
await repository.deleteConnection(dbConnection.id);
|
|
262
279
|
|
|
263
|
-
|
|
264
|
-
|
|
280
|
+
updateEnvironmentConnections(
|
|
281
|
+
environment,
|
|
265
282
|
nextMalloyConfig,
|
|
266
283
|
deleteConnectionFilesAfterRelease,
|
|
267
284
|
);
|
|
268
285
|
});
|
|
269
286
|
|
|
270
287
|
logger.info(
|
|
271
|
-
`Successfully deleted connection "${connectionName}" from
|
|
288
|
+
`Successfully deleted connection "${connectionName}" from environment "${environmentName}"`,
|
|
272
289
|
);
|
|
273
290
|
}
|
|
274
291
|
}
|