@malloy-publisher/server 0.0.226 → 0.0.227
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -11
- package/dist/package_load_worker.mjs +86 -24
- package/dist/server.mjs +655 -6994
- package/package.json +1 -4
- package/src/health.ts +3 -8
- package/src/mcp/error_messages.ts +8 -37
- package/src/mcp/handler_utils.ts +10 -129
- package/src/mcp/mcp_constants.ts +0 -16
- package/src/mcp/{agent_server.protocol.spec.ts → server.protocol.spec.ts} +14 -12
- package/src/mcp/server.ts +29 -37
- package/src/mcp/skills/build_skills_bundle.ts +1 -1
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/docs_search_tool.ts +1 -1
- package/src/mcp/tools/execute_query_tool.ts +2 -2
- package/src/mcp/tools/get_context_eval.ts +3 -3
- package/src/mcp/tools/get_context_tool.spec.ts +196 -1
- package/src/mcp/tools/get_context_tool.ts +165 -67
- package/src/package_load/package_load_pool.ts +3 -0
- package/src/package_load/package_load_worker.ts +68 -24
- package/src/package_load/protocol.ts +16 -0
- package/src/package_load/rpc_wait_accountant.spec.ts +109 -0
- package/src/package_load/rpc_wait_accountant.ts +76 -0
- package/src/package_load_metrics.spec.ts +114 -0
- package/src/package_load_metrics.ts +127 -0
- package/src/pg_helpers.spec.ts +2 -206
- package/src/pg_helpers.ts +4 -120
- package/src/server.ts +0 -16
- package/src/service/environment.ts +1 -1
- package/src/service/package.ts +82 -42
- package/src/test_helpers/metrics_harness.ts +40 -0
- package/tests/harness/mcp_test_setup.ts +1 -1
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +7 -31
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -6
- package/dxt/malloy_bridge.py +0 -354
- package/dxt/manifest.json +0 -22
- package/src/dto/connection.dto.spec.ts +0 -186
- package/src/dto/connection.dto.ts +0 -308
- package/src/dto/index.ts +0 -2
- package/src/dto/package.dto.spec.ts +0 -42
- package/src/dto/package.dto.ts +0 -27
- package/src/dto/validate.spec.ts +0 -76
- package/src/dto/validate.ts +0 -31
- package/src/ducklake_version.spec.ts +0 -43
- package/src/ducklake_version.ts +0 -26
- package/src/mcp/agent_server.spec.ts +0 -18
- package/src/mcp/agent_server.ts +0 -144
- package/src/mcp/prompts/handlers.ts +0 -84
- package/src/mcp/prompts/index.ts +0 -11
- package/src/mcp/prompts/prompt_definitions.ts +0 -160
- package/src/mcp/prompts/prompt_service.ts +0 -67
- package/src/mcp/prompts/utils.ts +0 -62
- package/src/mcp/resource_metadata.ts +0 -47
- package/src/mcp/resources/environment_resource.ts +0 -187
- package/src/mcp/resources/model_resource.ts +0 -155
- package/src/mcp/resources/notebook_resource.ts +0 -137
- package/src/mcp/resources/package_resource.ts +0 -373
- package/src/mcp/resources/query_resource.ts +0 -122
- package/src/mcp/resources/source_resource.ts +0 -141
- package/src/mcp/resources/view_resource.ts +0 -136
- package/src/mcp/tools/discovery_tools.ts +0 -280
- package/src/storage/BaseRepository.ts +0 -31
- package/src/storage/StorageManager.mock.ts +0 -50
- package/tests/harness/e2e.ts +0 -96
- package/tests/harness/mocks.ts +0 -39
- package/tests/harness/uris.ts +0 -31
- package/tests/integration/mcp/mcp_resource.integration.spec.ts +0 -655
- package/tests/integration/mcp/setup.spec.ts +0 -5
- package/tests/unit/mcp/prompt_definitions.test.ts +0 -102
- package/tests/unit/mcp/prompt_happy.test.ts +0 -51
package/src/pg_helpers.spec.ts
CHANGED
|
@@ -1,105 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
classifyPgError,
|
|
5
|
-
handlePgAttachError,
|
|
6
|
-
pgConnectTimeoutSeconds,
|
|
7
|
-
redactPgSecrets,
|
|
8
|
-
withPgConnectTimeout,
|
|
9
|
-
} from "./pg_helpers";
|
|
10
|
-
|
|
11
|
-
describe("pgConnectTimeoutSeconds", () => {
|
|
12
|
-
const ORIGINAL_TIMEOUT = process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
13
|
-
|
|
14
|
-
afterEach(() => {
|
|
15
|
-
if (ORIGINAL_TIMEOUT === undefined) {
|
|
16
|
-
delete process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
17
|
-
} else {
|
|
18
|
-
process.env.PG_CONNECT_TIMEOUT_SECONDS = ORIGINAL_TIMEOUT;
|
|
19
|
-
}
|
|
20
|
-
});
|
|
21
|
-
|
|
22
|
-
it("defaults to 5 when env unset", () => {
|
|
23
|
-
delete process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
24
|
-
expect(pgConnectTimeoutSeconds()).toBe(5);
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
it("honors PG_CONNECT_TIMEOUT_SECONDS override", () => {
|
|
28
|
-
process.env.PG_CONNECT_TIMEOUT_SECONDS = "12";
|
|
29
|
-
expect(pgConnectTimeoutSeconds()).toBe(12);
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it("falls back to 5 when env value is invalid", () => {
|
|
33
|
-
process.env.PG_CONNECT_TIMEOUT_SECONDS = "not-a-number";
|
|
34
|
-
expect(pgConnectTimeoutSeconds()).toBe(5);
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
it("falls back to 5 when env value is zero or negative", () => {
|
|
38
|
-
process.env.PG_CONNECT_TIMEOUT_SECONDS = "0";
|
|
39
|
-
expect(pgConnectTimeoutSeconds()).toBe(5);
|
|
40
|
-
process.env.PG_CONNECT_TIMEOUT_SECONDS = "-3";
|
|
41
|
-
expect(pgConnectTimeoutSeconds()).toBe(5);
|
|
42
|
-
});
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
describe("withPgConnectTimeout", () => {
|
|
46
|
-
it("appends to keyword form when missing", () => {
|
|
47
|
-
expect(withPgConnectTimeout("host=h dbname=d user=u password=p", 5)).toBe(
|
|
48
|
-
"host=h dbname=d user=u password=p connect_timeout=5",
|
|
49
|
-
);
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
it("appends to postgres: keyword form (DuckLake catalogUrl shape)", () => {
|
|
53
|
-
expect(
|
|
54
|
-
withPgConnectTimeout("postgres:host=h user=u password=p dbname=d", 5),
|
|
55
|
-
).toBe("postgres:host=h user=u password=p dbname=d connect_timeout=5");
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
it("does not override a user-supplied connect_timeout in keyword form", () => {
|
|
59
|
-
expect(withPgConnectTimeout("host=h connect_timeout=30", 99)).toBe(
|
|
60
|
-
"host=h connect_timeout=30",
|
|
61
|
-
);
|
|
62
|
-
});
|
|
63
|
-
|
|
64
|
-
it("appends to URI form with no query", () => {
|
|
65
|
-
expect(withPgConnectTimeout("postgresql://u:p@h:5432/d", 5)).toBe(
|
|
66
|
-
"postgresql://u:p@h:5432/d?connect_timeout=5",
|
|
67
|
-
);
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
it("appends to URI form with existing query", () => {
|
|
71
|
-
expect(
|
|
72
|
-
withPgConnectTimeout("postgresql://u:p@h/d?sslmode=require", 5),
|
|
73
|
-
).toBe("postgresql://u:p@h/d?sslmode=require&connect_timeout=5");
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
it("appends to URI with bare trailing ?", () => {
|
|
77
|
-
expect(withPgConnectTimeout("postgresql://h/d?", 5)).toBe(
|
|
78
|
-
"postgresql://h/d?connect_timeout=5",
|
|
79
|
-
);
|
|
80
|
-
});
|
|
81
|
-
|
|
82
|
-
it("does not double-append when URI already has connect_timeout (?-style)", () => {
|
|
83
|
-
expect(
|
|
84
|
-
withPgConnectTimeout("postgresql://h/d?connect_timeout=10", 5),
|
|
85
|
-
).toBe("postgresql://h/d?connect_timeout=10");
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
it("does not double-append when URI already has connect_timeout (&-style)", () => {
|
|
89
|
-
expect(
|
|
90
|
-
withPgConnectTimeout(
|
|
91
|
-
"postgresql://h/d?sslmode=require&connect_timeout=10",
|
|
92
|
-
5,
|
|
93
|
-
),
|
|
94
|
-
).toBe("postgresql://h/d?sslmode=require&connect_timeout=10");
|
|
95
|
-
});
|
|
96
|
-
|
|
97
|
-
it("recognizes postgres:// (alternative scheme) as URI form", () => {
|
|
98
|
-
expect(withPgConnectTimeout("postgres://u@h/d", 5)).toBe(
|
|
99
|
-
"postgres://u@h/d?connect_timeout=5",
|
|
100
|
-
);
|
|
101
|
-
});
|
|
102
|
-
});
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { redactPgSecrets } from "./pg_helpers";
|
|
103
3
|
|
|
104
4
|
describe("redactPgSecrets", () => {
|
|
105
5
|
it("redacts bare password values", () => {
|
|
@@ -120,107 +20,3 @@ describe("redactPgSecrets", () => {
|
|
|
120
20
|
);
|
|
121
21
|
});
|
|
122
22
|
});
|
|
123
|
-
|
|
124
|
-
describe("classifyPgError", () => {
|
|
125
|
-
it.each([
|
|
126
|
-
'password authentication failed for user "alice"',
|
|
127
|
-
"no pg_hba.conf entry for host",
|
|
128
|
-
'role "alice" does not exist',
|
|
129
|
-
'database "billing" does not exist',
|
|
130
|
-
"permission denied for relation foo",
|
|
131
|
-
])("classifies '%s' as auth error", (msg) => {
|
|
132
|
-
const result = classifyPgError(new Error(msg), "PG attach");
|
|
133
|
-
expect(result).toBeInstanceOf(ConnectionAuthError);
|
|
134
|
-
expect(result?.message).toContain("PG attach:");
|
|
135
|
-
});
|
|
136
|
-
|
|
137
|
-
it("returns undefined for unrelated errors", () => {
|
|
138
|
-
expect(
|
|
139
|
-
classifyPgError(
|
|
140
|
-
new Error('relation "users" does not exist'),
|
|
141
|
-
"PG attach",
|
|
142
|
-
),
|
|
143
|
-
).toBeUndefined();
|
|
144
|
-
expect(
|
|
145
|
-
classifyPgError(new Error("connection reset by peer"), "PG attach"),
|
|
146
|
-
).toBeUndefined();
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
it("returns undefined for non-Error values", () => {
|
|
150
|
-
expect(
|
|
151
|
-
classifyPgError("password authentication failed", "ctx"),
|
|
152
|
-
).toBeUndefined();
|
|
153
|
-
expect(classifyPgError(undefined, "ctx")).toBeUndefined();
|
|
154
|
-
});
|
|
155
|
-
|
|
156
|
-
it("redacts embedded passwords in the wrapped message", () => {
|
|
157
|
-
const result = classifyPgError(
|
|
158
|
-
new Error(
|
|
159
|
-
"password authentication failed: tried host=h password=hunter2",
|
|
160
|
-
),
|
|
161
|
-
"DuckLake attach",
|
|
162
|
-
);
|
|
163
|
-
expect(result?.message).toContain("password=***");
|
|
164
|
-
expect(result?.message).not.toContain("hunter2");
|
|
165
|
-
});
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
describe("handlePgAttachError", () => {
|
|
169
|
-
it("swallows 'already exists' errors", () => {
|
|
170
|
-
const outcome = handlePgAttachError(
|
|
171
|
-
new Error('database "db_x" already exists'),
|
|
172
|
-
"ctx",
|
|
173
|
-
);
|
|
174
|
-
expect(outcome.action).toBe("swallow");
|
|
175
|
-
});
|
|
176
|
-
|
|
177
|
-
it("swallows 'already attached' errors", () => {
|
|
178
|
-
const outcome = handlePgAttachError(
|
|
179
|
-
new Error("DuckLake catalog db_x is already attached"),
|
|
180
|
-
"ctx",
|
|
181
|
-
);
|
|
182
|
-
expect(outcome.action).toBe("swallow");
|
|
183
|
-
});
|
|
184
|
-
|
|
185
|
-
it("classifies libpq auth failures as ConnectionAuthError", () => {
|
|
186
|
-
const outcome = handlePgAttachError(
|
|
187
|
-
new Error('password authentication failed for user "alice"'),
|
|
188
|
-
"PG attach db_x",
|
|
189
|
-
);
|
|
190
|
-
expect(outcome.action).toBe("throw");
|
|
191
|
-
if (outcome.action === "throw") {
|
|
192
|
-
expect(outcome.error).toBeInstanceOf(ConnectionAuthError);
|
|
193
|
-
expect(outcome.error.message).toContain("PG attach db_x:");
|
|
194
|
-
}
|
|
195
|
-
});
|
|
196
|
-
|
|
197
|
-
it("passes through unrelated Error instances unchanged", () => {
|
|
198
|
-
const original = new Error("network unreachable");
|
|
199
|
-
const outcome = handlePgAttachError(original, "ctx");
|
|
200
|
-
expect(outcome.action).toBe("throw");
|
|
201
|
-
if (outcome.action === "throw") {
|
|
202
|
-
expect(outcome.error).toBe(original);
|
|
203
|
-
expect(outcome.error).not.toBeInstanceOf(ConnectionAuthError);
|
|
204
|
-
}
|
|
205
|
-
});
|
|
206
|
-
|
|
207
|
-
it("wraps non-Error throwables so callers always get an Error", () => {
|
|
208
|
-
const outcome = handlePgAttachError("a string was thrown", "ctx");
|
|
209
|
-
expect(outcome.action).toBe("throw");
|
|
210
|
-
if (outcome.action === "throw") {
|
|
211
|
-
expect(outcome.error).toBeInstanceOf(Error);
|
|
212
|
-
expect(outcome.error.message).toBe("a string was thrown");
|
|
213
|
-
}
|
|
214
|
-
});
|
|
215
|
-
|
|
216
|
-
it("prefers 'already attached' over auth classification when both keywords appear", () => {
|
|
217
|
-
// Defensive: if a future DuckDB version emits a combined message,
|
|
218
|
-
// 'already attached' wins so we don't bubble up a false auth failure
|
|
219
|
-
// on what is actually a benign idempotent re-attach.
|
|
220
|
-
const outcome = handlePgAttachError(
|
|
221
|
-
new Error("already attached; permission denied tail"),
|
|
222
|
-
"ctx",
|
|
223
|
-
);
|
|
224
|
-
expect(outcome.action).toBe("swallow");
|
|
225
|
-
});
|
|
226
|
-
});
|
package/src/pg_helpers.ts
CHANGED
|
@@ -1,64 +1,7 @@
|
|
|
1
|
-
// Postgres / libpq
|
|
2
|
-
//
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
// matters.
|
|
6
|
-
import { ConnectionAuthError } from "./errors";
|
|
7
|
-
|
|
8
|
-
// Default Postgres connect_timeout (seconds), used by the materialization
|
|
9
|
-
// storage catalog ATTACH so a slow or wedged libpq handshake fails the
|
|
10
|
-
// caller in seconds instead of stalling the worker until the K8s liveness
|
|
11
|
-
// probe trips.
|
|
12
|
-
//
|
|
13
|
-
// libpq enforces a documented minimum of 2 seconds — values below 2
|
|
14
|
-
// effectively round up to ~2s wall clock.
|
|
15
|
-
export function pgConnectTimeoutSeconds(): number {
|
|
16
|
-
const raw = process.env.PG_CONNECT_TIMEOUT_SECONDS;
|
|
17
|
-
if (!raw) return 5;
|
|
18
|
-
const parsed = Number.parseInt(raw, 10);
|
|
19
|
-
return Number.isFinite(parsed) && parsed > 0 ? parsed : 5;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
// libpq accepts both keyword=value form ("host=h dbname=d") and URI form
|
|
23
|
-
// ("postgresql://u:p@h/d?param=v"). The materialization-storage catalogUrl
|
|
24
|
-
// can also arrive as `postgres:<keyword=value>` (no `//`). We detect URI
|
|
25
|
-
// form (with `//`) so we know whether to append a new parameter using
|
|
26
|
-
// `?`/`&` or a leading space.
|
|
27
|
-
const URI_FORM_RE = /^[a-z][a-z0-9+.-]*:\/\//i;
|
|
28
|
-
|
|
29
|
-
// Match an existing connect_timeout key in either form. URI form uses
|
|
30
|
-
// `?key=` or `&key=`; keyword form uses whitespace separation or start-of-
|
|
31
|
-
// string. Without the `[?&]` alternatives a URI-form user-supplied timeout
|
|
32
|
-
// would be missed and we'd double-append, producing an invalid URL.
|
|
33
|
-
const HAS_CONNECT_TIMEOUT_RE = /[?&\s]connect_timeout=|^connect_timeout=/;
|
|
34
|
-
|
|
35
|
-
// Append `connect_timeout=N` to a libpq-compatible connection string if
|
|
36
|
-
// the caller hasn't already set one. Handles keyword form ("host=h ..."),
|
|
37
|
-
// URI form ("postgresql://..."), and the `postgres:host=h ...` keyword
|
|
38
|
-
// form with a scheme prefix used by DuckLake catalogUrls.
|
|
39
|
-
export function withPgConnectTimeout(
|
|
40
|
-
connectionString: string,
|
|
41
|
-
timeout: number,
|
|
42
|
-
): string {
|
|
43
|
-
if (HAS_CONNECT_TIMEOUT_RE.test(connectionString)) {
|
|
44
|
-
return connectionString;
|
|
45
|
-
}
|
|
46
|
-
if (URI_FORM_RE.test(connectionString)) {
|
|
47
|
-
// URI form: append as query parameter. `?` if no query string yet,
|
|
48
|
-
// `&` otherwise. A bare trailing `?` (empty query) gets no extra
|
|
49
|
-
// separator. We don't try to handle URL fragments — libpq URIs don't
|
|
50
|
-
// use them.
|
|
51
|
-
if (!connectionString.includes("?")) {
|
|
52
|
-
return `${connectionString}?connect_timeout=${timeout}`;
|
|
53
|
-
}
|
|
54
|
-
if (connectionString.endsWith("?")) {
|
|
55
|
-
return `${connectionString}connect_timeout=${timeout}`;
|
|
56
|
-
}
|
|
57
|
-
return `${connectionString}&connect_timeout=${timeout}`;
|
|
58
|
-
}
|
|
59
|
-
// Keyword=value form (with or without `postgres:` scheme prefix).
|
|
60
|
-
return `${connectionString} connect_timeout=${timeout}`;
|
|
61
|
-
}
|
|
1
|
+
// Postgres / libpq helper shared between `service/` (user-facing connections)
|
|
2
|
+
// and `storage/` (materialization-storage catalog). Lives at `src/` root so
|
|
3
|
+
// neither layer takes a dependency on the other — see CLAUDE.md's "Two
|
|
4
|
+
// parallel DuckLake/PG attach paths" note for why this matters.
|
|
62
5
|
|
|
63
6
|
// Redact libpq `password=...` values from a string before it goes into a
|
|
64
7
|
// log line or HTTP response body. Handles bare and quoted values.
|
|
@@ -68,62 +11,3 @@ export function withPgConnectTimeout(
|
|
|
68
11
|
export function redactPgSecrets(s: string): string {
|
|
69
12
|
return s.replace(/password=('[^']*'|"[^"]*"|\S+)/gi, "password=***");
|
|
70
13
|
}
|
|
71
|
-
|
|
72
|
-
// Substring-match libpq error patterns that indicate a non-retryable
|
|
73
|
-
// auth/permission failure. Returns a ConnectionAuthError when matched so
|
|
74
|
-
// callers can fast-fail with HTTP 422 (semantically "the supplied creds
|
|
75
|
-
// are bad; don't retry") instead of letting the raw error fall through to
|
|
76
|
-
// a generic 500 that retry loops treat as transient.
|
|
77
|
-
export function classifyPgError(
|
|
78
|
-
error: unknown,
|
|
79
|
-
context: string,
|
|
80
|
-
): ConnectionAuthError | undefined {
|
|
81
|
-
if (!(error instanceof Error)) return undefined;
|
|
82
|
-
const msg = error.message;
|
|
83
|
-
const patterns = [
|
|
84
|
-
/password authentication failed/i,
|
|
85
|
-
/pg_hba\.conf/i,
|
|
86
|
-
/role ".*" does not exist/i,
|
|
87
|
-
/database ".*" does not exist/i,
|
|
88
|
-
/permission denied/i,
|
|
89
|
-
];
|
|
90
|
-
if (!patterns.some((p) => p.test(msg))) return undefined;
|
|
91
|
-
return new ConnectionAuthError(`${context}: ${redactPgSecrets(msg)}`);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// Outcome of inspecting an error thrown by an `ATTACH` call:
|
|
95
|
-
// - `{ action: "swallow" }`: DuckDB reported the db is already attached
|
|
96
|
-
// (idempotent re-attach); caller should log and continue.
|
|
97
|
-
// - `{ action: "throw", error: ConnectionAuthError }`: classified as a
|
|
98
|
-
// non-retryable auth failure; caller should warn-log and throw it.
|
|
99
|
-
// - `{ action: "throw", error: <original> }`: unrecognized; caller
|
|
100
|
-
// should rethrow as-is to preserve the original cause for diagnosis.
|
|
101
|
-
//
|
|
102
|
-
// Extracted so the decision tree gets a direct unit test without needing
|
|
103
|
-
// to stub DuckDB or run a real ATTACH.
|
|
104
|
-
export type PgAttachErrorOutcome =
|
|
105
|
-
| { action: "swallow" }
|
|
106
|
-
| { action: "throw"; error: Error };
|
|
107
|
-
|
|
108
|
-
export function handlePgAttachError(
|
|
109
|
-
error: unknown,
|
|
110
|
-
context: string,
|
|
111
|
-
): PgAttachErrorOutcome {
|
|
112
|
-
if (
|
|
113
|
-
error instanceof Error &&
|
|
114
|
-
(error.message.includes("already exists") ||
|
|
115
|
-
error.message.includes("already attached"))
|
|
116
|
-
) {
|
|
117
|
-
return { action: "swallow" };
|
|
118
|
-
}
|
|
119
|
-
const authErr = classifyPgError(error, context);
|
|
120
|
-
if (authErr) {
|
|
121
|
-
return { action: "throw", error: authErr };
|
|
122
|
-
}
|
|
123
|
-
if (error instanceof Error) {
|
|
124
|
-
return { action: "throw", error };
|
|
125
|
-
}
|
|
126
|
-
// Non-Error thrown values get wrapped so the catch contract stays
|
|
127
|
-
// (always throws an Error).
|
|
128
|
-
return { action: "throw", error: new Error(String(error)) };
|
|
129
|
-
}
|
package/src/server.ts
CHANGED
|
@@ -41,7 +41,6 @@ import { queryConcurrency } from "./query_concurrency";
|
|
|
41
41
|
import { MaterializationController } from "./controller/materialization.controller";
|
|
42
42
|
import { ThemeController } from "./controller/theme.controller";
|
|
43
43
|
import { initializeMcpServer } from "./mcp/server";
|
|
44
|
-
import { startAgentMcpServer } from "./mcp/agent_server";
|
|
45
44
|
import { registerLegacyRoutes } from "./server-old";
|
|
46
45
|
import { EnvironmentStore } from "./service/environment_store";
|
|
47
46
|
import { MaterializationService } from "./service/materialization_service";
|
|
@@ -50,8 +49,6 @@ import { PackageMemoryGovernor } from "./service/package_memory_governor";
|
|
|
50
49
|
import { ThemeStore } from "./service/theme_store";
|
|
51
50
|
import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
|
|
52
51
|
|
|
53
|
-
export { normalizeQueryArray } from "./query_param_utils";
|
|
54
|
-
|
|
55
52
|
// Parse command line arguments
|
|
56
53
|
function parseArgs() {
|
|
57
54
|
const args = process.argv.slice(2);
|
|
@@ -162,7 +159,6 @@ parseArgs();
|
|
|
162
159
|
const PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
|
|
163
160
|
const PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
|
|
164
161
|
const MCP_PORT = Number(process.env.MCP_PORT || 4040);
|
|
165
|
-
const AGENT_MCP_PORT = Number(process.env.AGENT_MCP_PORT || 4041);
|
|
166
162
|
const MCP_ENDPOINT = "/mcp";
|
|
167
163
|
const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
|
|
168
164
|
process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0,
|
|
@@ -1868,21 +1864,9 @@ mcpServer.timeout = 600000;
|
|
|
1868
1864
|
mcpServer.keepAliveTimeout = 600000;
|
|
1869
1865
|
mcpServer.headersTimeout = 600000;
|
|
1870
1866
|
|
|
1871
|
-
// Separate, isolated MCP server for the agent retrieval tools (get_context,
|
|
1872
|
-
// search_docs) on its own listener. Kept apart from the core MCP server above.
|
|
1873
|
-
const agentMcpServer = startAgentMcpServer(
|
|
1874
|
-
environmentStore,
|
|
1875
|
-
PUBLISHER_HOST,
|
|
1876
|
-
AGENT_MCP_PORT,
|
|
1877
|
-
);
|
|
1878
|
-
agentMcpServer.timeout = 600000;
|
|
1879
|
-
agentMcpServer.keepAliveTimeout = 600000;
|
|
1880
|
-
agentMcpServer.headersTimeout = 600000;
|
|
1881
|
-
|
|
1882
1867
|
registerSignalHandlers(
|
|
1883
1868
|
mainServer,
|
|
1884
1869
|
mcpServer,
|
|
1885
1870
|
SHUTDOWN_DRAIN_DURATION_SECONDS,
|
|
1886
1871
|
SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS,
|
|
1887
|
-
agentMcpServer,
|
|
1888
1872
|
);
|
|
@@ -792,7 +792,7 @@ export class Environment {
|
|
|
792
792
|
// reloadAllModelsForPackage, ...) acquire the lock themselves.
|
|
793
793
|
//
|
|
794
794
|
// INVARIANT: callers that consume the returned Package on the fast
|
|
795
|
-
// path (notably MCP
|
|
795
|
+
// path (notably the MCP query tools and Model.getModel()) must
|
|
796
796
|
// remain in-memory only. If any code reachable from a `Package`
|
|
797
797
|
// method ever grows new disk I/O against the canonical tree, that
|
|
798
798
|
// path needs to be bracketed by `withPackageLock`; otherwise a
|
package/src/service/package.ts
CHANGED
|
@@ -29,6 +29,11 @@ import {
|
|
|
29
29
|
} from "../errors";
|
|
30
30
|
import { formatDuration, logger } from "../logger";
|
|
31
31
|
import { recordBuildPlanComputeDuration } from "../materialization_metrics";
|
|
32
|
+
import {
|
|
33
|
+
LOAD_DURATION_BUCKETS_MS,
|
|
34
|
+
recordPackageLoadPhases,
|
|
35
|
+
type PackageLoadStatus,
|
|
36
|
+
} from "../package_load_metrics";
|
|
32
37
|
import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
|
|
33
38
|
import {
|
|
34
39
|
BuildManifest,
|
|
@@ -73,6 +78,23 @@ function toTableNameManifest(
|
|
|
73
78
|
return out;
|
|
74
79
|
}
|
|
75
80
|
|
|
81
|
+
/**
|
|
82
|
+
* Classify a failed package load into the `status` label shared by the
|
|
83
|
+
* `malloy_package_load_duration` histogram and the per-phase load metrics, so
|
|
84
|
+
* both slice failures identically. A real Malloy/model compile error is a 4xx
|
|
85
|
+
* `compilation_error`; a rewrapped pool-infrastructure failure is a transient
|
|
86
|
+
* `pool_unavailable`; anything else is a generic `error`.
|
|
87
|
+
*/
|
|
88
|
+
function packageLoadFailureStatus(error: unknown): PackageLoadStatus {
|
|
89
|
+
if (error instanceof ModelCompilationError || error instanceof MalloyError) {
|
|
90
|
+
return "compilation_error";
|
|
91
|
+
}
|
|
92
|
+
if (error instanceof ServiceUnavailableError) {
|
|
93
|
+
return "pool_unavailable";
|
|
94
|
+
}
|
|
95
|
+
return "error";
|
|
96
|
+
}
|
|
97
|
+
|
|
76
98
|
export class Package {
|
|
77
99
|
private environmentName: string;
|
|
78
100
|
private packageName: string;
|
|
@@ -127,6 +149,10 @@ export class Package {
|
|
|
127
149
|
{
|
|
128
150
|
description: "Time taken to load a Malloy package",
|
|
129
151
|
unit: "ms",
|
|
152
|
+
// OTel's default buckets top out at 10s, censoring the slow-load tail.
|
|
153
|
+
// Use the shared load-duration buckets (→5min) so p95/p99 of large
|
|
154
|
+
// package loads are resolvable. See LOAD_DURATION_BUCKETS_MS.
|
|
155
|
+
advice: { explicitBucketBoundaries: LOAD_DURATION_BUCKETS_MS },
|
|
130
156
|
},
|
|
131
157
|
);
|
|
132
158
|
|
|
@@ -248,16 +274,9 @@ export class Package {
|
|
|
248
274
|
console.error(error);
|
|
249
275
|
const endTime = performance.now();
|
|
250
276
|
const executionTime = endTime - startTime;
|
|
251
|
-
const status =
|
|
252
|
-
error instanceof ModelCompilationError ||
|
|
253
|
-
error instanceof MalloyError
|
|
254
|
-
? "compilation_error"
|
|
255
|
-
: error instanceof ServiceUnavailableError
|
|
256
|
-
? "pool_unavailable"
|
|
257
|
-
: "error";
|
|
258
277
|
this.packageLoadHistogram.record(executionTime, {
|
|
259
278
|
malloy_package_name: packageName,
|
|
260
|
-
status,
|
|
279
|
+
status: packageLoadFailureStatus(error),
|
|
261
280
|
});
|
|
262
281
|
// Clean up the package directory on failure, but NOT when packagePath
|
|
263
282
|
// is an in-place mount symlink (watch mode). Removing it would unmount
|
|
@@ -361,6 +380,10 @@ export class Package {
|
|
|
361
380
|
workerDurationMs: outcome.loadDurationMs,
|
|
362
381
|
dispatchOverheadMs:
|
|
363
382
|
workerDoneTime - dispatchTime - outcome.loadDurationMs,
|
|
383
|
+
// Phase split of the worker duration (remainder = setup + extraction).
|
|
384
|
+
compileDurationMs: outcome.timings.compileDurationMs,
|
|
385
|
+
schemaFetchDurationMs: outcome.timings.schemaFetchDurationMs,
|
|
386
|
+
schemaFetchCount: outcome.timings.schemaFetchCount,
|
|
364
387
|
modelCount: outcome.models.length,
|
|
365
388
|
databaseCount: databases.length,
|
|
366
389
|
});
|
|
@@ -400,43 +423,59 @@ export class Package {
|
|
|
400
423
|
// hydration path.)
|
|
401
424
|
const models = new Map<string, Model>();
|
|
402
425
|
const renderTagWarnings: ApiPackageWarning[] = [];
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
426
|
+
try {
|
|
427
|
+
for (const sm of outcome.models) {
|
|
428
|
+
if (sm.compilationError) {
|
|
429
|
+
const err = Model.deserializeCompilationError(
|
|
430
|
+
sm.compilationError,
|
|
431
|
+
);
|
|
432
|
+
logger.error("Model compilation failed", {
|
|
433
|
+
packageName,
|
|
434
|
+
modelPath: sm.modelPath,
|
|
435
|
+
error: err.message,
|
|
436
|
+
});
|
|
437
|
+
// The outer catch in Package.create records the total metric +
|
|
438
|
+
// cleans the package directory.
|
|
439
|
+
throw err;
|
|
440
|
+
}
|
|
441
|
+
const model = Model.fromSerialized(
|
|
407
442
|
packageName,
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
// The outer catch in Package.create records the metric +
|
|
412
|
-
// cleans the package directory.
|
|
413
|
-
throw err;
|
|
414
|
-
}
|
|
415
|
-
const model = Model.fromSerialized(
|
|
416
|
-
packageName,
|
|
417
|
-
packagePath,
|
|
418
|
-
malloyConfig,
|
|
419
|
-
sm,
|
|
420
|
-
);
|
|
421
|
-
// Validate renderer tags on the main thread (the renderer is too heavy
|
|
422
|
-
// to load inside the pure-CPU package-load worker). A misconfigured tag
|
|
423
|
-
// is logged as a warning naming the target; it does not fail the load.
|
|
424
|
-
// The findings also ride the package response as non-fatal `warnings`.
|
|
425
|
-
for (const w of await model.validateRenderTags()) {
|
|
426
|
-
renderTagWarnings.push({ model: sm.modelPath, ...w });
|
|
427
|
-
}
|
|
428
|
-
// Reject unquoted `#@ persist name=` annotations the same way: an
|
|
429
|
-
// unquoted name is dropped from the build plan, so the source would
|
|
430
|
-
// publish but never materialize. Scan the raw `.malloy` source (the
|
|
431
|
-
// ground truth for quoting); throws a ModelCompilationError (424).
|
|
432
|
-
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
433
|
-
const modelSource = await fs.readFile(
|
|
434
|
-
path.join(packagePath, sm.modelPath),
|
|
435
|
-
"utf-8",
|
|
443
|
+
packagePath,
|
|
444
|
+
malloyConfig,
|
|
445
|
+
sm,
|
|
436
446
|
);
|
|
437
|
-
|
|
447
|
+
// Validate renderer tags on the main thread (the renderer is too
|
|
448
|
+
// heavy to load inside the pure-CPU package-load worker). A
|
|
449
|
+
// misconfigured tag is logged as a warning naming the target; it
|
|
450
|
+
// does not fail the load. The findings also ride the package
|
|
451
|
+
// response as non-fatal `warnings`.
|
|
452
|
+
for (const w of await model.validateRenderTags()) {
|
|
453
|
+
renderTagWarnings.push({ model: sm.modelPath, ...w });
|
|
454
|
+
}
|
|
455
|
+
// Reject unquoted `#@ persist name=` annotations the same way: an
|
|
456
|
+
// unquoted name is dropped from the build plan, so the source would
|
|
457
|
+
// publish but never materialize. Scan the raw `.malloy` source (the
|
|
458
|
+
// ground truth for quoting); throws a ModelCompilationError (424).
|
|
459
|
+
if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
|
|
460
|
+
const modelSource = await fs.readFile(
|
|
461
|
+
path.join(packagePath, sm.modelPath),
|
|
462
|
+
"utf-8",
|
|
463
|
+
);
|
|
464
|
+
assertPersistNamesQuoted(modelSource, sm.modelPath);
|
|
465
|
+
}
|
|
466
|
+
models.set(sm.modelPath, model);
|
|
438
467
|
}
|
|
439
|
-
|
|
468
|
+
} catch (err) {
|
|
469
|
+
// Record the load's phase cost tagged with the terminal status before
|
|
470
|
+
// the error propagates to the outer catch (which records the total).
|
|
471
|
+
// Only in-band compile failures reach here — the worker already
|
|
472
|
+
// produced `outcome.timings`; a pool failure throws before `outcome`
|
|
473
|
+
// exists and carries no timings, so it's simply not recorded.
|
|
474
|
+
recordPackageLoadPhases(
|
|
475
|
+
outcome.timings,
|
|
476
|
+
packageLoadFailureStatus(err),
|
|
477
|
+
);
|
|
478
|
+
throw err;
|
|
440
479
|
}
|
|
441
480
|
|
|
442
481
|
const endTime = performance.now();
|
|
@@ -445,6 +484,7 @@ export class Package {
|
|
|
445
484
|
malloy_package_name: packageName,
|
|
446
485
|
status: "success",
|
|
447
486
|
});
|
|
487
|
+
recordPackageLoadPhases(outcome.timings, "success");
|
|
448
488
|
logger.info(`Successfully loaded package ${packageName}`, {
|
|
449
489
|
packageName,
|
|
450
490
|
duration: formatDuration(executionTime),
|
|
@@ -62,6 +62,15 @@ export interface MetricsHarness {
|
|
|
62
62
|
* fired for this gauge yet.
|
|
63
63
|
*/
|
|
64
64
|
collectGauge(name: string): Promise<number | undefined>;
|
|
65
|
+
/**
|
|
66
|
+
* Force a collection cycle and return the aggregate `count`/`sum` across all
|
|
67
|
+
* data points for the named histogram (optionally scoped to one label set).
|
|
68
|
+
* `count` is 0 when the histogram has not been emitted yet.
|
|
69
|
+
*/
|
|
70
|
+
collectHistogram(
|
|
71
|
+
name: string,
|
|
72
|
+
attributeFilter?: Record<string, string | number | boolean>,
|
|
73
|
+
): Promise<{ count: number; sum: number; boundaries: number[] }>;
|
|
65
74
|
shutdown(): Promise<void>;
|
|
66
75
|
}
|
|
67
76
|
|
|
@@ -100,6 +109,37 @@ export async function startMetricsHarness(): Promise<MetricsHarness> {
|
|
|
100
109
|
}
|
|
101
110
|
return total;
|
|
102
111
|
},
|
|
112
|
+
async collectHistogram(
|
|
113
|
+
name: string,
|
|
114
|
+
attributeFilter?: Record<string, string | number | boolean>,
|
|
115
|
+
): Promise<{ count: number; sum: number; boundaries: number[] }> {
|
|
116
|
+
const result = await reader.collect();
|
|
117
|
+
let count = 0;
|
|
118
|
+
let sum = 0;
|
|
119
|
+
let boundaries: number[] = [];
|
|
120
|
+
for (const rm of result.resourceMetrics.scopeMetrics) {
|
|
121
|
+
for (const metric of rm.metrics) {
|
|
122
|
+
if (metric.descriptor.name !== name) continue;
|
|
123
|
+
for (const dp of metric.dataPoints) {
|
|
124
|
+
if (attributeFilter) {
|
|
125
|
+
const allMatch = Object.entries(attributeFilter).every(
|
|
126
|
+
([k, v]) => dp.attributes?.[k] === v,
|
|
127
|
+
);
|
|
128
|
+
if (!allMatch) continue;
|
|
129
|
+
}
|
|
130
|
+
const point = dp.value as {
|
|
131
|
+
count: number;
|
|
132
|
+
sum?: number;
|
|
133
|
+
buckets?: { boundaries: number[] };
|
|
134
|
+
};
|
|
135
|
+
count += point.count;
|
|
136
|
+
sum += point.sum ?? 0;
|
|
137
|
+
if (point.buckets) boundaries = point.buckets.boundaries;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
return { count, sum, boundaries };
|
|
142
|
+
},
|
|
103
143
|
async collectGauge(name: string): Promise<number | undefined> {
|
|
104
144
|
const result = await reader.collect();
|
|
105
145
|
for (const rm of result.resourceMetrics.scopeMetrics) {
|
|
@@ -30,7 +30,7 @@ let portCounter = 0;
|
|
|
30
30
|
|
|
31
31
|
// True mutex: the previous Promise-based lock had a TOCTOU race where two beforeAll hooks
|
|
32
32
|
// could both pass the `if (initializationLock)` check and run setup concurrently, sharing
|
|
33
|
-
// one publisher.db / server singleton and causing flaky
|
|
33
|
+
// one publisher.db / server singleton and causing flaky tool calls and port collisions.
|
|
34
34
|
const e2eSetupMutex = new Mutex();
|
|
35
35
|
|
|
36
36
|
/**
|