@malloy-publisher/server 0.0.225 → 0.0.227
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -11
- package/dist/app/api-doc.yaml +29 -24
- package/dist/app/assets/{EnvironmentPage-DYTeXDll.js → EnvironmentPage-DvOJ7L_b.js} +1 -1
- package/dist/app/assets/{HomePage-pDK2BPJY.js → HomePage-CXguJsXS.js} +1 -1
- package/dist/app/assets/{LightMode-C2bwGPY1.js → LightMode-ZsshUznu.js} +1 -1
- package/dist/app/assets/{MainPage-WtBulMH_.js → MainPage-BIe0VwBa.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-hMgOtflG.js → MaterializationsPage-BuZ6UJVx.js} +1 -1
- package/dist/app/assets/{ModelPage-B2N5kYII.js → ModelPage-DsPf-s8B.js} +1 -1
- package/dist/app/assets/{PackagePage-CEN90nQG.js → PackagePage-CEVNAKZa.js} +1 -1
- package/dist/app/assets/{RouteError-BG2c5Zf0.js → RouteError-Chn7lL96.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DNfeUwEZ.js → ThemeEditorPage-DWC_FdNU.js} +1 -1
- package/dist/app/assets/{WorkbookPage-NKI1BhFS.js → WorkbookPage-CGrsFz8p.js} +1 -1
- package/dist/app/assets/{core-C6anj5c0.es-DDLHqpzt.js → core-vVgoO8IR.es-BD_THWs_.js} +1 -1
- package/dist/app/assets/{index-DMQtnaf4.js → index-BioohWQj.js} +1 -1
- package/dist/app/assets/{index-CzNqKMVl.js → index-D6YtyiJ0.js} +1 -1
- package/dist/app/assets/{index-C6gZ6sSY.js → index-DNUZpnaa.js} +4 -4
- package/dist/app/assets/{index-JXgvyZna.js → index-gEWxu09x.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +71 -15
- package/dist/package_load_worker.mjs +169 -40
- package/dist/server.mjs +782 -7050
- package/package.json +1 -4
- package/src/controller/package.controller.spec.ts +17 -17
- package/src/controller/package.controller.ts +7 -6
- package/src/health.ts +3 -8
- package/src/logger.spec.ts +193 -0
- package/src/logger.ts +115 -18
- package/src/mcp/error_messages.ts +8 -37
- package/src/mcp/handler_utils.ts +10 -129
- package/src/mcp/mcp_constants.ts +0 -16
- package/src/mcp/{agent_server.protocol.spec.ts → server.protocol.spec.ts} +14 -12
- package/src/mcp/server.ts +29 -37
- package/src/mcp/skills/build_skills_bundle.ts +1 -1
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/docs_search_tool.ts +1 -1
- package/src/mcp/tools/execute_query_tool.ts +2 -2
- package/src/mcp/tools/get_context_eval.ts +3 -3
- package/src/mcp/tools/get_context_tool.spec.ts +196 -1
- package/src/mcp/tools/get_context_tool.ts +165 -67
- package/src/package_load/package_load_pool.ts +8 -1
- package/src/package_load/package_load_worker.ts +75 -24
- package/src/package_load/protocol.ts +21 -1
- package/src/package_load/rpc_wait_accountant.spec.ts +109 -0
- package/src/package_load/rpc_wait_accountant.ts +76 -0
- package/src/package_load_metrics.spec.ts +114 -0
- package/src/package_load_metrics.ts +127 -0
- package/src/pg_helpers.spec.ts +2 -206
- package/src/pg_helpers.ts +4 -120
- package/src/server.ts +0 -16
- package/src/service/build_plan.spec.ts +42 -60
- package/src/service/build_plan.ts +37 -61
- package/src/service/environment.ts +5 -1
- package/src/service/materialization_test_fixtures.ts +6 -8
- package/src/service/package.ts +165 -86
- package/src/service/package_manifest.spec.ts +22 -1
- package/src/service/package_manifest.ts +29 -0
- package/src/service/persistence_policy.spec.ts +234 -0
- package/src/test_helpers/metrics_harness.ts +40 -0
- package/tests/harness/mcp_test_setup.ts +1 -1
- package/tests/integration/mcp/mcp_transport.integration.spec.ts +7 -31
- package/tests/integration/watch-mode/watch_mode.integration.spec.ts +0 -6
- package/dxt/malloy_bridge.py +0 -354
- package/dxt/manifest.json +0 -22
- package/src/dto/connection.dto.spec.ts +0 -186
- package/src/dto/connection.dto.ts +0 -308
- package/src/dto/index.ts +0 -2
- package/src/dto/package.dto.spec.ts +0 -42
- package/src/dto/package.dto.ts +0 -27
- package/src/dto/validate.spec.ts +0 -76
- package/src/dto/validate.ts +0 -31
- package/src/ducklake_version.spec.ts +0 -43
- package/src/ducklake_version.ts +0 -26
- package/src/mcp/agent_server.spec.ts +0 -18
- package/src/mcp/agent_server.ts +0 -144
- package/src/mcp/prompts/handlers.ts +0 -84
- package/src/mcp/prompts/index.ts +0 -11
- package/src/mcp/prompts/prompt_definitions.ts +0 -160
- package/src/mcp/prompts/prompt_service.ts +0 -67
- package/src/mcp/prompts/utils.ts +0 -62
- package/src/mcp/resource_metadata.ts +0 -47
- package/src/mcp/resources/environment_resource.ts +0 -187
- package/src/mcp/resources/model_resource.ts +0 -155
- package/src/mcp/resources/notebook_resource.ts +0 -137
- package/src/mcp/resources/package_resource.ts +0 -373
- package/src/mcp/resources/query_resource.ts +0 -122
- package/src/mcp/resources/source_resource.ts +0 -141
- package/src/mcp/resources/view_resource.ts +0 -136
- package/src/mcp/tools/discovery_tools.ts +0 -280
- package/src/service/materialization_cron_gate.spec.ts +0 -181
- package/src/storage/BaseRepository.ts +0 -31
- package/src/storage/StorageManager.mock.ts +0 -50
- package/tests/harness/e2e.ts +0 -96
- package/tests/harness/mocks.ts +0 -39
- package/tests/harness/uris.ts +0 -31
- package/tests/integration/mcp/mcp_resource.integration.spec.ts +0 -655
- package/tests/integration/mcp/setup.spec.ts +0 -5
- package/tests/unit/mcp/prompt_definitions.test.ts +0 -102
- package/tests/unit/mcp/prompt_happy.test.ts +0 -51
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@malloy-publisher/server",
|
|
3
3
|
"description": "Malloy Publisher Server",
|
|
4
|
-
"version": "0.0.
|
|
4
|
+
"version": "0.0.227",
|
|
5
5
|
"main": "dist/server.mjs",
|
|
6
6
|
"bin": {
|
|
7
7
|
"malloy-publisher": "dist/server.mjs"
|
|
@@ -58,8 +58,6 @@
|
|
|
58
58
|
"aws-sdk": "^2.1692.0",
|
|
59
59
|
"body-parser": "^1.20.2",
|
|
60
60
|
"chokidar": "^4.0.3",
|
|
61
|
-
"class-transformer": "^0.5.1",
|
|
62
|
-
"class-validator": "^0.14.1",
|
|
63
61
|
"cors": "^2.8.5",
|
|
64
62
|
"express": "^4.21.0",
|
|
65
63
|
"extract-zip": "^2.0.1",
|
|
@@ -74,7 +72,6 @@
|
|
|
74
72
|
"uuid": "^11.0.3"
|
|
75
73
|
},
|
|
76
74
|
"devDependencies": {
|
|
77
|
-
"@anthropic-ai/dxt": "^0.1.0",
|
|
78
75
|
"@eslint/compat": "^1.2.7",
|
|
79
76
|
"@eslint/eslintrc": "^3.3.1",
|
|
80
77
|
"@eslint/js": "^9.23.0",
|
|
@@ -18,7 +18,7 @@ describe("PackageController.addPackage explores validation", () => {
|
|
|
18
18
|
"Invalid explores entry 'missing.malloy' in publisher.json: file not found";
|
|
19
19
|
const mockPackage = {
|
|
20
20
|
formatInvalidExplores: () => invalidMsg,
|
|
21
|
-
|
|
21
|
+
formatInvalidPersistencePolicy: () => "",
|
|
22
22
|
};
|
|
23
23
|
const unloadPackage = sinon.stub().resolves(undefined);
|
|
24
24
|
const deletePackage = sinon.stub().resolves(undefined);
|
|
@@ -55,7 +55,7 @@ describe("PackageController.addPackage explores validation", () => {
|
|
|
55
55
|
"Invalid explores entry 'missing.malloy' in publisher.json: file not found";
|
|
56
56
|
const mockPackage = {
|
|
57
57
|
formatInvalidExplores: () => invalidMsg,
|
|
58
|
-
|
|
58
|
+
formatInvalidPersistencePolicy: () => "",
|
|
59
59
|
};
|
|
60
60
|
// installPackage mimics the real contract: invoke the validator and, if it
|
|
61
61
|
// returns a message, throw BadRequestError (after its internal rollback).
|
|
@@ -104,7 +104,7 @@ describe("PackageController.addPackage explores validation", () => {
|
|
|
104
104
|
it("persists when explores are valid (no-location)", async () => {
|
|
105
105
|
const mockPackage = {
|
|
106
106
|
formatInvalidExplores: () => "",
|
|
107
|
-
|
|
107
|
+
formatInvalidPersistencePolicy: () => "",
|
|
108
108
|
};
|
|
109
109
|
const addPackage = sinon.stub().resolves(mockPackage);
|
|
110
110
|
const getEnvironment = sinon.stub().resolves({ addPackage });
|
|
@@ -127,22 +127,22 @@ describe("PackageController.addPackage explores validation", () => {
|
|
|
127
127
|
});
|
|
128
128
|
});
|
|
129
129
|
|
|
130
|
-
describe("PackageController.addPackage
|
|
130
|
+
describe("PackageController.addPackage persistence policy validation", () => {
|
|
131
131
|
afterEach(() => {
|
|
132
132
|
sinon.restore();
|
|
133
133
|
});
|
|
134
134
|
|
|
135
|
-
it("rejects a publish whose
|
|
136
|
-
// Valid explores but an invalid
|
|
137
|
-
// still 400 (strict-at-publish,
|
|
138
|
-
// warns) and roll back via
|
|
135
|
+
it("rejects a publish whose persistence policy is invalid (no-location path)", async () => {
|
|
136
|
+
// Valid explores but an invalid persistence policy (e.g. a schedule on a
|
|
137
|
+
// package-scoped package): the publish must still 400 (strict-at-publish,
|
|
138
|
+
// same split as explores — load merely warns) and roll back via
|
|
139
|
+
// unloadPackage.
|
|
139
140
|
const cronMsg =
|
|
140
|
-
|
|
141
|
-
"
|
|
142
|
-
"'orders' resolves to unset.";
|
|
141
|
+
'materialization.schedule (cron) in publisher.json requires "scope": ' +
|
|
142
|
+
'"version".';
|
|
143
143
|
const mockPackage = {
|
|
144
144
|
formatInvalidExplores: () => "",
|
|
145
|
-
|
|
145
|
+
formatInvalidPersistencePolicy: () => cronMsg,
|
|
146
146
|
};
|
|
147
147
|
const unloadPackage = sinon.stub().resolves(undefined);
|
|
148
148
|
const addPackage = sinon.stub().resolves(mockPackage);
|
|
@@ -165,13 +165,13 @@ describe("PackageController.addPackage cron schedule validation", () => {
|
|
|
165
165
|
expect(addPackageToDatabase.called).toBe(false);
|
|
166
166
|
});
|
|
167
167
|
|
|
168
|
-
it("location path: the
|
|
168
|
+
it("location path: the persistence-policy gate runs inside installPackage's rollback window", async () => {
|
|
169
169
|
const cronMsg =
|
|
170
|
-
|
|
171
|
-
"
|
|
170
|
+
'materialization.schedule (cron) in publisher.json requires "scope": ' +
|
|
171
|
+
'"version".';
|
|
172
172
|
const mockPackage = {
|
|
173
173
|
formatInvalidExplores: () => "",
|
|
174
|
-
|
|
174
|
+
formatInvalidPersistencePolicy: () => cronMsg,
|
|
175
175
|
};
|
|
176
176
|
const installPackage = sinon
|
|
177
177
|
.stub()
|
|
@@ -225,7 +225,7 @@ describe("PackageController.updatePackage explores validation", () => {
|
|
|
225
225
|
const mockPackage = {
|
|
226
226
|
formatInvalidExplores: (override?: string[]) =>
|
|
227
227
|
override?.includes("nope.malloy") ? invalidMsg : "",
|
|
228
|
-
|
|
228
|
+
formatInvalidPersistencePolicy: () => "",
|
|
229
229
|
};
|
|
230
230
|
const installPackage = sinon
|
|
231
231
|
.stub()
|
|
@@ -8,21 +8,22 @@ type ApiPackage = components["schemas"]["Package"];
|
|
|
8
8
|
/**
|
|
9
9
|
* Everything that is strict-at-publish, joined into one 400 message (or
|
|
10
10
|
* undefined when the package is publishable): invalid explores entries plus
|
|
11
|
-
* the
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
11
|
+
* the Malloy Persistence policy gate (scope is package-level; a
|
|
12
|
+
* `materialization.schedule` is package-root-only + version-scope-only and
|
|
13
|
+
* mutually exclusive with freshness; per-source `sharing`/`schedule` are
|
|
14
|
+
* retired — see Package.persistencePolicyWarnings). At startup/reload both are
|
|
15
|
+
* warn-only instead (fail-safe; see Package.loadViaWorker).
|
|
15
16
|
*/
|
|
16
17
|
function formatPublishRejections(
|
|
17
18
|
pkg: {
|
|
18
19
|
formatInvalidExplores(exploresOverride?: string[]): string;
|
|
19
|
-
|
|
20
|
+
formatInvalidPersistencePolicy(): string;
|
|
20
21
|
},
|
|
21
22
|
exploresOverride?: string[],
|
|
22
23
|
): string | undefined {
|
|
23
24
|
const message = [
|
|
24
25
|
pkg.formatInvalidExplores(exploresOverride),
|
|
25
|
-
pkg.
|
|
26
|
+
pkg.formatInvalidPersistencePolicy(),
|
|
26
27
|
]
|
|
27
28
|
.filter(Boolean)
|
|
28
29
|
.join("\n");
|
package/src/health.ts
CHANGED
|
@@ -74,10 +74,10 @@ export function registerSignalHandlers(
|
|
|
74
74
|
mcpServer: Server,
|
|
75
75
|
shutdownDrainDurationSeconds: number = 0,
|
|
76
76
|
shutdownGracefulCloseTimeoutSeconds: number = 0,
|
|
77
|
-
agentMcpServer?: Server,
|
|
78
77
|
): void {
|
|
79
|
-
//
|
|
80
|
-
// K8s will SIGKILL after terminationGracePeriodSeconds
|
|
78
|
+
// On SIGTERM, drain in-flight requests before closing the servers (see the
|
|
79
|
+
// shutdown sequence above). K8s will SIGKILL after terminationGracePeriodSeconds
|
|
80
|
+
// if this handler does not exit first.
|
|
81
81
|
process.once("SIGTERM", async () => {
|
|
82
82
|
logger.info("========== SIGTERM RECEIVED ==========");
|
|
83
83
|
markNotReady();
|
|
@@ -97,7 +97,6 @@ export function registerSignalHandlers(
|
|
|
97
97
|
server,
|
|
98
98
|
mcpServer,
|
|
99
99
|
shutdownGracefulCloseTimeoutSeconds,
|
|
100
|
-
agentMcpServer,
|
|
101
100
|
);
|
|
102
101
|
});
|
|
103
102
|
}
|
|
@@ -116,7 +115,6 @@ export async function performGracefulShutdownAfterDrain(
|
|
|
116
115
|
server: Server,
|
|
117
116
|
mcpServer: Server,
|
|
118
117
|
shutdownGracefulCloseTimeoutSeconds: number,
|
|
119
|
-
agentMcpServer?: Server,
|
|
120
118
|
): Promise<void> {
|
|
121
119
|
const closeServer = (server: Server, name: string) =>
|
|
122
120
|
new Promise<void>((resolve) => {
|
|
@@ -137,9 +135,6 @@ export async function performGracefulShutdownAfterDrain(
|
|
|
137
135
|
await Promise.all([
|
|
138
136
|
closeServer(server, "Main server"),
|
|
139
137
|
closeServer(mcpServer, "MCP server"),
|
|
140
|
-
...(agentMcpServer
|
|
141
|
-
? [closeServer(agentMcpServer, "Agent MCP server")]
|
|
142
|
-
: []),
|
|
143
138
|
]);
|
|
144
139
|
|
|
145
140
|
try {
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import type { AxiosError } from "axios";
|
|
3
|
+
import { buildAxiosErrorLog, redactSensitive } from "./logger";
|
|
4
|
+
|
|
5
|
+
describe("redactSensitive", () => {
|
|
6
|
+
it("masks every known credential field at the top level", () => {
|
|
7
|
+
const input = {
|
|
8
|
+
password: "hunter2",
|
|
9
|
+
connectionString: "postgresql://u:p@h/db",
|
|
10
|
+
serviceAccountKeyJson: '{"private_key":"-----BEGIN..."}',
|
|
11
|
+
privateKey: "-----BEGIN PRIVATE KEY-----",
|
|
12
|
+
privateKeyPass: "passphrase",
|
|
13
|
+
secret: "gcs-hmac-secret",
|
|
14
|
+
secretAccessKey: "aws-secret",
|
|
15
|
+
sessionToken: "aws-session",
|
|
16
|
+
sasUrl: "https://a.blob.core.windows.net/c?sig=abc",
|
|
17
|
+
clientSecret: "azure-client-secret",
|
|
18
|
+
oauthClientSecret: "databricks-oauth-secret",
|
|
19
|
+
peakaKey: "peaka-api-key",
|
|
20
|
+
token: "databricks-pat",
|
|
21
|
+
accessToken: "motherduck-token",
|
|
22
|
+
};
|
|
23
|
+
const out = redactSensitive(input) as Record<string, unknown>;
|
|
24
|
+
for (const key of Object.keys(input)) {
|
|
25
|
+
expect(out[key]).toBe("[REDACTED]");
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
it("keeps non-secret fields intact", () => {
|
|
30
|
+
const input = {
|
|
31
|
+
name: "bigquery",
|
|
32
|
+
type: "bigquery",
|
|
33
|
+
defaultProjectId: "my-project",
|
|
34
|
+
host: "db.example.com",
|
|
35
|
+
port: 5432,
|
|
36
|
+
};
|
|
37
|
+
expect(redactSensitive(input)).toEqual(input);
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
it("redacts secrets nested inside the environment PATCH payload shape", () => {
|
|
41
|
+
const input = {
|
|
42
|
+
name: "demo___malloy-samples",
|
|
43
|
+
connections: [
|
|
44
|
+
{
|
|
45
|
+
name: "bigquery",
|
|
46
|
+
type: "bigquery",
|
|
47
|
+
bigqueryConnection: {
|
|
48
|
+
defaultProjectId: "vscode-demo",
|
|
49
|
+
serviceAccountKeyJson: '{"private_key":"-----BEGIN..."}',
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
name: "warehouse",
|
|
54
|
+
type: "snowflake",
|
|
55
|
+
snowflakeConnection: {
|
|
56
|
+
account: "acct",
|
|
57
|
+
username: "svc",
|
|
58
|
+
password: "s3cret",
|
|
59
|
+
privateKey: "-----BEGIN PRIVATE KEY-----",
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
],
|
|
63
|
+
};
|
|
64
|
+
expect(redactSensitive(input)).toEqual({
|
|
65
|
+
name: "demo___malloy-samples",
|
|
66
|
+
connections: [
|
|
67
|
+
{
|
|
68
|
+
name: "bigquery",
|
|
69
|
+
type: "bigquery",
|
|
70
|
+
bigqueryConnection: {
|
|
71
|
+
defaultProjectId: "vscode-demo",
|
|
72
|
+
serviceAccountKeyJson: "[REDACTED]",
|
|
73
|
+
},
|
|
74
|
+
},
|
|
75
|
+
{
|
|
76
|
+
name: "warehouse",
|
|
77
|
+
type: "snowflake",
|
|
78
|
+
snowflakeConnection: {
|
|
79
|
+
account: "acct",
|
|
80
|
+
username: "svc",
|
|
81
|
+
password: "[REDACTED]",
|
|
82
|
+
privateKey: "[REDACTED]",
|
|
83
|
+
},
|
|
84
|
+
},
|
|
85
|
+
],
|
|
86
|
+
});
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it("recurses through arrays of secrets", () => {
|
|
90
|
+
const out = redactSensitive([{ password: "a" }, { password: "b" }]);
|
|
91
|
+
expect(out).toEqual([
|
|
92
|
+
{ password: "[REDACTED]" },
|
|
93
|
+
{ password: "[REDACTED]" },
|
|
94
|
+
]);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("passes primitives through unchanged", () => {
|
|
98
|
+
expect(redactSensitive("plain")).toBe("plain");
|
|
99
|
+
expect(redactSensitive(42)).toBe(42);
|
|
100
|
+
expect(redactSensitive(null)).toBeNull();
|
|
101
|
+
expect(redactSensitive(undefined)).toBeUndefined();
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
it("preserves Date values instead of destructuring them", () => {
|
|
105
|
+
const d = new Date("2026-07-08T00:00:00.000Z");
|
|
106
|
+
const out = redactSensitive({ createdAt: d }) as { createdAt: Date };
|
|
107
|
+
expect(out.createdAt).toBeInstanceOf(Date);
|
|
108
|
+
expect(out.createdAt.getTime()).toBe(d.getTime());
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it("does not mutate the input object", () => {
|
|
112
|
+
const input = { password: "hunter2", nested: { token: "abc" } };
|
|
113
|
+
redactSensitive(input);
|
|
114
|
+
expect(input.password).toBe("hunter2");
|
|
115
|
+
expect(input.nested.token).toBe("abc");
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
it("tolerates circular references", () => {
|
|
119
|
+
const input: Record<string, unknown> = { password: "hunter2" };
|
|
120
|
+
input.self = input;
|
|
121
|
+
const out = redactSensitive(input) as Record<string, unknown>;
|
|
122
|
+
expect(out.password).toBe("[REDACTED]");
|
|
123
|
+
expect(out.self).toBe("[Circular]");
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("matches credential keys case-insensitively", () => {
|
|
127
|
+
const out = redactSensitive({
|
|
128
|
+
Password: "hunter2",
|
|
129
|
+
Authorization: "Bearer abc",
|
|
130
|
+
}) as Record<string, unknown>;
|
|
131
|
+
expect(out.Password).toBe("[REDACTED]");
|
|
132
|
+
expect(out.Authorization).toBe("[REDACTED]");
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it("masks credential HTTP headers", () => {
|
|
136
|
+
const out = redactSensitive({
|
|
137
|
+
"content-type": "application/json",
|
|
138
|
+
authorization: "Bearer secret-token",
|
|
139
|
+
cookie: "session=abc",
|
|
140
|
+
}) as Record<string, unknown>;
|
|
141
|
+
expect(out["content-type"]).toBe("application/json");
|
|
142
|
+
expect(out.authorization).toBe("[REDACTED]");
|
|
143
|
+
expect(out.cookie).toBe("[REDACTED]");
|
|
144
|
+
});
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
describe("buildAxiosErrorLog", () => {
|
|
148
|
+
it("redacts response data and headers on a server-side error", () => {
|
|
149
|
+
const error = {
|
|
150
|
+
response: {
|
|
151
|
+
config: { url: "http://worker/api" },
|
|
152
|
+
status: 500,
|
|
153
|
+
headers: { authorization: "Bearer abc", "content-type": "json" },
|
|
154
|
+
data: { connection: { serviceAccountKeyJson: "PRIVATE_KEY" } },
|
|
155
|
+
},
|
|
156
|
+
} as unknown as AxiosError;
|
|
157
|
+
const { message, meta } = buildAxiosErrorLog(error);
|
|
158
|
+
expect(message).toBe("Axios server-side error");
|
|
159
|
+
expect(meta).toEqual({
|
|
160
|
+
url: "http://worker/api",
|
|
161
|
+
status: 500,
|
|
162
|
+
headers: { authorization: "[REDACTED]", "content-type": "json" },
|
|
163
|
+
data: { connection: { serviceAccountKeyJson: "[REDACTED]" } },
|
|
164
|
+
});
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
it("never logs the raw request object on a client-side error", () => {
|
|
168
|
+
const error = {
|
|
169
|
+
request: { _header: "GET / HTTP/1.1\r\nAuthorization: Bearer abc" },
|
|
170
|
+
config: { method: "get", url: "http://worker/api" },
|
|
171
|
+
code: "ECONNABORTED",
|
|
172
|
+
} as unknown as AxiosError;
|
|
173
|
+
const { message, meta } = buildAxiosErrorLog(error);
|
|
174
|
+
expect(message).toBe("Axios client-side error");
|
|
175
|
+
expect(meta).toEqual({
|
|
176
|
+
method: "get",
|
|
177
|
+
url: "http://worker/api",
|
|
178
|
+
code: "ECONNABORTED",
|
|
179
|
+
});
|
|
180
|
+
expect(JSON.stringify(meta)).not.toContain("Bearer abc");
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it("logs only the message on a setup error", () => {
|
|
184
|
+
const error = {
|
|
185
|
+
message: "getaddrinfo ENOTFOUND",
|
|
186
|
+
config: { headers: { Authorization: "Bearer abc" } },
|
|
187
|
+
} as unknown as AxiosError;
|
|
188
|
+
const { message, meta } = buildAxiosErrorLog(error);
|
|
189
|
+
expect(message).toBe("Axios unknown error");
|
|
190
|
+
expect(meta).toEqual({ message: "getaddrinfo ENOTFOUND" });
|
|
191
|
+
expect(JSON.stringify(meta)).not.toContain("Bearer abc");
|
|
192
|
+
});
|
|
193
|
+
});
|
package/src/logger.ts
CHANGED
|
@@ -84,6 +84,73 @@ export function formatDuration(durationMs: number): string {
|
|
|
84
84
|
return `${durationMs.toFixed(2)}ms`;
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
+
// Connection endpoints carry provider credentials in their request/response
|
|
88
|
+
// bodies (BigQuery serviceAccountKeyJson, Snowflake privateKey/password, Postgres
|
|
89
|
+
// password/connectionString, S3 secretAccessKey/sessionToken, Azure clientSecret/
|
|
90
|
+
// sasUrl, Databricks token, ...), and axios error logs can carry credential HTTP
|
|
91
|
+
// headers (Authorization, Cookie). Those values are masked by key name before
|
|
92
|
+
// they reach a log transport. Matching is case-insensitive so header casing
|
|
93
|
+
// (Authorization vs authorization) does not slip through.
|
|
94
|
+
const SENSITIVE_KEY_NAMES = [
|
|
95
|
+
// connection config credentials
|
|
96
|
+
"password",
|
|
97
|
+
"connectionString",
|
|
98
|
+
"serviceAccountKeyJson",
|
|
99
|
+
"privateKey",
|
|
100
|
+
"privateKeyPass",
|
|
101
|
+
"secret",
|
|
102
|
+
"secretAccessKey",
|
|
103
|
+
"sessionToken",
|
|
104
|
+
"sasUrl",
|
|
105
|
+
"clientSecret",
|
|
106
|
+
"oauthClientSecret",
|
|
107
|
+
"peakaKey",
|
|
108
|
+
"token",
|
|
109
|
+
"accessToken",
|
|
110
|
+
// credential-bearing HTTP headers
|
|
111
|
+
"authorization",
|
|
112
|
+
"proxy-authorization",
|
|
113
|
+
"cookie",
|
|
114
|
+
"set-cookie",
|
|
115
|
+
"x-api-key",
|
|
116
|
+
];
|
|
117
|
+
const SENSITIVE_KEYS: ReadonlySet<string> = new Set(
|
|
118
|
+
SENSITIVE_KEY_NAMES.map((name) => name.toLowerCase()),
|
|
119
|
+
);
|
|
120
|
+
|
|
121
|
+
const REDACTED = "[REDACTED]";
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Deep-copy `value`, replacing any property whose key names a credential
|
|
125
|
+
* (SENSITIVE_KEYS) with a placeholder. The match is by key name (case-
|
|
126
|
+
* insensitive) and recurses through nested objects and arrays, since
|
|
127
|
+
* connections arrive nested under `connections[]`. Primitives and Dates pass
|
|
128
|
+
* through unchanged and the input is never mutated; used to keep secrets out of
|
|
129
|
+
* the request/response and axios-error logs.
|
|
130
|
+
*/
|
|
131
|
+
export function redactSensitive(
|
|
132
|
+
value: unknown,
|
|
133
|
+
seen: WeakSet<object> = new WeakSet(),
|
|
134
|
+
): unknown {
|
|
135
|
+
if (value === null || typeof value !== "object" || value instanceof Date) {
|
|
136
|
+
return value;
|
|
137
|
+
}
|
|
138
|
+
if (seen.has(value)) {
|
|
139
|
+
return "[Circular]";
|
|
140
|
+
}
|
|
141
|
+
seen.add(value);
|
|
142
|
+
if (Array.isArray(value)) {
|
|
143
|
+
return value.map((item) => redactSensitive(item, seen));
|
|
144
|
+
}
|
|
145
|
+
const result: Record<string, unknown> = {};
|
|
146
|
+
for (const [key, val] of Object.entries(value as Record<string, unknown>)) {
|
|
147
|
+
result[key] = SENSITIVE_KEYS.has(key.toLowerCase())
|
|
148
|
+
? REDACTED
|
|
149
|
+
: redactSensitive(val, seen);
|
|
150
|
+
}
|
|
151
|
+
return result;
|
|
152
|
+
}
|
|
153
|
+
|
|
87
154
|
export const loggerMiddleware: RequestHandler = (req, res, next) => {
|
|
88
155
|
const startTime = performance.now();
|
|
89
156
|
const resJson = res.json;
|
|
@@ -102,14 +169,14 @@ export const loggerMiddleware: RequestHandler = (req, res, next) => {
|
|
|
102
169
|
const logMetadata: Record<string, unknown> = {
|
|
103
170
|
statusCode: res.statusCode,
|
|
104
171
|
duration: formatDuration(durationMs),
|
|
105
|
-
payload: req.body,
|
|
172
|
+
payload: redactSensitive(req.body),
|
|
106
173
|
params: req.params,
|
|
107
174
|
query: req.query,
|
|
108
175
|
};
|
|
109
176
|
|
|
110
177
|
// Only include response body if response logging is enabled
|
|
111
178
|
if (!DISABLE_RESPONSE_LOGGING) {
|
|
112
|
-
logMetadata.response = res.locals.body;
|
|
179
|
+
logMetadata.response = redactSensitive(res.locals.body);
|
|
113
180
|
}
|
|
114
181
|
|
|
115
182
|
// Add traceId to log metadata if present
|
|
@@ -130,23 +197,53 @@ export const loggerMiddleware: RequestHandler = (req, res, next) => {
|
|
|
130
197
|
next();
|
|
131
198
|
};
|
|
132
199
|
|
|
133
|
-
|
|
200
|
+
/**
|
|
201
|
+
* Build the message + metadata logged for an AxiosError, with secrets removed.
|
|
202
|
+
* A failing upstream call can echo a connection config in `response.data` or
|
|
203
|
+
* return credential headers, and the raw `error.request` / `error` objects hold
|
|
204
|
+
* the outgoing Authorization header. Response fields are redacted and the raw
|
|
205
|
+
* request object is reduced to a few safe descriptors, so credentials never
|
|
206
|
+
* reach the log. Split out from `logAxiosError` so it can be unit tested.
|
|
207
|
+
*/
|
|
208
|
+
export function buildAxiosErrorLog(error: AxiosError): {
|
|
209
|
+
message: string;
|
|
210
|
+
meta: Record<string, unknown>;
|
|
211
|
+
} {
|
|
134
212
|
if (error.response) {
|
|
135
213
|
// The request was made and the server responded with a status code
|
|
136
|
-
// that falls out of the range of 2xx
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
//
|
|
150
|
-
|
|
214
|
+
// that falls out of the range of 2xx.
|
|
215
|
+
return {
|
|
216
|
+
message: "Axios server-side error",
|
|
217
|
+
meta: {
|
|
218
|
+
url: error.response.config.url,
|
|
219
|
+
status: error.response.status,
|
|
220
|
+
headers: redactSensitive(error.response.headers),
|
|
221
|
+
data: redactSensitive(error.response.data),
|
|
222
|
+
},
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
if (error.request) {
|
|
226
|
+
// The request was made but no response was received. `error.request` is
|
|
227
|
+
// the raw ClientRequest/XHR and carries the outgoing headers (including
|
|
228
|
+
// Authorization), so log only safe descriptors from the config.
|
|
229
|
+
return {
|
|
230
|
+
message: "Axios client-side error",
|
|
231
|
+
meta: {
|
|
232
|
+
method: error.config?.method,
|
|
233
|
+
url: error.config?.url,
|
|
234
|
+
code: error.code,
|
|
235
|
+
},
|
|
236
|
+
};
|
|
151
237
|
}
|
|
238
|
+
// Something happened setting up the request. The AxiosError still holds
|
|
239
|
+
// `config.headers`, so log only the message.
|
|
240
|
+
return {
|
|
241
|
+
message: "Axios unknown error",
|
|
242
|
+
meta: { message: error.message },
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export const logAxiosError = (error: AxiosError) => {
|
|
247
|
+
const { message, meta } = buildAxiosErrorLog(error);
|
|
248
|
+
logger.error(message, meta);
|
|
152
249
|
};
|
|
@@ -3,31 +3,6 @@ export interface ErrorDetails {
|
|
|
3
3
|
suggestions: string[];
|
|
4
4
|
}
|
|
5
5
|
|
|
6
|
-
export interface InvalidParamDetail {
|
|
7
|
-
name: string;
|
|
8
|
-
reason: string;
|
|
9
|
-
value?: unknown;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
export function getInvalidParamsError(
|
|
13
|
-
params: InvalidParamDetail[],
|
|
14
|
-
): ErrorDetails {
|
|
15
|
-
const reasons = params
|
|
16
|
-
.map(
|
|
17
|
-
(p) =>
|
|
18
|
-
`${p.name}: ${p.reason}${p.value !== undefined ? ` (value: ${JSON.stringify(p.value)})` : ""}`,
|
|
19
|
-
)
|
|
20
|
-
.join("; ");
|
|
21
|
-
return {
|
|
22
|
-
message: `Invalid parameter(s) provided. ${reasons}`,
|
|
23
|
-
suggestions: [
|
|
24
|
-
"Check the parameter names and values against the prompt's argument definition.",
|
|
25
|
-
"Ensure all required parameters are provided and have the correct data types.",
|
|
26
|
-
"For URI parameters, ensure they are correctly formatted.",
|
|
27
|
-
],
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
|
|
31
6
|
/**
|
|
32
7
|
* Generates error details for a resource not found scenario.
|
|
33
8
|
* @param resourceUriOrContext The URI that was not found, or a context string (e.g., "Package 'X'", "Model 'Y' in package 'X'").
|
|
@@ -36,13 +11,9 @@ export function getInvalidParamsError(
|
|
|
36
11
|
export function getNotFoundError(resourceUriOrContext: string): ErrorDetails {
|
|
37
12
|
const baseMessage = `Resource not found: ${resourceUriOrContext}`;
|
|
38
13
|
const suggestions: string[] = [
|
|
39
|
-
`Verify the identifier
|
|
40
|
-
`If using a URI, ensure it follows the correct format (e.g., malloy://environment/...) and includes the right path segments (e.g., /models/, /sources/, /queries/, /views/).`,
|
|
41
|
-
];
|
|
42
|
-
|
|
43
|
-
suggestions.push(
|
|
14
|
+
`Verify the identifier (${resourceUriOrContext}) is spelled correctly and exists. Check capitalization and path separators.`,
|
|
44
15
|
"Check if the resource exists and is correctly named in your Malloy environment structure or the specific model file.",
|
|
45
|
-
|
|
16
|
+
];
|
|
46
17
|
|
|
47
18
|
return {
|
|
48
19
|
message: baseMessage,
|
|
@@ -52,7 +23,7 @@ export function getNotFoundError(resourceUriOrContext: string): ErrorDetails {
|
|
|
52
23
|
|
|
53
24
|
/**
|
|
54
25
|
* Generates generic error details for internal server errors.
|
|
55
|
-
* @param operation The operation that failed (e.g., '
|
|
26
|
+
* @param operation The operation that failed (e.g., 'executeQuery').
|
|
56
27
|
* @param error Optional: The underlying error object or message.
|
|
57
28
|
* @returns ErrorDetails object.
|
|
58
29
|
*/
|
|
@@ -73,7 +44,7 @@ export function getInternalError(
|
|
|
73
44
|
|
|
74
45
|
/**
|
|
75
46
|
* Generates detailed error information for Malloy compilation or query execution errors.
|
|
76
|
-
* @param operation The operation that failed (e.g., '
|
|
47
|
+
* @param operation The operation that failed (e.g., 'executeQuery (load model)', 'executeQuery').
|
|
77
48
|
* @param modelIdentifier A path or URI identifying the model/notebook involved.
|
|
78
49
|
* @param error The underlying Malloy error object or other error.
|
|
79
50
|
* @returns ErrorDetails object.
|
|
@@ -144,21 +115,21 @@ export function getMalloyErrorDetails(
|
|
|
144
115
|
refined = true;
|
|
145
116
|
const [, viewName, sourceName] = viewNotFoundMatch;
|
|
146
117
|
suggestions.unshift(
|
|
147
|
-
`Suggestion: View '${viewName}' was not found in source '${sourceName}'. Check the view name spelling or
|
|
118
|
+
`Suggestion: View '${viewName}' was not found in source '${sourceName}'. Check the view name spelling or call malloy_getContext with sourceName '${sourceName}' to see the list of available views. Views are defined within sources like 'source: ${sourceName} is ... extend { view: ${viewName} is { ... } }'.`,
|
|
148
119
|
);
|
|
149
120
|
} else if (sourceNotFoundMatch) {
|
|
150
121
|
refined = true;
|
|
151
122
|
const [, sourceName] = sourceNotFoundMatch;
|
|
152
123
|
suggestions.unshift(
|
|
153
124
|
`Suggestion: Source '${sourceName}' was not found or could not be accessed. Verify its definition (e.g., \`source: ${sourceName} is table('...')\` or \`duckdb.sql("...")\`) and ensure any associated connections are valid.`,
|
|
154
|
-
`Suggestion: Check the spelling of '${sourceName}'. You can list sources in the
|
|
125
|
+
`Suggestion: Check the spelling of '${sourceName}'. You can list the sources in the package using malloy_getContext.`,
|
|
155
126
|
);
|
|
156
127
|
} else if (queryNotFoundMatch) {
|
|
157
128
|
refined = true;
|
|
158
129
|
const [, queryName] = queryNotFoundMatch;
|
|
159
130
|
suggestions.unshift(
|
|
160
131
|
`Suggestion: Named query '${queryName}' was not found. Verify its definition (e.g., \`query: ${queryName} is source_name -> { ... }\`) within the model '${modelIdentifier}'.`,
|
|
161
|
-
`Suggestion: Check the spelling of '${queryName}'. Ensure it's a named query (defined with \`query:\`), not a view. You can list named queries using
|
|
132
|
+
`Suggestion: Check the spelling of '${queryName}'. Ensure it's a named query (defined with \`query:\`), not a view. You can list named queries using malloy_getContext.`,
|
|
162
133
|
);
|
|
163
134
|
} else if (fieldNotFoundMatch) {
|
|
164
135
|
refined = true;
|
|
@@ -166,7 +137,7 @@ export function getMalloyErrorDetails(
|
|
|
166
137
|
fieldNotFoundMatch;
|
|
167
138
|
suggestions.unshift(
|
|
168
139
|
`Suggestion: Field '${fieldName}' was not found in ${fieldContextType} '${fieldContextName}'. Check the spelling of the field name within the definition of '${fieldContextName}'.`,
|
|
169
|
-
`Suggestion: Ensure the field is defined directly or inherited correctly in the '${fieldContextName}' ${fieldContextType}. You can inspect the ${fieldContextType}
|
|
140
|
+
`Suggestion: Ensure the field is defined directly or inherited correctly in the '${fieldContextName}' ${fieldContextType}. You can inspect the ${fieldContextType}'s fields using malloy_getContext.`,
|
|
170
141
|
);
|
|
171
142
|
} else if (referenceErrorMatch) {
|
|
172
143
|
refined = true;
|