@malloy-publisher/server 0.0.231 → 0.0.233
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +4 -0
- package/dist/app/api-doc.yaml +135 -13
- package/dist/app/assets/{EnvironmentPage-wa_EPkwK.js → EnvironmentPage-DutP7T8h.js} +1 -1
- package/dist/app/assets/{HomePage-jnCrupQp.js → HomePage-BcxDrBfl.js} +1 -1
- package/dist/app/assets/{LightMode-DYbwNULZ.js → LightMode-BJukGxgz.js} +1 -1
- package/dist/app/assets/{MainPage-CuJLrPNI.js → MainPage-DXbwlMeF.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-D_67x2ee.js → MaterializationsPage-BBQksmTU.js} +1 -1
- package/dist/app/assets/{ModelPage-D5JtAWqR.js → ModelPage-C6tK51uU.js} +1 -1
- package/dist/app/assets/{PackagePage-BRwtqUSG.js → PackagePage-Bo3cwwZE.js} +1 -1
- package/dist/app/assets/{RouteError-CBNNrnSD.js → RouteError-BufkcAKE.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-CTCeBneA.js → ThemeEditorPage-DICvvKpa.js} +1 -1
- package/dist/app/assets/{WorkbookPage-SN6f1RBm.js → WorkbookPage-Dkwt75Nj.js} +1 -1
- package/dist/app/assets/{core-Dp3q5Ieu.es-CD5FvM2s.js → core-C0nunIQT.es-DlMLKZBK.js} +1 -1
- package/dist/app/assets/{index-DU4r7GdU.js → index-BabP-V-S.js} +346 -321
- package/dist/app/assets/{index-BLCx1EdC.js → index-BusxL5Pt.js} +1 -1
- package/dist/app/assets/{index-C_tJstcx.js → index-CmEVVe-8.js} +15 -15
- package/dist/app/assets/{index-CfmBVB6M.js → index-Cs4WVm2z.js} +1 -1
- package/dist/app/assets/{index-B3Nn8Vm2.js → index-qnhU9CGo.js} +266 -265
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +2 -0
- package/dist/package_load_worker.mjs +11 -1
- package/dist/server.mjs +22627 -1260
- package/package.json +12 -12
- package/scripts/bake-duckdb-extensions.js +4 -1
- package/src/config.spec.ts +39 -0
- package/src/config.ts +135 -0
- package/src/controller/materialization.controller.spec.ts +62 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/controller/package.controller.spec.ts +6 -0
- package/src/controller/package.controller.ts +7 -2
- package/src/controller/query.controller.ts +26 -21
- package/src/errors.ts +19 -0
- package/src/json_utils.spec.ts +51 -0
- package/src/json_utils.ts +33 -0
- package/src/logger.spec.ts +18 -1
- package/src/logger.ts +3 -1
- package/src/materialization_metrics.spec.ts +89 -4
- package/src/materialization_metrics.ts +155 -5
- package/src/mcp/query_envelope.spec.ts +229 -0
- package/src/mcp/query_envelope.ts +230 -0
- package/src/mcp/server.protocol.spec.ts +128 -16
- package/src/mcp/skills/build_skills_bundle.ts +94 -4
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/skills/skills_bundle.spec.ts +113 -4
- package/src/mcp/tool_response.spec.ts +108 -0
- package/src/mcp/tool_response.ts +138 -0
- package/src/mcp/tools/compile_tool.spec.ts +112 -4
- package/src/mcp/tools/compile_tool.ts +61 -30
- package/src/mcp/tools/docs_search_tool.ts +6 -16
- package/src/mcp/tools/embedding_index.spec.ts +1236 -0
- package/src/mcp/tools/embedding_index.ts +808 -0
- package/src/mcp/tools/execute_query_tool.spec.ts +23 -3
- package/src/mcp/tools/execute_query_tool.ts +90 -151
- package/src/mcp/tools/get_context_eval.ts +194 -45
- package/src/mcp/tools/get_context_tool.spec.ts +358 -5
- package/src/mcp/tools/get_context_tool.ts +202 -56
- package/src/mcp/tools/reload_package_tool.ts +3 -29
- package/src/pg_helpers.spec.ts +201 -0
- package/src/pg_helpers.ts +44 -5
- package/src/server.ts +24 -0
- package/src/service/build_plan.spec.ts +128 -2
- package/src/service/build_plan.ts +239 -17
- package/src/service/compile_fragment_techniques.spec.ts +156 -0
- package/src/service/connection.spec.ts +371 -1
- package/src/service/connection.ts +339 -20
- package/src/service/connection_config.spec.ts +108 -0
- package/src/service/connection_config.ts +47 -8
- package/src/service/connection_federation.spec.ts +184 -0
- package/src/service/duckdb_instance_isolation.spec.ts +137 -0
- package/src/service/embedding_provider.spec.ts +329 -0
- package/src/service/embedding_provider.ts +236 -0
- package/src/service/environment.ts +274 -12
- package/src/service/environment_store.spec.ts +678 -3
- package/src/service/environment_store.ts +449 -33
- package/src/service/environment_store_clone.spec.ts +350 -0
- package/src/service/manifest_loader.spec.ts +68 -13
- package/src/service/manifest_loader.ts +67 -19
- package/src/service/materialization_build_session.spec.ts +435 -0
- package/src/service/materialization_build_session.ts +681 -0
- package/src/service/materialization_eligibility.spec.ts +158 -0
- package/src/service/materialization_eligibility.ts +305 -0
- package/src/service/materialization_serve_transform.spec.ts +1003 -0
- package/src/service/materialization_serve_transform.ts +779 -0
- package/src/service/materialization_service.spec.ts +774 -7
- package/src/service/materialization_service.ts +1107 -42
- package/src/service/materialization_test_fixtures.ts +7 -0
- package/src/service/model.spec.ts +207 -0
- package/src/service/model.ts +569 -59
- package/src/service/model_limits.spec.ts +28 -0
- package/src/service/model_limits.ts +21 -0
- package/src/service/model_storage_serve.spec.ts +193 -0
- package/src/service/model_storage_serve_joins.spec.ts +193 -0
- package/src/service/package.spec.ts +196 -0
- package/src/service/package.ts +385 -17
- package/src/service/persistence_policy.spec.ts +109 -0
- package/src/storage/duckdb/schema.ts +37 -0
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +177 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +104 -0
- package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +235 -0
|
@@ -3,6 +3,7 @@ import { describe, expect, it } from "bun:test";
|
|
|
3
3
|
import { PayloadTooLargeError } from "../errors";
|
|
4
4
|
import {
|
|
5
5
|
assertWithinModelResponseLimits,
|
|
6
|
+
queryRowLimitSource,
|
|
6
7
|
resolveModelQueryRowLimit,
|
|
7
8
|
} from "./model_limits";
|
|
8
9
|
|
|
@@ -179,3 +180,30 @@ describe("assertWithinModelResponseLimits", () => {
|
|
|
179
180
|
).not.toThrow();
|
|
180
181
|
});
|
|
181
182
|
});
|
|
183
|
+
|
|
184
|
+
describe("queryRowLimitSource", () => {
|
|
185
|
+
/**
|
|
186
|
+
* Must mirror resolveModelQueryRowLimit's own `requested` condition: if the
|
|
187
|
+
* two ever disagree, the reported source describes a different limit than
|
|
188
|
+
* the one actually pushed into the SQL.
|
|
189
|
+
*/
|
|
190
|
+
it("reports the query when it carried its own positive limit", () => {
|
|
191
|
+
expect(queryRowLimitSource(10)).toBe("query");
|
|
192
|
+
expect(queryRowLimitSource(1)).toBe("query");
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
it("reports the server default when the query carried none", () => {
|
|
196
|
+
expect(queryRowLimitSource(undefined)).toBe("server_default");
|
|
197
|
+
expect(queryRowLimitSource(0)).toBe("server_default");
|
|
198
|
+
expect(queryRowLimitSource(-1)).toBe("server_default");
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
it("agrees with which limit resolveModelQueryRowLimit actually used", () => {
|
|
202
|
+
const config = { defaultLimit: 1000, maxRows: 0 };
|
|
203
|
+
for (const userLimit of [undefined, 0, -1, 1, 10, 5000]) {
|
|
204
|
+
const used = resolveModelQueryRowLimit(userLimit, config);
|
|
205
|
+
const cameFromQuery = queryRowLimitSource(userLimit) === "query";
|
|
206
|
+
expect(cameFromQuery).toBe(used !== config.defaultLimit);
|
|
207
|
+
}
|
|
208
|
+
});
|
|
209
|
+
});
|
|
@@ -67,6 +67,27 @@ export function resolveModelQueryRowLimit(
|
|
|
67
67
|
return Math.min(requested, maxRows + 1);
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
+
/** Whether the cap came from the query itself or from the server default. */
|
|
71
|
+
export type QueryRowLimitSource = "query" | "server_default";
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Which of the two the cap in {@link resolveModelQueryRowLimit} came from.
|
|
75
|
+
*
|
|
76
|
+
* The distinction is the whole difference between "the database cut this off
|
|
77
|
+
* and you were not told" and "you asked for exactly this many". A `limit:` or
|
|
78
|
+
* `top:` the author wrote is deliberate and complete for what it asked; only the
|
|
79
|
+
* silently-applied default means rows were probably left behind.
|
|
80
|
+
*
|
|
81
|
+
* The condition deliberately mirrors `requested` above and must stay in step
|
|
82
|
+
* with it, so a change to which limit wins cannot leave the reported source
|
|
83
|
+
* describing the other one.
|
|
84
|
+
*/
|
|
85
|
+
export function queryRowLimitSource(
|
|
86
|
+
userLimit: number | undefined,
|
|
87
|
+
): QueryRowLimitSource {
|
|
88
|
+
return userLimit && userLimit > 0 ? "query" : "server_default";
|
|
89
|
+
}
|
|
90
|
+
|
|
70
91
|
export interface ModelResponseLimitsConfig {
|
|
71
92
|
/** Result of {@link getMaxQueryRows}. `0` disables the row cap. */
|
|
72
93
|
maxRows: number;
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// End-to-end routing contract for the `storage=` virtual-source serve path,
|
|
2
|
+
// exercised through the real Model.getQueryResults against a real in-memory
|
|
3
|
+
// DuckDB. The ORIGINAL source returns 0; the serve binding points at a table
|
|
4
|
+
// returning 60 — so the value observed proves whether the query was routed
|
|
5
|
+
// through the serve-shape transform (60) or served live from the original
|
|
6
|
+
// model (0). This pins the mode gate, the binding gate, and the safe fallback.
|
|
7
|
+
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
8
|
+
import {
|
|
9
|
+
FixedConnectionMap,
|
|
10
|
+
InMemoryURLReader,
|
|
11
|
+
modelDefToModelInfo,
|
|
12
|
+
Runtime,
|
|
13
|
+
} from "@malloydata/malloy";
|
|
14
|
+
import { afterEach, describe, expect, it } from "bun:test";
|
|
15
|
+
import { Model } from "./model";
|
|
16
|
+
import type { ServeBinding } from "./materialization_serve_transform";
|
|
17
|
+
|
|
18
|
+
const ROOT = "file:///storage-serve-e2e/";
|
|
19
|
+
const QUERY = "run: X -> { aggregate: t is total.sum() }";
|
|
20
|
+
const BINDING: ServeBinding = {
|
|
21
|
+
sourceName: "X",
|
|
22
|
+
connectionName: "duckdb",
|
|
23
|
+
virtualHandle: "h",
|
|
24
|
+
tablePath: "mz_real",
|
|
25
|
+
schema: [{ name: "total", type: "BIGINT" }],
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* A Model whose original source `X` yields total=0, with a real DuckDB
|
|
30
|
+
* connection carrying `mz_real` (total=60) that the serve binding rebinds to.
|
|
31
|
+
*/
|
|
32
|
+
async function buildModel(opts?: {
|
|
33
|
+
originalText?: string;
|
|
34
|
+
storageSQL?: string;
|
|
35
|
+
}): Promise<Model> {
|
|
36
|
+
const duckdb = new DuckDBConnection("duckdb", ":memory:");
|
|
37
|
+
// DuckDB shares a process-global :memory: db by connection name, so a prior
|
|
38
|
+
// test's table can linger — replace it.
|
|
39
|
+
await duckdb.runSQL(
|
|
40
|
+
opts?.storageSQL ??
|
|
41
|
+
"CREATE OR REPLACE TABLE mz_real AS SELECT 60 AS total",
|
|
42
|
+
);
|
|
43
|
+
const connMap = new Map<string, DuckDBConnection>([["duckdb", duckdb]]);
|
|
44
|
+
const originalText =
|
|
45
|
+
opts?.originalText ?? `source: X is duckdb.sql("SELECT 0 AS total")`;
|
|
46
|
+
const urlReader = new InMemoryURLReader(
|
|
47
|
+
new Map([[`${ROOT}m.malloy`, originalText]]),
|
|
48
|
+
);
|
|
49
|
+
const runtime = new Runtime({
|
|
50
|
+
urlReader,
|
|
51
|
+
connections: new FixedConnectionMap(connMap, "duckdb"),
|
|
52
|
+
});
|
|
53
|
+
const mm = runtime.loadModel(new URL(`${ROOT}m.malloy`), {
|
|
54
|
+
importBaseURL: new URL(ROOT),
|
|
55
|
+
});
|
|
56
|
+
const compiled = await mm.getModel();
|
|
57
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
58
|
+
const modelDef = (compiled as any)._modelDef;
|
|
59
|
+
const modelInfo = modelDefToModelInfo(modelDef);
|
|
60
|
+
const model = new Model(
|
|
61
|
+
"pkg",
|
|
62
|
+
"m.malloy",
|
|
63
|
+
{},
|
|
64
|
+
"model",
|
|
65
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
66
|
+
mm as any,
|
|
67
|
+
modelDef,
|
|
68
|
+
undefined,
|
|
69
|
+
undefined,
|
|
70
|
+
undefined,
|
|
71
|
+
undefined,
|
|
72
|
+
undefined,
|
|
73
|
+
undefined,
|
|
74
|
+
undefined,
|
|
75
|
+
modelInfo,
|
|
76
|
+
);
|
|
77
|
+
model.setServeMalloyConfig(connMap);
|
|
78
|
+
return model;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function runTotal(model: Model): Promise<number> {
|
|
82
|
+
const res = await model.getQueryResults(
|
|
83
|
+
undefined,
|
|
84
|
+
undefined,
|
|
85
|
+
QUERY,
|
|
86
|
+
{},
|
|
87
|
+
true,
|
|
88
|
+
);
|
|
89
|
+
// compactResult is the row value array: [{ t: <sum> }].
|
|
90
|
+
const rows = res.compactResult as unknown as { t: number }[];
|
|
91
|
+
return Number(rows[0].t);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
describe("storage= serve routing (end-to-end)", () => {
|
|
95
|
+
afterEach(() => {
|
|
96
|
+
delete process.env.PERSIST_STORAGE_MODE;
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it("mode=on + a binding routes the query to the materialized table", async () => {
|
|
100
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
101
|
+
const model = await buildModel();
|
|
102
|
+
model.setServeBindings([BINDING]);
|
|
103
|
+
expect(await runTotal(model)).toBe(60);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it("mode=off serves live even with a binding present (kill switch)", async () => {
|
|
107
|
+
process.env.PERSIST_STORAGE_MODE = "off";
|
|
108
|
+
const model = await buildModel();
|
|
109
|
+
model.setServeBindings([BINDING]);
|
|
110
|
+
expect(await runTotal(model)).toBe(0);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it("mode=write-only serves live (build-only rung)", async () => {
|
|
114
|
+
process.env.PERSIST_STORAGE_MODE = "write-only";
|
|
115
|
+
const model = await buildModel();
|
|
116
|
+
model.setServeBindings([BINDING]);
|
|
117
|
+
expect(await runTotal(model)).toBe(0);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
it("mode=on with no bindings serves live", async () => {
|
|
121
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
122
|
+
const model = await buildModel();
|
|
123
|
+
expect(await runTotal(model)).toBe(0);
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it("falls back to live when the serve shape cannot compile (bad connection)", async () => {
|
|
127
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
128
|
+
const model = await buildModel();
|
|
129
|
+
// A binding on a connection the config doesn't have — the serve-shape
|
|
130
|
+
// compile throws, and the query must still succeed, served live.
|
|
131
|
+
model.setServeBindings([{ ...BINDING, connectionName: "missing_conn" }]);
|
|
132
|
+
expect(await runTotal(model)).toBe(0);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it("a binding for a source this model does not define does not break serving the model's own sources", async () => {
|
|
136
|
+
// Bindings are pushed to EVERY model in the package, so a model can receive
|
|
137
|
+
// a binding for a source it doesn't define (defined in a sibling model).
|
|
138
|
+
// That source has no field list here, so its narrowed schema is empty — the
|
|
139
|
+
// serve shape must simply omit it, not emit an empty `type: {}` that fails
|
|
140
|
+
// the whole shape model and drops storage serving for X too.
|
|
141
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
142
|
+
const model = await buildModel();
|
|
143
|
+
model.setServeBindings([
|
|
144
|
+
BINDING,
|
|
145
|
+
{
|
|
146
|
+
sourceName: "OtherModelSource", // not defined in this model
|
|
147
|
+
connectionName: "duckdb",
|
|
148
|
+
virtualHandle: "h2",
|
|
149
|
+
tablePath: "mz_real",
|
|
150
|
+
schema: [{ name: "total", type: "BIGINT" }],
|
|
151
|
+
},
|
|
152
|
+
]);
|
|
153
|
+
expect(await runTotal(model)).toBe(60);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
it("does not serve an except:-hidden column from storage (vector A)", async () => {
|
|
157
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
158
|
+
// X hides `secret` via except:, but the built table still carries it (the
|
|
159
|
+
// build's getSQL projected it) and the binding schema captured it.
|
|
160
|
+
const model = await buildModel({
|
|
161
|
+
originalText:
|
|
162
|
+
`source: X is duckdb.sql("SELECT 0 AS total, 'live' AS secret") ` +
|
|
163
|
+
`extend { except: secret }`,
|
|
164
|
+
storageSQL:
|
|
165
|
+
"CREATE OR REPLACE TABLE mz_real AS " +
|
|
166
|
+
"SELECT 60 AS total, 'from_storage' AS secret",
|
|
167
|
+
});
|
|
168
|
+
model.setServeBindings([
|
|
169
|
+
{
|
|
170
|
+
...BINDING,
|
|
171
|
+
schema: [
|
|
172
|
+
{ name: "total", type: "BIGINT" },
|
|
173
|
+
{ name: "secret", type: "VARCHAR" },
|
|
174
|
+
],
|
|
175
|
+
},
|
|
176
|
+
]);
|
|
177
|
+
// The public column still serves from storage (narrowing didn't break it).
|
|
178
|
+
expect(await runTotal(model)).toBe(60);
|
|
179
|
+
// A query on the hidden column must NOT be served from storage: the shape
|
|
180
|
+
// no longer declares `secret`, so it falls back to live, where X's
|
|
181
|
+
// `except:` makes `secret` undefined → the query is refused. (Before the
|
|
182
|
+
// fix this returned 'from_storage' — the leak.)
|
|
183
|
+
await expect(
|
|
184
|
+
model.getQueryResults(
|
|
185
|
+
undefined,
|
|
186
|
+
undefined,
|
|
187
|
+
"run: X -> { group_by: secret }",
|
|
188
|
+
{},
|
|
189
|
+
true,
|
|
190
|
+
),
|
|
191
|
+
).rejects.toThrow();
|
|
192
|
+
});
|
|
193
|
+
});
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// Model-level routing contract for JOINS and VIEWS in the `storage=` serve path,
|
|
2
|
+
// through the real Model.getQueryResults against a real in-memory DuckDB. This
|
|
3
|
+
// exercises the glue the transform-level tests can't: serveBindingsWithRefinements
|
|
4
|
+
// reading a REAL compiled modelDef, mapping a join's `sourceID` to the joined
|
|
5
|
+
// source name, and lifting join/view declarations verbatim from the on-disk
|
|
6
|
+
// source file (as it does in production — hence a real temp file, not
|
|
7
|
+
// InMemoryURLReader alone), plus the shape-compile escalation that keeps one
|
|
8
|
+
// un-carriable refinement from disabling all storage serving.
|
|
9
|
+
//
|
|
10
|
+
// Signals: the live sources return region_name 'LIVE' and amount 10; the bound
|
|
11
|
+
// (materialized) tables return 'STORE' and amount 99 — so the value observed
|
|
12
|
+
// tells which path ran.
|
|
13
|
+
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
14
|
+
import {
|
|
15
|
+
FixedConnectionMap,
|
|
16
|
+
InMemoryURLReader,
|
|
17
|
+
modelDefToModelInfo,
|
|
18
|
+
Runtime,
|
|
19
|
+
} from "@malloydata/malloy";
|
|
20
|
+
import { afterAll, afterEach, describe, expect, it } from "bun:test";
|
|
21
|
+
import { mkdtempSync, rmSync, writeFileSync } from "fs";
|
|
22
|
+
import { tmpdir } from "os";
|
|
23
|
+
import { join } from "path";
|
|
24
|
+
import { pathToFileURL } from "url";
|
|
25
|
+
import { Model } from "./model";
|
|
26
|
+
import type { ServeBinding } from "./materialization_serve_transform";
|
|
27
|
+
|
|
28
|
+
const MODEL_SRC = `source: regions is duckdb.sql("SELECT 'r1' AS region_id, 'LIVE' AS region_name")
|
|
29
|
+
source: orders is duckdb.sql("SELECT 10 AS amount, 'r1' AS region_id") extend {
|
|
30
|
+
join_one: regions is regions on region_id = regions.region_id
|
|
31
|
+
measure: total is amount.sum()
|
|
32
|
+
view: by_region is { group_by: regions.region_name; aggregate: total }
|
|
33
|
+
}
|
|
34
|
+
`;
|
|
35
|
+
|
|
36
|
+
const ORDERS_BINDING: ServeBinding = {
|
|
37
|
+
sourceName: "orders",
|
|
38
|
+
connectionName: "duckdb",
|
|
39
|
+
virtualHandle: "orders_h",
|
|
40
|
+
tablePath: "orders_mz",
|
|
41
|
+
schema: [
|
|
42
|
+
{ name: "amount", type: "BIGINT" },
|
|
43
|
+
{ name: "region_id", type: "VARCHAR" },
|
|
44
|
+
],
|
|
45
|
+
};
|
|
46
|
+
const REGIONS_BINDING: ServeBinding = {
|
|
47
|
+
sourceName: "regions",
|
|
48
|
+
connectionName: "duckdb",
|
|
49
|
+
virtualHandle: "regions_h",
|
|
50
|
+
tablePath: "regions_mz",
|
|
51
|
+
schema: [
|
|
52
|
+
{ name: "region_id", type: "VARCHAR" },
|
|
53
|
+
{ name: "region_name", type: "VARCHAR" },
|
|
54
|
+
],
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
const tmpDirs: string[] = [];
|
|
58
|
+
|
|
59
|
+
async function buildModel(): Promise<Model> {
|
|
60
|
+
const dir = mkdtempSync(join(tmpdir(), "mz-join-"));
|
|
61
|
+
tmpDirs.push(dir);
|
|
62
|
+
const file = join(dir, "m.malloy");
|
|
63
|
+
writeFileSync(file, MODEL_SRC);
|
|
64
|
+
const fileUrl = pathToFileURL(file).toString();
|
|
65
|
+
|
|
66
|
+
const duckdb = new DuckDBConnection("duckdb", ":memory:");
|
|
67
|
+
// Materialized tables carry distinct values (region_name 'STORE', amount 99)
|
|
68
|
+
// from the live sources ('LIVE', 10), so a result value proves which ran.
|
|
69
|
+
await duckdb.runSQL(
|
|
70
|
+
"CREATE OR REPLACE TABLE orders_mz AS SELECT 99 AS amount, 'r1' AS region_id",
|
|
71
|
+
);
|
|
72
|
+
await duckdb.runSQL(
|
|
73
|
+
"CREATE OR REPLACE TABLE regions_mz AS SELECT 'r1' AS region_id, 'STORE' AS region_name",
|
|
74
|
+
);
|
|
75
|
+
const connMap = new Map<string, DuckDBConnection>([["duckdb", duckdb]]);
|
|
76
|
+
|
|
77
|
+
// Register the source under its real file:// URL so the compiled modelDef's
|
|
78
|
+
// locations point at the file liftText reads back from disk.
|
|
79
|
+
const urlReader = new InMemoryURLReader(new Map([[fileUrl, MODEL_SRC]]));
|
|
80
|
+
const runtime = new Runtime({
|
|
81
|
+
urlReader,
|
|
82
|
+
connections: new FixedConnectionMap(connMap, "duckdb"),
|
|
83
|
+
});
|
|
84
|
+
const mm = runtime.loadModel(new URL(fileUrl), {
|
|
85
|
+
importBaseURL: new URL(".", fileUrl),
|
|
86
|
+
});
|
|
87
|
+
const compiled = await mm.getModel();
|
|
88
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
89
|
+
const modelDef = (compiled as any)._modelDef;
|
|
90
|
+
const modelInfo = modelDefToModelInfo(modelDef);
|
|
91
|
+
const model = new Model(
|
|
92
|
+
"pkg",
|
|
93
|
+
"m.malloy",
|
|
94
|
+
{},
|
|
95
|
+
"model",
|
|
96
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
97
|
+
mm as any,
|
|
98
|
+
modelDef,
|
|
99
|
+
undefined,
|
|
100
|
+
undefined,
|
|
101
|
+
undefined,
|
|
102
|
+
undefined,
|
|
103
|
+
undefined,
|
|
104
|
+
undefined,
|
|
105
|
+
undefined,
|
|
106
|
+
modelInfo,
|
|
107
|
+
);
|
|
108
|
+
model.setServeMalloyConfig(connMap);
|
|
109
|
+
return model;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Run a query and return the first result row. */
|
|
113
|
+
async function runRow<T>(model: Model, query: string): Promise<T> {
|
|
114
|
+
const res = await model.getQueryResults(
|
|
115
|
+
undefined,
|
|
116
|
+
undefined,
|
|
117
|
+
query,
|
|
118
|
+
{},
|
|
119
|
+
true,
|
|
120
|
+
);
|
|
121
|
+
return (res.compactResult as unknown as T[])[0];
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
const JOIN_QUERY =
|
|
125
|
+
"run: orders -> { group_by: regions.region_name; aggregate: t is amount.sum() }";
|
|
126
|
+
const VIEW_QUERY = "run: orders -> by_region";
|
|
127
|
+
const PLAIN_QUERY = "run: orders -> { aggregate: t is amount.sum() }";
|
|
128
|
+
|
|
129
|
+
describe("storage= serve routing with joins and views (end-to-end)", () => {
|
|
130
|
+
afterEach(() => {
|
|
131
|
+
delete process.env.PERSIST_STORAGE_MODE;
|
|
132
|
+
});
|
|
133
|
+
afterAll(() => {
|
|
134
|
+
for (const d of tmpDirs) rmSync(d, { recursive: true, force: true });
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it("serves a join query from the materialized tables when both sources are bound", async () => {
|
|
138
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
139
|
+
const model = await buildModel();
|
|
140
|
+
model.setServeBindings([ORDERS_BINDING, REGIONS_BINDING]);
|
|
141
|
+
const row = await runRow<{ region_name: string }>(model, JOIN_QUERY);
|
|
142
|
+
expect(row.region_name).toBe("STORE"); // join ran over the bound tables
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it("serves a named view (grouping by a joined field) from the materialized tables", async () => {
|
|
146
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
147
|
+
const model = await buildModel();
|
|
148
|
+
model.setServeBindings([ORDERS_BINDING, REGIONS_BINDING]);
|
|
149
|
+
const row = await runRow<{ region_name: string; total: number }>(
|
|
150
|
+
model,
|
|
151
|
+
VIEW_QUERY,
|
|
152
|
+
);
|
|
153
|
+
expect(row.region_name).toBe("STORE");
|
|
154
|
+
expect(Number(row.total)).toBe(99); // amount from orders_mz, not live 10
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
it("falls back to live when the joined source is not materialized (the gate)", async () => {
|
|
158
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
159
|
+
const model = await buildModel();
|
|
160
|
+
// Only `orders` is bound; the join to `regions` is not carried, so a query
|
|
161
|
+
// traversing it cannot compile against the serve shape → live.
|
|
162
|
+
model.setServeBindings([ORDERS_BINDING]);
|
|
163
|
+
expect(
|
|
164
|
+
(await runRow<{ region_name: string }>(model, JOIN_QUERY)).region_name,
|
|
165
|
+
).toBe("LIVE");
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("a view that reaches a non-materialized join does not disable base serving (escalation)", async () => {
|
|
169
|
+
process.env.PERSIST_STORAGE_MODE = "on";
|
|
170
|
+
const model = await buildModel();
|
|
171
|
+
// Only `orders` bound: the join is gated out, so the emitted `by_region`
|
|
172
|
+
// view references an absent join and the RICH shape fails to compile. The
|
|
173
|
+
// escalation must drop the view category and keep serving base + measures,
|
|
174
|
+
// rather than losing all storage serving. A plain aggregate must still be
|
|
175
|
+
// served from storage (amount 99), while the view query falls back to live.
|
|
176
|
+
model.setServeBindings([ORDERS_BINDING]);
|
|
177
|
+
expect(Number((await runRow<{ t: number }>(model, PLAIN_QUERY)).t)).toBe(
|
|
178
|
+
99,
|
|
179
|
+
);
|
|
180
|
+
expect(
|
|
181
|
+
(await runRow<{ region_name: string }>(model, VIEW_QUERY)).region_name,
|
|
182
|
+
).toBe("LIVE");
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
it("mode=off serves live even with both bindings present", async () => {
|
|
186
|
+
process.env.PERSIST_STORAGE_MODE = "off";
|
|
187
|
+
const model = await buildModel();
|
|
188
|
+
model.setServeBindings([ORDERS_BINDING, REGIONS_BINDING]);
|
|
189
|
+
expect(
|
|
190
|
+
(await runRow<{ region_name: string }>(model, JOIN_QUERY)).region_name,
|
|
191
|
+
).toBe("LIVE");
|
|
192
|
+
});
|
|
193
|
+
});
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { DuckDBConnection } from "@malloydata/db-duckdb";
|
|
2
2
|
import "@malloydata/db-duckdb/native";
|
|
3
|
+
import { MalloyConfig } from "@malloydata/malloy";
|
|
3
4
|
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
4
5
|
import { Stats } from "fs";
|
|
5
6
|
import fs from "fs/promises";
|
|
@@ -411,6 +412,201 @@ describe("service/package", () => {
|
|
|
411
412
|
rowCount: 3,
|
|
412
413
|
});
|
|
413
414
|
});
|
|
415
|
+
|
|
416
|
+
it("should probe an xlsx database file", async () => {
|
|
417
|
+
// Real committed fixture: DuckDB's excel extension reads it via the
|
|
418
|
+
// same duckdb.table() replacement scan the csv path uses.
|
|
419
|
+
await fs.copyFile(
|
|
420
|
+
join("tests", "fixtures", "xlsx", "database.xlsx"),
|
|
421
|
+
join(testPackageDirectory, "database.xlsx"),
|
|
422
|
+
);
|
|
423
|
+
const conn = new DuckDBConnection("duckdb");
|
|
424
|
+
|
|
425
|
+
// Unlike parquet/csv (statically linked), excel is downloaded from
|
|
426
|
+
// extensions.duckdb.org on demand. Some CI runners (notably the
|
|
427
|
+
// hosted macOS fleet, per bake-duckdb-extensions.js) cannot reach
|
|
428
|
+
// that CDN, so skip the read there rather than fail; the Linux e2e
|
|
429
|
+
// and the extension-independent specs still cover this path.
|
|
430
|
+
try {
|
|
431
|
+
await conn.runSQL("INSTALL excel; LOAD excel;");
|
|
432
|
+
} catch {
|
|
433
|
+
console.warn(
|
|
434
|
+
"Skipping xlsx probe test: excel extension could not be installed (offline runner)",
|
|
435
|
+
);
|
|
436
|
+
await conn.close();
|
|
437
|
+
return;
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
// @ts-expect-error Accessing private static method for testing
|
|
441
|
+
const info = await Package.getDatabaseInfo(
|
|
442
|
+
testPackageDirectory,
|
|
443
|
+
"database.xlsx",
|
|
444
|
+
conn,
|
|
445
|
+
);
|
|
446
|
+
|
|
447
|
+
expect(info).toEqual({
|
|
448
|
+
name: "database.xlsx",
|
|
449
|
+
columns: [
|
|
450
|
+
{ name: "Name", type: "string" },
|
|
451
|
+
// xlsx stores every number as a double, so numeric columns
|
|
452
|
+
// always surface as Malloy `number`.
|
|
453
|
+
{ name: "Value", type: "number" },
|
|
454
|
+
],
|
|
455
|
+
rowCount: 3,
|
|
456
|
+
});
|
|
457
|
+
});
|
|
458
|
+
});
|
|
459
|
+
|
|
460
|
+
describe("getDatabasePaths", () => {
|
|
461
|
+
it("should list xlsx files alongside parquet and csv", async () => {
|
|
462
|
+
await fs.copyFile(
|
|
463
|
+
join("tests", "fixtures", "xlsx", "database.xlsx"),
|
|
464
|
+
join(testPackageDirectory, "database.xlsx"),
|
|
465
|
+
);
|
|
466
|
+
|
|
467
|
+
// @ts-expect-error Accessing private static method for testing
|
|
468
|
+
const paths = await Package.getDatabasePaths(testPackageDirectory);
|
|
469
|
+
|
|
470
|
+
expect(paths.sort()).toEqual(["database.csv", "database.xlsx"]);
|
|
471
|
+
});
|
|
472
|
+
|
|
473
|
+
it("should skip Excel owner files (~$name.xlsx)", async () => {
|
|
474
|
+
// Excel creates this sibling while a workbook is open; it is not a
|
|
475
|
+
// valid zip, so probing it would throw and drop the package.
|
|
476
|
+
await fs.writeFile(
|
|
477
|
+
join(testPackageDirectory, "~$database.xlsx"),
|
|
478
|
+
"not a real xlsx",
|
|
479
|
+
);
|
|
480
|
+
|
|
481
|
+
// @ts-expect-error Accessing private static method for testing
|
|
482
|
+
const paths = await Package.getDatabasePaths(testPackageDirectory);
|
|
483
|
+
|
|
484
|
+
expect(paths).toEqual(["database.csv"]);
|
|
485
|
+
});
|
|
486
|
+
});
|
|
487
|
+
|
|
488
|
+
describe("readDatabases", () => {
|
|
489
|
+
it("should report an unreadable database file instead of failing the package", async () => {
|
|
490
|
+
// A corrupt or partial spreadsheet must not take the whole package
|
|
491
|
+
// down, but it must not vanish silently either: it stays in the
|
|
492
|
+
// listing carrying `error` instead of `info`, the same way a model
|
|
493
|
+
// that fails to compile is reported. Extension-independent:
|
|
494
|
+
// broken.xlsx throws whether excel loads or not, so this holds on
|
|
495
|
+
// runners without the CDN too.
|
|
496
|
+
await fs.writeFile(
|
|
497
|
+
join(testPackageDirectory, "broken.xlsx"),
|
|
498
|
+
"not a real xlsx",
|
|
499
|
+
);
|
|
500
|
+
const conn = new DuckDBConnection("duckdb");
|
|
501
|
+
const malloyConfig = {
|
|
502
|
+
connections: { lookupConnection: async () => conn },
|
|
503
|
+
} as unknown as MalloyConfig;
|
|
504
|
+
|
|
505
|
+
const databases =
|
|
506
|
+
// @ts-expect-error Accessing private static method for testing
|
|
507
|
+
(await Package.readDatabases(
|
|
508
|
+
testPackageDirectory,
|
|
509
|
+
malloyConfig,
|
|
510
|
+
)) as {
|
|
511
|
+
path: string;
|
|
512
|
+
info?: unknown;
|
|
513
|
+
error?: string;
|
|
514
|
+
}[];
|
|
515
|
+
|
|
516
|
+
// The valid file still loads with a full probe.
|
|
517
|
+
const good = databases.find((d) => d.path === "database.csv");
|
|
518
|
+
expect(good?.info).toBeDefined();
|
|
519
|
+
expect(good?.error).toBeUndefined();
|
|
520
|
+
|
|
521
|
+
// The broken one is visible over the API, with a reason.
|
|
522
|
+
const bad = databases.find((d) => d.path === "broken.xlsx");
|
|
523
|
+
expect(bad).toBeDefined();
|
|
524
|
+
expect(bad?.info).toBeUndefined();
|
|
525
|
+
expect(bad?.error).toBeTruthy();
|
|
526
|
+
});
|
|
527
|
+
});
|
|
528
|
+
});
|
|
529
|
+
|
|
530
|
+
// A host is authoritative about WHICH TABLE backs a source, never about
|
|
531
|
+
// WHETHER the source may be served frozen — that is decided by compiling the
|
|
532
|
+
// model. These assert the gate that enforces it, because the failure is silent:
|
|
533
|
+
// an admitted binding for a `given`-filtered source serves one caller's rows to
|
|
534
|
+
// everyone, and no error is raised. Scenarios
|
|
535
|
+
// `host-binding-honors-row-level-access` and `host-binding-of-unplanned-source`
|
|
536
|
+
// prove it end to end; these keep it from being refactored away.
|
|
537
|
+
describe("bindStorageServeBindings eligibility gate", () => {
|
|
538
|
+
/** A structurally complete manifest entry — the shape a host sends. */
|
|
539
|
+
const entry = (sourceName: string, table: string) => ({
|
|
540
|
+
sourceEntityId: `eid-${sourceName}`,
|
|
541
|
+
sourceName,
|
|
542
|
+
physicalTableName: table,
|
|
543
|
+
storageConnectionName: "lake",
|
|
544
|
+
schema: [{ name: "region", type: "VARCHAR" }],
|
|
545
|
+
});
|
|
546
|
+
|
|
547
|
+
const packageWith = (
|
|
548
|
+
eligibility:
|
|
549
|
+
| { eligible: string[]; refused: Record<string, string> }
|
|
550
|
+
| undefined,
|
|
551
|
+
) => {
|
|
552
|
+
const pkg = new Package(
|
|
553
|
+
"testProject",
|
|
554
|
+
"testPackage",
|
|
555
|
+
testPackageDirectory,
|
|
556
|
+
{ name: "testPackage" },
|
|
557
|
+
[],
|
|
558
|
+
new Map(),
|
|
559
|
+
);
|
|
560
|
+
// Private by design: its only writer is the load path, which needs a
|
|
561
|
+
// compiled package. Set directly to isolate the filter.
|
|
562
|
+
(
|
|
563
|
+
pkg as unknown as { sourceEligibility: typeof eligibility }
|
|
564
|
+
).sourceEligibility = eligibility;
|
|
565
|
+
return pkg;
|
|
566
|
+
};
|
|
567
|
+
|
|
568
|
+
const boundSources = (pkg: Package) =>
|
|
569
|
+
(pkg.getPackageMetadata().storageServeBindings ?? []).map(
|
|
570
|
+
(b) => b.sourceName,
|
|
571
|
+
);
|
|
572
|
+
|
|
573
|
+
it("drops a refused source's binding and keeps its eligible neighbour", () => {
|
|
574
|
+
const pkg = packageWith({
|
|
575
|
+
eligible: ["all_rollup"],
|
|
576
|
+
refused: { scoped_rollup: "references a given" },
|
|
577
|
+
});
|
|
578
|
+
|
|
579
|
+
pkg.bindStorageServeBindings({
|
|
580
|
+
a: entry("all_rollup", "t_all"),
|
|
581
|
+
b: entry("scoped_rollup", "t_all"),
|
|
582
|
+
});
|
|
583
|
+
|
|
584
|
+
expect(boundSources(pkg)).toEqual(["all_rollup"]);
|
|
585
|
+
});
|
|
586
|
+
|
|
587
|
+
it("refuses every binding when the build plan could not be computed", () => {
|
|
588
|
+
// The plan compute does live schema RPCs, so a warehouse blip during load
|
|
589
|
+
// is the ordinary way this happens — and the package still serves. An
|
|
590
|
+
// unexamined binding must not read as an eligible one.
|
|
591
|
+
const pkg = packageWith(undefined);
|
|
592
|
+
|
|
593
|
+
pkg.bindStorageServeBindings({ a: entry("all_rollup", "t_all") });
|
|
594
|
+
|
|
595
|
+
expect(boundSources(pkg)).toEqual([]);
|
|
596
|
+
});
|
|
597
|
+
|
|
598
|
+
it("refuses a source the plan never contained", () => {
|
|
599
|
+
// Malloy admits only query-shaped sources as build roots, so `#@ persist`
|
|
600
|
+
// on a filtered pass-through produces no plan entry — neither eligible nor
|
|
601
|
+
// refused. Absence is not eligibility.
|
|
602
|
+
const pkg = packageWith({ eligible: ["all_rows"], refused: {} });
|
|
603
|
+
|
|
604
|
+
pkg.bindStorageServeBindings({
|
|
605
|
+
a: entry("all_rows", "t_all"),
|
|
606
|
+
b: entry("scoped", "t_all"),
|
|
607
|
+
});
|
|
608
|
+
|
|
609
|
+
expect(boundSources(pkg)).toEqual(["all_rows"]);
|
|
414
610
|
});
|
|
415
611
|
});
|
|
416
612
|
});
|