@malloy-publisher/server 0.0.219 → 0.0.221
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +291 -20
- package/dist/app/assets/{EnvironmentPage-gehnjfC6.js → EnvironmentPage-D6G5n6mY.js} +1 -1
- package/dist/app/assets/{HomePage-8LQBytE4.js → HomePage-DgZluD8u.js} +1 -1
- package/dist/app/assets/LightMode-w9v0pEbg.js +1 -0
- package/dist/app/assets/MainPage-Bi_Tukvr.js +2 -0
- package/dist/app/assets/{MaterializationsPage-D7P1Kp6O.js → MaterializationsPage-BVERjnmG.js} +1 -1
- package/dist/app/assets/ModelPage-2OjS259-.js +1 -0
- package/dist/app/assets/{PackagePage-Cz9fVwSG.js → PackagePage-C8L2On4z.js} +1 -1
- package/dist/app/assets/{RouteError-UONCloyN.js → RouteError-DJoNC_Vl.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-B-hJ1zgd.js +1 -0
- package/dist/app/assets/{WorkbookPage-Bhzqvbq_.js → WorkbookPage-CTbDgGeJ.js} +1 -1
- package/dist/app/assets/{core-BiGj7BML.es-kMHAa8tP.js → core-D9Hl0IDY.es-CmgzHn4c.js} +1 -1
- package/dist/app/assets/github-dark-DenFmJkN.es-DHJKELXO.js +1 -0
- package/dist/app/assets/index-8E2uLeV9.js +2541 -0
- package/dist/app/assets/index-BEjJCJjX.js +1761 -0
- package/dist/app/assets/index-BF8PkFm8.js +1 -0
- package/dist/app/assets/index-Cr-asgoV.js +527 -0
- package/dist/app/assets/index-DcCw_qxr.js +18 -0
- package/dist/app/assets/index-w_0OQJgZ.js +23 -0
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +20 -2
- package/dist/runtime/publisher.js +6 -1
- package/dist/server.mjs +412 -55
- package/dist/sshcrypto-8m50vnmb.node +0 -0
- package/package.json +12 -12
- package/src/config.theme.spec.ts +178 -0
- package/src/config.ts +179 -0
- package/src/controller/materialization.controller.spec.ts +4 -4
- package/src/controller/materialization.controller.ts +2 -2
- package/src/controller/package.controller.spec.ts +89 -1
- package/src/controller/package.controller.ts +28 -3
- package/src/controller/theme.controller.ts +83 -0
- package/src/dto/connection.dto.ts +6 -0
- package/src/materialization_metrics.ts +1 -1
- package/src/package_load/package_load_pool.ts +2 -1
- package/src/package_load/package_load_worker.ts +5 -2
- package/src/package_load/protocol.ts +2 -1
- package/src/runtime/publisher.js +6 -1
- package/src/server.ts +58 -1
- package/src/service/build_plan.spec.ts +61 -17
- package/src/service/build_plan.ts +35 -11
- package/src/service/connection.ts +52 -6
- package/src/service/connection_fingerprint.spec.ts +102 -0
- package/src/service/manifest_loader.spec.ts +5 -5
- package/src/service/manifest_loader.ts +4 -4
- package/src/service/materialization_cron_gate.spec.ts +137 -0
- package/src/service/materialization_schedule_surface.spec.ts +30 -1
- package/src/service/materialization_service.spec.ts +47 -37
- package/src/service/materialization_service.ts +59 -50
- package/src/service/materialization_test_fixtures.ts +22 -6
- package/src/service/model.ts +1 -1
- package/src/service/package.ts +50 -4
- package/src/service/package_manifest.spec.ts +42 -4
- package/src/service/package_manifest.ts +53 -4
- package/src/service/theme_store.ts +199 -0
- package/src/storage/DatabaseInterface.ts +1 -1
- package/src/storage/StorageManager.ts +17 -0
- package/src/storage/duckdb/schema.ts +27 -6
- package/src/theme_key_parity.spec.ts +57 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +1 -1
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +5 -5
- package/tests/unit/controller/theme.controller.spec.ts +141 -0
- package/tests/unit/service/theme_store.spec.ts +139 -0
- package/dist/app/assets/MainPage-DDaZLJw-.js +0 -2
- package/dist/app/assets/ModelPage-Do_vhxOc.js +0 -1
- package/dist/app/assets/index-VzRbxcF7.js +0 -455
- package/dist/app/assets/index-ddq4-5hu.js +0 -1760
- package/dist/app/assets/index-qOQF9CXq.js +0 -40
- package/dist/app/assets/index.umd-B2kmxDh7.js +0 -2467
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
import { components } from "../api";
|
|
3
|
+
import {
|
|
4
|
+
buildEnvironmentMalloyConfig,
|
|
5
|
+
EnvironmentMalloyConfig,
|
|
6
|
+
} from "./connection";
|
|
7
|
+
|
|
8
|
+
type ApiConnection = components["schemas"]["Connection"];
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The connection `fingerprint` seam: when a connection carries an API
|
|
12
|
+
* fingerprint, the resolved Malloy connection's getDigest() must return it
|
|
13
|
+
* verbatim. Every consumer of a connection's identity — build planning, the
|
|
14
|
+
* package-load worker's digest RPC, and Malloy's serve-time manifest
|
|
15
|
+
* resolution — reads getDigest() off a connection resolved through the same
|
|
16
|
+
* environment lookup wrapper, so this one seam is what keeps build-time and
|
|
17
|
+
* serve-time source ids in agreement.
|
|
18
|
+
*
|
|
19
|
+
* Uses a postgres connection because it is constructed lazily (no network I/O
|
|
20
|
+
* until a query runs), so digests can be read offline.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
function pgConnection(overrides: Partial<ApiConnection> = {}): ApiConnection {
|
|
24
|
+
return {
|
|
25
|
+
name: "pg",
|
|
26
|
+
type: "postgres",
|
|
27
|
+
postgresConnection: {
|
|
28
|
+
host: "db.example.com",
|
|
29
|
+
port: 5432,
|
|
30
|
+
databaseName: "analytics",
|
|
31
|
+
userName: "svc_user",
|
|
32
|
+
password: "hunter2",
|
|
33
|
+
},
|
|
34
|
+
...overrides,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function digestOf(connection: ApiConnection): Promise<string> {
|
|
39
|
+
let config: EnvironmentMalloyConfig | undefined;
|
|
40
|
+
try {
|
|
41
|
+
config = buildEnvironmentMalloyConfig([connection]);
|
|
42
|
+
const resolved = await config.malloyConfig.connections.lookupConnection(
|
|
43
|
+
connection.name,
|
|
44
|
+
);
|
|
45
|
+
return resolved.getDigest();
|
|
46
|
+
} finally {
|
|
47
|
+
await config?.releaseConnections();
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
describe("connection fingerprint", () => {
|
|
52
|
+
it("uses the fingerprint verbatim as the connection digest", async () => {
|
|
53
|
+
const digest = await digestOf(
|
|
54
|
+
pgConnection({ fingerprint: "fp-opaque-token-1" }),
|
|
55
|
+
);
|
|
56
|
+
expect(digest).toBe("fp-opaque-token-1");
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("keeps the digest stable across a credential rotation", async () => {
|
|
60
|
+
const before = await digestOf(pgConnection({ fingerprint: "fp-stable" }));
|
|
61
|
+
const after = await digestOf(
|
|
62
|
+
pgConnection({
|
|
63
|
+
fingerprint: "fp-stable",
|
|
64
|
+
postgresConnection: {
|
|
65
|
+
host: "db.example.com",
|
|
66
|
+
port: 5432,
|
|
67
|
+
databaseName: "analytics",
|
|
68
|
+
userName: "rotated_user",
|
|
69
|
+
password: "rotated-password",
|
|
70
|
+
},
|
|
71
|
+
}),
|
|
72
|
+
);
|
|
73
|
+
expect(after).toBe(before);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("changes the digest when the fingerprint changes", async () => {
|
|
77
|
+
const one = await digestOf(pgConnection({ fingerprint: "fp-1" }));
|
|
78
|
+
const two = await digestOf(pgConnection({ fingerprint: "fp-2" }));
|
|
79
|
+
expect(one).not.toBe(two);
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it("falls back to the locally derived digest when omitted", async () => {
|
|
83
|
+
const digest = await digestOf(pgConnection());
|
|
84
|
+
expect(typeof digest).toBe("string");
|
|
85
|
+
expect(digest.length).toBeGreaterThan(0);
|
|
86
|
+
// Behavior is unchanged from before the fingerprint existed: the digest
|
|
87
|
+
// is the connection's own derivation, deterministic for one config.
|
|
88
|
+
expect(await digestOf(pgConnection())).toBe(digest);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it("round-trips the fingerprint on the API connection", async () => {
|
|
92
|
+
const config = buildEnvironmentMalloyConfig([
|
|
93
|
+
pgConnection({ fingerprint: "fp-round-trip" }),
|
|
94
|
+
]);
|
|
95
|
+
try {
|
|
96
|
+
const api = config.apiConnections.find((c) => c.name === "pg");
|
|
97
|
+
expect(api?.fingerprint).toBe("fp-round-trip");
|
|
98
|
+
} finally {
|
|
99
|
+
await config.releaseConnections();
|
|
100
|
+
}
|
|
101
|
+
});
|
|
102
|
+
});
|
|
@@ -29,9 +29,9 @@ describe("fetchManifestEntries", () => {
|
|
|
29
29
|
builtAt: new Date().toISOString(),
|
|
30
30
|
strict: false,
|
|
31
31
|
entries: {
|
|
32
|
-
b1: {
|
|
32
|
+
b1: { sourceEntityId: "b1", physicalTableName: "schema.orders_mz" },
|
|
33
33
|
b2: {
|
|
34
|
-
|
|
34
|
+
sourceEntityId: "b2",
|
|
35
35
|
physicalTableName: "schema.daily_mz",
|
|
36
36
|
connectionName: "bq",
|
|
37
37
|
},
|
|
@@ -48,7 +48,7 @@ describe("fetchManifestEntries", () => {
|
|
|
48
48
|
|
|
49
49
|
it("reads via a file:// URI", async () => {
|
|
50
50
|
const file = await writeManifest({
|
|
51
|
-
entries: { b1: {
|
|
51
|
+
entries: { b1: { sourceEntityId: "b1", physicalTableName: "t1" } },
|
|
52
52
|
});
|
|
53
53
|
|
|
54
54
|
const entries = await fetchManifestEntries(pathToFileURL(file).href);
|
|
@@ -59,8 +59,8 @@ describe("fetchManifestEntries", () => {
|
|
|
59
59
|
it("skips entries without a physicalTableName", async () => {
|
|
60
60
|
const file = await writeManifest({
|
|
61
61
|
entries: {
|
|
62
|
-
good: {
|
|
63
|
-
bad: {
|
|
62
|
+
good: { sourceEntityId: "good", physicalTableName: "t1" },
|
|
63
|
+
bad: { sourceEntityId: "bad" },
|
|
64
64
|
},
|
|
65
65
|
});
|
|
66
66
|
|
|
@@ -58,7 +58,7 @@ async function readManifestBytes(uri: string): Promise<string> {
|
|
|
58
58
|
|
|
59
59
|
/**
|
|
60
60
|
* Fetch and parse the control-plane-computed build manifest at `uri`, returning
|
|
61
|
-
* the Malloy-runtime binding map (`
|
|
61
|
+
* the Malloy-runtime binding map (`sourceEntityId -> { tableName }`). The wire manifest
|
|
62
62
|
* keys physical tables under `physicalTableName`; the Malloy runtime consumes
|
|
63
63
|
* `tableName`, so we translate here. Entries missing a physical table are
|
|
64
64
|
* skipped. Throws if the URI can't be read or parsed.
|
|
@@ -80,16 +80,16 @@ export async function fetchManifestEntries(
|
|
|
80
80
|
}
|
|
81
81
|
|
|
82
82
|
const entries: BuildManifest["entries"] = {};
|
|
83
|
-
for (const [
|
|
83
|
+
for (const [sourceEntityId, entry] of Object.entries(parsed.entries ?? {})) {
|
|
84
84
|
const physicalTableName = entry?.physicalTableName;
|
|
85
85
|
if (!physicalTableName) {
|
|
86
86
|
logger.warn("Manifest entry has no physicalTableName; skipping", {
|
|
87
87
|
uri,
|
|
88
|
-
|
|
88
|
+
sourceEntityId,
|
|
89
89
|
});
|
|
90
90
|
continue;
|
|
91
91
|
}
|
|
92
|
-
entries[
|
|
92
|
+
entries[sourceEntityId] = { tableName: physicalTableName };
|
|
93
93
|
}
|
|
94
94
|
return entries;
|
|
95
95
|
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import * as fs from "fs/promises";
|
|
3
|
+
import * as os from "os";
|
|
4
|
+
import * as path from "path";
|
|
5
|
+
|
|
6
|
+
import { Environment } from "./environment";
|
|
7
|
+
import type { Package } from "./package";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The publish gate for the package-level materialization cron (§9.4
|
|
11
|
+
* artifact-anchored scheduling): a declared `materialization.schedule` is the
|
|
12
|
+
* power tier and is valid only when every persist source in the compiled
|
|
13
|
+
* build plan resolves to an explicit `sharing=private`. These tests run a real
|
|
14
|
+
* `Environment` + `Package.create` over temp dirs, so they also prove
|
|
15
|
+
* end-to-end that the pinned compiler ACCEPTS `sharing=` / `refresh=` keys in
|
|
16
|
+
* the `#@ persist` annotation and that the values survive to the wire build
|
|
17
|
+
* plan verbatim (unset stays null — never defaulted to "shared").
|
|
18
|
+
*/
|
|
19
|
+
describe("materialization cron gate", () => {
|
|
20
|
+
let rootDir: string;
|
|
21
|
+
let envPath: string;
|
|
22
|
+
|
|
23
|
+
async function loadPackage(model: string, schedule?: string) {
|
|
24
|
+
const dir = path.join(envPath, "pkg");
|
|
25
|
+
await fs.mkdir(dir, { recursive: true });
|
|
26
|
+
await fs.writeFile(
|
|
27
|
+
path.join(dir, "publisher.json"),
|
|
28
|
+
JSON.stringify({
|
|
29
|
+
name: "pkg",
|
|
30
|
+
description: "fixture",
|
|
31
|
+
...(schedule ? { materialization: { schedule } } : {}),
|
|
32
|
+
}),
|
|
33
|
+
);
|
|
34
|
+
await fs.writeFile(path.join(dir, "model.malloy"), model);
|
|
35
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
36
|
+
await env.addPackage("pkg");
|
|
37
|
+
return env.getPackage("pkg", false);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function sourceByName(pkg: Package, name: string) {
|
|
41
|
+
const sources = pkg.getBuildPlan()?.sources ?? {};
|
|
42
|
+
const found = Object.values(sources).find((s) => s.name === name);
|
|
43
|
+
if (!found) throw new Error(`persist source '${name}' not in build plan`);
|
|
44
|
+
return found;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
beforeEach(async () => {
|
|
48
|
+
rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-cron-"));
|
|
49
|
+
envPath = path.join(rootDir, "env");
|
|
50
|
+
await fs.mkdir(envPath, { recursive: true });
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
afterEach(async () => {
|
|
54
|
+
await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
const MIXED_MODEL = `##! experimental.persistence
|
|
58
|
+
|
|
59
|
+
#@ persist name="priv_table" sharing=private refresh=incremental
|
|
60
|
+
source: priv is duckdb.sql("SELECT 1 as x")
|
|
61
|
+
|
|
62
|
+
#@ persist name="open_table" sharing=shared
|
|
63
|
+
source: open is duckdb.sql("SELECT 2 as x")
|
|
64
|
+
|
|
65
|
+
#@ persist name="unset_table"
|
|
66
|
+
source: unspecified is duckdb.sql("SELECT 3 as x")
|
|
67
|
+
`;
|
|
68
|
+
|
|
69
|
+
const ALL_PRIVATE_MODEL = `##! experimental.persistence
|
|
70
|
+
|
|
71
|
+
#@ persist name="a_table" sharing=private
|
|
72
|
+
source: a is duckdb.sql("SELECT 1 as x")
|
|
73
|
+
|
|
74
|
+
#@ persist name="b_table" sharing=private refresh=full
|
|
75
|
+
source: b is duckdb.sql("SELECT 2 as x")
|
|
76
|
+
`;
|
|
77
|
+
|
|
78
|
+
it(
|
|
79
|
+
"surfaces declared sharing/refresh verbatim on the build plan (null when unset)",
|
|
80
|
+
async () => {
|
|
81
|
+
const pkg = await loadPackage(MIXED_MODEL);
|
|
82
|
+
|
|
83
|
+
const priv = sourceByName(pkg, "priv");
|
|
84
|
+
expect(priv.sharing).toBe("private");
|
|
85
|
+
expect(priv.refresh).toBe("incremental");
|
|
86
|
+
|
|
87
|
+
const open = sourceByName(pkg, "open");
|
|
88
|
+
expect(open.sharing).toBe("shared");
|
|
89
|
+
expect(open.refresh).toBeNull();
|
|
90
|
+
|
|
91
|
+
// Unset must be reported as null — distinguishable from an explicit
|
|
92
|
+
// "shared" — because the control plane applies the platform default
|
|
93
|
+
// (unset => shared) itself.
|
|
94
|
+
const unspecified = sourceByName(pkg, "unspecified");
|
|
95
|
+
expect(unspecified.sharing).toBeNull();
|
|
96
|
+
expect(unspecified.refresh).toBeNull();
|
|
97
|
+
},
|
|
98
|
+
{ timeout: 30000 },
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
it(
|
|
102
|
+
"rejects a cron over shared/unset persist sources, pointing at freshness.window",
|
|
103
|
+
async () => {
|
|
104
|
+
const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
|
|
105
|
+
|
|
106
|
+
const warnings = pkg.scheduleWarnings();
|
|
107
|
+
// One actionable message per offending source; the compliant private
|
|
108
|
+
// source is not flagged.
|
|
109
|
+
expect(warnings).toHaveLength(2);
|
|
110
|
+
const joined = pkg.formatInvalidSchedule();
|
|
111
|
+
expect(joined).toContain("'open' resolves to 'shared'");
|
|
112
|
+
expect(joined).toContain("'unspecified' resolves to unset");
|
|
113
|
+
expect(joined).not.toContain("'priv'");
|
|
114
|
+
expect(joined).toContain("materialization.freshness.window");
|
|
115
|
+
},
|
|
116
|
+
{ timeout: 30000 },
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
it(
|
|
120
|
+
"allows a cron when every persist source is explicitly private",
|
|
121
|
+
async () => {
|
|
122
|
+
const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
|
|
123
|
+
expect(pkg.scheduleWarnings()).toEqual([]);
|
|
124
|
+
expect(pkg.formatInvalidSchedule()).toBe("");
|
|
125
|
+
},
|
|
126
|
+
{ timeout: 30000 },
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
it(
|
|
130
|
+
"is inert when no cron is declared, whatever the sources' sharing",
|
|
131
|
+
async () => {
|
|
132
|
+
const pkg = await loadPackage(MIXED_MODEL);
|
|
133
|
+
expect(pkg.scheduleWarnings()).toEqual([]);
|
|
134
|
+
},
|
|
135
|
+
{ timeout: 30000 },
|
|
136
|
+
);
|
|
137
|
+
});
|
|
@@ -53,6 +53,30 @@ describe("materialization schedule surfacing", () => {
|
|
|
53
53
|
const pkg = await env.getPackage("pkg", false);
|
|
54
54
|
expect(pkg.getPackageMetadata().materialization).toEqual({
|
|
55
55
|
schedule: "0 6 * * *",
|
|
56
|
+
freshness: null,
|
|
57
|
+
});
|
|
58
|
+
},
|
|
59
|
+
{ timeout: 20000 },
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
it(
|
|
63
|
+
"surfaces the manifest's materialization.freshness on load",
|
|
64
|
+
async () => {
|
|
65
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
66
|
+
await writePackageDir({
|
|
67
|
+
materialization: {
|
|
68
|
+
freshness: { window: "24h", fallback: "stale_ok" },
|
|
69
|
+
},
|
|
70
|
+
});
|
|
71
|
+
await env.addPackage("pkg");
|
|
72
|
+
|
|
73
|
+
// Freshness rides the same channel as schedule: present-and-verbatim
|
|
74
|
+
// when declared, null when unset — the control plane owns the
|
|
75
|
+
// scheduling/gating logic and only needs the values surfaced.
|
|
76
|
+
const pkg = await env.getPackage("pkg", false);
|
|
77
|
+
expect(pkg.getPackageMetadata().materialization).toEqual({
|
|
78
|
+
schedule: null,
|
|
79
|
+
freshness: { window: "24h", fallback: "stale_ok" },
|
|
56
80
|
});
|
|
57
81
|
},
|
|
58
82
|
{ timeout: 20000 },
|
|
@@ -71,6 +95,7 @@ describe("materialization schedule surfacing", () => {
|
|
|
71
95
|
const pkg = await env.getPackage("pkg", false);
|
|
72
96
|
expect(pkg.getPackageMetadata().materialization).toEqual({
|
|
73
97
|
schedule: null,
|
|
98
|
+
freshness: null,
|
|
74
99
|
});
|
|
75
100
|
},
|
|
76
101
|
{ timeout: 20000 },
|
|
@@ -91,11 +116,15 @@ describe("materialization schedule surfacing", () => {
|
|
|
91
116
|
name: "pkg",
|
|
92
117
|
description: "updated",
|
|
93
118
|
});
|
|
94
|
-
expect(updated.materialization).toEqual({
|
|
119
|
+
expect(updated.materialization).toEqual({
|
|
120
|
+
schedule: "0 6 * * *",
|
|
121
|
+
freshness: null,
|
|
122
|
+
});
|
|
95
123
|
|
|
96
124
|
const pkg = await env.getPackage("pkg", false);
|
|
97
125
|
expect(pkg.getPackageMetadata().materialization).toEqual({
|
|
98
126
|
schedule: "0 6 * * *",
|
|
127
|
+
freshness: null,
|
|
99
128
|
});
|
|
100
129
|
},
|
|
101
130
|
{ timeout: 20000 },
|
|
@@ -236,13 +236,15 @@ describe("MaterializationService", () => {
|
|
|
236
236
|
).toMatchObject({ mode: "orchestrated" });
|
|
237
237
|
});
|
|
238
238
|
|
|
239
|
-
it("rejects instructions referencing an unknown
|
|
239
|
+
it("rejects instructions referencing an unknown sourceEntityId before creating", async () => {
|
|
240
240
|
setPackage(ctx.environmentStore, {
|
|
241
241
|
getBuildPlan: () => makeBuildPlan(),
|
|
242
242
|
});
|
|
243
243
|
await expect(
|
|
244
244
|
ctx.service.createMaterialization("my-env", "pkg", {
|
|
245
|
-
buildInstructions: [
|
|
245
|
+
buildInstructions: [
|
|
246
|
+
makeInstruction({ sourceEntityId: "ghost" }),
|
|
247
|
+
],
|
|
246
248
|
}),
|
|
247
249
|
).rejects.toThrow(BadRequestError);
|
|
248
250
|
expect(ctx.repository.createMaterialization.called).toBe(false);
|
|
@@ -347,7 +349,7 @@ describe("MaterializationService", () => {
|
|
|
347
349
|
strict: false,
|
|
348
350
|
entries: {
|
|
349
351
|
b1: {
|
|
350
|
-
|
|
352
|
+
sourceEntityId: "b1",
|
|
351
353
|
physicalTableName: "orders_mz",
|
|
352
354
|
connectionName: "duckdb",
|
|
353
355
|
},
|
|
@@ -383,7 +385,7 @@ describe("MaterializationService", () => {
|
|
|
383
385
|
strict: false,
|
|
384
386
|
entries: {
|
|
385
387
|
b1: {
|
|
386
|
-
|
|
388
|
+
sourceEntityId: "b1",
|
|
387
389
|
physicalTableName: "orders_mz",
|
|
388
390
|
connectionName: "duckdb",
|
|
389
391
|
},
|
|
@@ -423,7 +425,7 @@ describe("MaterializationService", () => {
|
|
|
423
425
|
strict: false,
|
|
424
426
|
entries: {
|
|
425
427
|
b1: {
|
|
426
|
-
|
|
428
|
+
sourceEntityId: "b1",
|
|
427
429
|
// Container path on a hyphenated BigQuery project: each
|
|
428
430
|
// segment must be backtick-quoted independently.
|
|
429
431
|
physicalTableName: "my-proj.ds.engaged",
|
|
@@ -460,7 +462,7 @@ describe("MaterializationService", () => {
|
|
|
460
462
|
strict: false,
|
|
461
463
|
entries: {
|
|
462
464
|
b1: {
|
|
463
|
-
|
|
465
|
+
sourceEntityId: "b1",
|
|
464
466
|
physicalTableName: "orders_mz",
|
|
465
467
|
connectionName: "duckdb",
|
|
466
468
|
},
|
|
@@ -481,7 +483,7 @@ describe("MaterializationService", () => {
|
|
|
481
483
|
});
|
|
482
484
|
|
|
483
485
|
describe("stagingSuffix", () => {
|
|
484
|
-
it("derives a short, stable suffix from the
|
|
486
|
+
it("derives a short, stable suffix from the sourceEntityId", () => {
|
|
485
487
|
expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
|
|
486
488
|
});
|
|
487
489
|
});
|
|
@@ -492,17 +494,17 @@ describe("deriveSelfInstructions", () => {
|
|
|
492
494
|
ctx = createMocks();
|
|
493
495
|
});
|
|
494
496
|
|
|
495
|
-
it("carries forward unchanged
|
|
497
|
+
it("carries forward unchanged sourceEntityIds and builds the rest (deduping repeats)", () => {
|
|
496
498
|
const compiled = compiledWith(
|
|
497
499
|
{
|
|
498
|
-
s1: fakeSource({ name: "s1",
|
|
499
|
-
s2: fakeSource({ name: "s2",
|
|
500
|
+
s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
|
|
501
|
+
s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
|
|
500
502
|
},
|
|
501
503
|
[["s1", "s2"], ["s2"]],
|
|
502
504
|
);
|
|
503
505
|
const priorEntries = {
|
|
504
506
|
b1aaaaaaaaaaaaaa: {
|
|
505
|
-
|
|
507
|
+
sourceEntityId: "b1aaaaaaaaaaaaaa",
|
|
506
508
|
physicalTableName: "s1_prev",
|
|
507
509
|
connectionName: "duckdb",
|
|
508
510
|
},
|
|
@@ -519,7 +521,7 @@ describe("deriveSelfInstructions", () => {
|
|
|
519
521
|
).deriveSelfInstructions(compiled, undefined, priorEntries);
|
|
520
522
|
|
|
521
523
|
expect(instructions).toHaveLength(1);
|
|
522
|
-
expect(instructions[0].
|
|
524
|
+
expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
|
|
523
525
|
expect(instructions[0].physicalTableName).toBe("s2");
|
|
524
526
|
expect(instructions[0].realization).toBe("COPY");
|
|
525
527
|
expect(instructions[0].materializedTableId).toBe("local-b2bbbbbbbbbb");
|
|
@@ -531,8 +533,8 @@ describe("deriveSelfInstructions", () => {
|
|
|
531
533
|
it("honors the sourceNames filter (excluded sources are neither built nor carried)", () => {
|
|
532
534
|
const compiled = compiledWith(
|
|
533
535
|
{
|
|
534
|
-
s1: fakeSource({ name: "s1",
|
|
535
|
-
s2: fakeSource({ name: "s2",
|
|
536
|
+
s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
|
|
537
|
+
s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
|
|
536
538
|
},
|
|
537
539
|
[["s1", "s2"]],
|
|
538
540
|
);
|
|
@@ -547,7 +549,7 @@ describe("deriveSelfInstructions", () => {
|
|
|
547
549
|
).deriveSelfInstructions(compiled, ["s2"], {});
|
|
548
550
|
|
|
549
551
|
expect(instructions).toHaveLength(1);
|
|
550
|
-
expect(instructions[0].
|
|
552
|
+
expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
|
|
551
553
|
expect(Object.keys(carried as Record<string, unknown>)).toHaveLength(0);
|
|
552
554
|
});
|
|
553
555
|
});
|
|
@@ -561,7 +563,7 @@ describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
|
|
|
561
563
|
it("returns entries from the most recent successful run, excluding the in-flight one", async () => {
|
|
562
564
|
const entries = {
|
|
563
565
|
b1: {
|
|
564
|
-
|
|
566
|
+
sourceEntityId: "b1",
|
|
565
567
|
physicalTableName: "orders_v1",
|
|
566
568
|
connectionName: "duckdb",
|
|
567
569
|
},
|
|
@@ -599,7 +601,9 @@ describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
|
|
|
599
601
|
manifest: {
|
|
600
602
|
builtAt: "t",
|
|
601
603
|
strict: false,
|
|
602
|
-
entries: {
|
|
604
|
+
entries: {
|
|
605
|
+
b1: { sourceEntityId: "b1", physicalTableName: "t1" },
|
|
606
|
+
},
|
|
603
607
|
},
|
|
604
608
|
}),
|
|
605
609
|
]);
|
|
@@ -679,8 +683,8 @@ describe("executeInstructedBuild", () => {
|
|
|
679
683
|
it("builds instructed sources in dependency order and seeds carried entries", async () => {
|
|
680
684
|
const runSQL = sinon.stub().resolves();
|
|
681
685
|
const connection = { runSQL } as unknown as MalloyConnection;
|
|
682
|
-
const s1 = fakeSource({ name: "s1",
|
|
683
|
-
const s2 = fakeSource({ name: "s2",
|
|
686
|
+
const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
|
|
687
|
+
const s2 = fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" });
|
|
684
688
|
const compiled = compiledWith(
|
|
685
689
|
{ s1, s2 },
|
|
686
690
|
[["s1"], ["s2"]],
|
|
@@ -691,13 +695,13 @@ describe("executeInstructedBuild", () => {
|
|
|
691
695
|
compiled,
|
|
692
696
|
[
|
|
693
697
|
{
|
|
694
|
-
|
|
698
|
+
sourceEntityId: "b1aaaaaaaaaaaaaa",
|
|
695
699
|
materializedTableId: "mt-1",
|
|
696
700
|
physicalTableName: "s1_v1",
|
|
697
701
|
realization: "COPY",
|
|
698
702
|
},
|
|
699
703
|
{
|
|
700
|
-
|
|
704
|
+
sourceEntityId: "b2bbbbbbbbbbbbbb",
|
|
701
705
|
materializedTableId: "mt-2",
|
|
702
706
|
physicalTableName: "s2_v1",
|
|
703
707
|
realization: "COPY",
|
|
@@ -705,7 +709,7 @@ describe("executeInstructedBuild", () => {
|
|
|
705
709
|
],
|
|
706
710
|
{
|
|
707
711
|
carried0: {
|
|
708
|
-
|
|
712
|
+
sourceEntityId: "carried0",
|
|
709
713
|
physicalTableName: "carried_tbl",
|
|
710
714
|
connectionName: "duckdb",
|
|
711
715
|
},
|
|
@@ -725,8 +729,14 @@ describe("executeInstructedBuild", () => {
|
|
|
725
729
|
// though the caller instructed it. Both must now build, `mid` first.
|
|
726
730
|
const runSQL = sinon.stub().resolves();
|
|
727
731
|
const connection = { runSQL } as unknown as MalloyConnection;
|
|
728
|
-
const root = fakeSource({
|
|
729
|
-
|
|
732
|
+
const root = fakeSource({
|
|
733
|
+
name: "root",
|
|
734
|
+
sourceEntityId: "brootaaaaaaaaaa",
|
|
735
|
+
});
|
|
736
|
+
const mid = fakeSource({
|
|
737
|
+
name: "mid",
|
|
738
|
+
sourceEntityId: "bmidbbbbbbbbbbb",
|
|
739
|
+
});
|
|
730
740
|
const compiled = {
|
|
731
741
|
graphs: [
|
|
732
742
|
{
|
|
@@ -750,13 +760,13 @@ describe("executeInstructedBuild", () => {
|
|
|
750
760
|
compiled,
|
|
751
761
|
[
|
|
752
762
|
{
|
|
753
|
-
|
|
763
|
+
sourceEntityId: "brootaaaaaaaaaa",
|
|
754
764
|
materializedTableId: "mt-r",
|
|
755
765
|
physicalTableName: "root_v1",
|
|
756
766
|
realization: "COPY",
|
|
757
767
|
},
|
|
758
768
|
{
|
|
759
|
-
|
|
769
|
+
sourceEntityId: "bmidbbbbbbbbbbb",
|
|
760
770
|
materializedTableId: "mt-m",
|
|
761
771
|
physicalTableName: "mid_v1",
|
|
762
772
|
realization: "COPY",
|
|
@@ -779,7 +789,7 @@ describe("executeInstructedBuild", () => {
|
|
|
779
789
|
});
|
|
780
790
|
|
|
781
791
|
it("throws when an instructed graph's connection is missing", async () => {
|
|
782
|
-
const s1 = fakeSource({ name: "s1",
|
|
792
|
+
const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
|
|
783
793
|
const compiled = compiledWith({ s1 }, [["s1"]], new Map());
|
|
784
794
|
|
|
785
795
|
await expect(
|
|
@@ -787,7 +797,7 @@ describe("executeInstructedBuild", () => {
|
|
|
787
797
|
compiled,
|
|
788
798
|
[
|
|
789
799
|
{
|
|
790
|
-
|
|
800
|
+
sourceEntityId: "b1aaaaaaaaaaaaaa",
|
|
791
801
|
materializedTableId: "mt-1",
|
|
792
802
|
physicalTableName: "s1_v1",
|
|
793
803
|
realization: "COPY",
|
|
@@ -809,15 +819,15 @@ describe("buildOneSource", () => {
|
|
|
809
819
|
connection: { runSQL: sinon.SinonStub },
|
|
810
820
|
physicalTableName: string,
|
|
811
821
|
dialectName?: string,
|
|
812
|
-
): Promise<{
|
|
822
|
+
): Promise<{ sourceEntityId: string; physicalTableName: string }> {
|
|
813
823
|
const source = fakeSource({
|
|
814
824
|
name: "orders",
|
|
815
|
-
|
|
825
|
+
sourceEntityId: "abcdef1234567890",
|
|
816
826
|
sql: "SELECT * FROM t",
|
|
817
827
|
dialectName,
|
|
818
828
|
});
|
|
819
829
|
const instruction: BuildInstruction = {
|
|
820
|
-
|
|
830
|
+
sourceEntityId: "abcdef1234567890",
|
|
821
831
|
materializedTableId: "mt-1",
|
|
822
832
|
physicalTableName,
|
|
823
833
|
realization: "COPY",
|
|
@@ -830,7 +840,7 @@ describe("buildOneSource", () => {
|
|
|
830
840
|
c: unknown,
|
|
831
841
|
d: Record<string, string>,
|
|
832
842
|
m: Manifest,
|
|
833
|
-
) => Promise<{
|
|
843
|
+
) => Promise<{ sourceEntityId: string; physicalTableName: string }>;
|
|
834
844
|
}
|
|
835
845
|
).buildOneSource(
|
|
836
846
|
source,
|
|
@@ -853,7 +863,7 @@ describe("buildOneSource", () => {
|
|
|
853
863
|
'ALTER TABLE "orders_v1_abcdef123456" RENAME TO "orders_v1"',
|
|
854
864
|
]);
|
|
855
865
|
expect(entry.physicalTableName).toBe("orders_v1");
|
|
856
|
-
expect(entry.
|
|
866
|
+
expect(entry.sourceEntityId).toBe("abcdef1234567890");
|
|
857
867
|
});
|
|
858
868
|
|
|
859
869
|
it("drops the staging table and rethrows when the build SQL fails", async () => {
|
|
@@ -919,7 +929,7 @@ describe("runBuild (branch behavior)", () => {
|
|
|
919
929
|
function stubEngine(): RunBuildInternals {
|
|
920
930
|
const entries = {
|
|
921
931
|
"build-orders": {
|
|
922
|
-
|
|
932
|
+
sourceEntityId: "build-orders",
|
|
923
933
|
physicalTableName: '"orders_v1"',
|
|
924
934
|
connectionName: "duckdb",
|
|
925
935
|
},
|
|
@@ -995,11 +1005,11 @@ describe("autoLoadManifest", () => {
|
|
|
995
1005
|
const reloadAllModelsForPackage = sinon.stub().resolves();
|
|
996
1006
|
const entries = {
|
|
997
1007
|
b1: {
|
|
998
|
-
|
|
1008
|
+
sourceEntityId: "b1",
|
|
999
1009
|
physicalTableName: "orders_v1",
|
|
1000
1010
|
connectionName: "duckdb",
|
|
1001
1011
|
},
|
|
1002
|
-
b2: {
|
|
1012
|
+
b2: { sourceEntityId: "b2", physicalTableName: undefined },
|
|
1003
1013
|
};
|
|
1004
1014
|
|
|
1005
1015
|
await (
|
|
@@ -1033,7 +1043,7 @@ describe("autoLoadManifest", () => {
|
|
|
1033
1043
|
) => Promise<void>;
|
|
1034
1044
|
}
|
|
1035
1045
|
).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", {
|
|
1036
|
-
b1: {
|
|
1046
|
+
b1: { sourceEntityId: "b1", physicalTableName: "t" },
|
|
1037
1047
|
}),
|
|
1038
1048
|
).resolves.toBeUndefined();
|
|
1039
1049
|
});
|