@malloy-publisher/server 0.0.219 → 0.0.221
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +291 -20
- package/dist/app/assets/{EnvironmentPage-gehnjfC6.js → EnvironmentPage-D6G5n6mY.js} +1 -1
- package/dist/app/assets/{HomePage-8LQBytE4.js → HomePage-DgZluD8u.js} +1 -1
- package/dist/app/assets/LightMode-w9v0pEbg.js +1 -0
- package/dist/app/assets/MainPage-Bi_Tukvr.js +2 -0
- package/dist/app/assets/{MaterializationsPage-D7P1Kp6O.js → MaterializationsPage-BVERjnmG.js} +1 -1
- package/dist/app/assets/ModelPage-2OjS259-.js +1 -0
- package/dist/app/assets/{PackagePage-Cz9fVwSG.js → PackagePage-C8L2On4z.js} +1 -1
- package/dist/app/assets/{RouteError-UONCloyN.js → RouteError-DJoNC_Vl.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-B-hJ1zgd.js +1 -0
- package/dist/app/assets/{WorkbookPage-Bhzqvbq_.js → WorkbookPage-CTbDgGeJ.js} +1 -1
- package/dist/app/assets/{core-BiGj7BML.es-kMHAa8tP.js → core-D9Hl0IDY.es-CmgzHn4c.js} +1 -1
- package/dist/app/assets/github-dark-DenFmJkN.es-DHJKELXO.js +1 -0
- package/dist/app/assets/index-8E2uLeV9.js +2541 -0
- package/dist/app/assets/index-BEjJCJjX.js +1761 -0
- package/dist/app/assets/index-BF8PkFm8.js +1 -0
- package/dist/app/assets/index-Cr-asgoV.js +527 -0
- package/dist/app/assets/index-DcCw_qxr.js +18 -0
- package/dist/app/assets/index-w_0OQJgZ.js +23 -0
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +20 -2
- package/dist/runtime/publisher.js +6 -1
- package/dist/server.mjs +412 -55
- package/dist/sshcrypto-8m50vnmb.node +0 -0
- package/package.json +12 -12
- package/src/config.theme.spec.ts +178 -0
- package/src/config.ts +179 -0
- package/src/controller/materialization.controller.spec.ts +4 -4
- package/src/controller/materialization.controller.ts +2 -2
- package/src/controller/package.controller.spec.ts +89 -1
- package/src/controller/package.controller.ts +28 -3
- package/src/controller/theme.controller.ts +83 -0
- package/src/dto/connection.dto.ts +6 -0
- package/src/materialization_metrics.ts +1 -1
- package/src/package_load/package_load_pool.ts +2 -1
- package/src/package_load/package_load_worker.ts +5 -2
- package/src/package_load/protocol.ts +2 -1
- package/src/runtime/publisher.js +6 -1
- package/src/server.ts +58 -1
- package/src/service/build_plan.spec.ts +61 -17
- package/src/service/build_plan.ts +35 -11
- package/src/service/connection.ts +52 -6
- package/src/service/connection_fingerprint.spec.ts +102 -0
- package/src/service/manifest_loader.spec.ts +5 -5
- package/src/service/manifest_loader.ts +4 -4
- package/src/service/materialization_cron_gate.spec.ts +137 -0
- package/src/service/materialization_schedule_surface.spec.ts +30 -1
- package/src/service/materialization_service.spec.ts +47 -37
- package/src/service/materialization_service.ts +59 -50
- package/src/service/materialization_test_fixtures.ts +22 -6
- package/src/service/model.ts +1 -1
- package/src/service/package.ts +50 -4
- package/src/service/package_manifest.spec.ts +42 -4
- package/src/service/package_manifest.ts +53 -4
- package/src/service/theme_store.ts +199 -0
- package/src/storage/DatabaseInterface.ts +1 -1
- package/src/storage/StorageManager.ts +17 -0
- package/src/storage/duckdb/schema.ts +27 -6
- package/src/theme_key_parity.spec.ts +57 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +1 -1
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +5 -5
- package/tests/unit/controller/theme.controller.spec.ts +141 -0
- package/tests/unit/service/theme_store.spec.ts +139 -0
- package/dist/app/assets/MainPage-DDaZLJw-.js +0 -2
- package/dist/app/assets/ModelPage-Do_vhxOc.js +0 -1
- package/dist/app/assets/index-VzRbxcF7.js +0 -455
- package/dist/app/assets/index-ddq4-5hu.js +0 -1760
- package/dist/app/assets/index-qOQF9CXq.js +0 -40
- package/dist/app/assets/index.umd-B2kmxDh7.js +0 -2467
|
@@ -5,6 +5,30 @@ import { EnvironmentStore } from "../service/environment_store";
|
|
|
5
5
|
|
|
6
6
|
type ApiPackage = components["schemas"]["Package"];
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* Everything that is strict-at-publish, joined into one 400 message (or
|
|
10
|
+
* undefined when the package is publishable): invalid explores entries plus
|
|
11
|
+
* the materialization cron gate (a package-level `schedule` requires every
|
|
12
|
+
* persist source to resolve to explicit `sharing=private` — see
|
|
13
|
+
* Package.scheduleWarnings). At startup/reload both are warn-only instead
|
|
14
|
+
* (fail-safe; see Package.loadViaWorker).
|
|
15
|
+
*/
|
|
16
|
+
function formatPublishRejections(
|
|
17
|
+
pkg: {
|
|
18
|
+
formatInvalidExplores(exploresOverride?: string[]): string;
|
|
19
|
+
formatInvalidSchedule(): string;
|
|
20
|
+
},
|
|
21
|
+
exploresOverride?: string[],
|
|
22
|
+
): string | undefined {
|
|
23
|
+
const message = [
|
|
24
|
+
pkg.formatInvalidExplores(exploresOverride),
|
|
25
|
+
pkg.formatInvalidSchedule(),
|
|
26
|
+
]
|
|
27
|
+
.filter(Boolean)
|
|
28
|
+
.join("\n");
|
|
29
|
+
return message || undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
8
32
|
export class PackageController {
|
|
9
33
|
private environmentStore: EnvironmentStore;
|
|
10
34
|
|
|
@@ -100,7 +124,7 @@ export class PackageController {
|
|
|
100
124
|
bodyLocation,
|
|
101
125
|
stagingPath,
|
|
102
126
|
),
|
|
103
|
-
(pkg) => pkg
|
|
127
|
+
(pkg) => formatPublishRejections(pkg),
|
|
104
128
|
);
|
|
105
129
|
} else {
|
|
106
130
|
result = await environment.addPackage(packageName);
|
|
@@ -114,7 +138,7 @@ export class PackageController {
|
|
|
114
138
|
}
|
|
115
139
|
|
|
116
140
|
if (!body.location) {
|
|
117
|
-
const invalidMsg = result
|
|
141
|
+
const invalidMsg = formatPublishRejections(result);
|
|
118
142
|
if (invalidMsg) {
|
|
119
143
|
await environment.unloadPackage(packageName).catch(() => {
|
|
120
144
|
/* best-effort; the package is not persisted below */
|
|
@@ -177,7 +201,8 @@ export class PackageController {
|
|
|
177
201
|
stagingPath,
|
|
178
202
|
),
|
|
179
203
|
(pkg) =>
|
|
180
|
-
|
|
204
|
+
formatPublishRejections(
|
|
205
|
+
pkg,
|
|
181
206
|
body.explores?.map(normalizeModelPath),
|
|
182
207
|
),
|
|
183
208
|
);
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { components } from "../api";
|
|
2
|
+
import { sanitizeTheme, Theme } from "../config";
|
|
3
|
+
import { BadRequestError, FrozenConfigError } from "../errors";
|
|
4
|
+
import { isPublisherConfigFrozen } from "../config";
|
|
5
|
+
import { ThemeStore } from "../service/theme_store";
|
|
6
|
+
|
|
7
|
+
type ApiTheme = components["schemas"]["Theme"];
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Handlers for the in-app Theme Editor.
|
|
11
|
+
*
|
|
12
|
+
* GET /api/v0/theme → current saved theme (or built-in default)
|
|
13
|
+
* PUT /api/v0/theme → save the supplied theme
|
|
14
|
+
* DELETE /api/v0/theme → reset to boot seed / built-in default
|
|
15
|
+
*/
|
|
16
|
+
export class ThemeController {
|
|
17
|
+
constructor(
|
|
18
|
+
private themeStore: ThemeStore,
|
|
19
|
+
private serverRoot: string,
|
|
20
|
+
) {}
|
|
21
|
+
|
|
22
|
+
getTheme = async (): Promise<ApiTheme> => {
|
|
23
|
+
const theme = await this.themeStore.get();
|
|
24
|
+
// Always return an object so the client never has to deal with the
|
|
25
|
+
// null/undefined ambiguity. An empty object means "no overrides yet,
|
|
26
|
+
// use SDK defaults."
|
|
27
|
+
return theme ?? {};
|
|
28
|
+
};
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* PUT persists the supplied theme verbatim, including a literal `{}`.
|
|
32
|
+
* An empty `{}` is an intentional "clear all overrides" save: it writes
|
|
33
|
+
* an (empty) row so the boot seed from publisher.config.json's `theme`
|
|
34
|
+
* block is suppressed on subsequent loads. This deliberately differs
|
|
35
|
+
* from DELETE (resetTheme), which drops the row and lets that config
|
|
36
|
+
* seed re-apply. Note the persisted `"{}"` sanitizes back to `undefined`
|
|
37
|
+
* on reload (sanitizeTheme returns undefined for `{}`), but the row
|
|
38
|
+
* still exists, so the seed stays suppressed. This PUT-`{}` path is not
|
|
39
|
+
* reachable from the Theme Editor UI today (its clear affordance is the
|
|
40
|
+
* Reset button, which calls DELETE); it exists only via the API.
|
|
41
|
+
*/
|
|
42
|
+
putTheme = async (body: unknown): Promise<ApiTheme> => {
|
|
43
|
+
if (isPublisherConfigFrozen(this.serverRoot)) {
|
|
44
|
+
throw new FrozenConfigError(
|
|
45
|
+
'Cannot edit theme: publisher.config.json has "frozenConfig": true.',
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
// Treat a literal `{}` as an intentional "clear all overrides" PUT
|
|
49
|
+
// (the only other clear path is DELETE, which re-seeds from the
|
|
50
|
+
// config file). sanitizeTheme returns undefined for both `{}` and
|
|
51
|
+
// garbage, so we have to disambiguate here.
|
|
52
|
+
const isEmptyObject =
|
|
53
|
+
typeof body === "object" &&
|
|
54
|
+
body !== null &&
|
|
55
|
+
!Array.isArray(body) &&
|
|
56
|
+
Object.keys(body as Record<string, unknown>).length === 0;
|
|
57
|
+
const sanitized = sanitizeTheme(body, "PUT /api/v0/theme");
|
|
58
|
+
if (!sanitized && !isEmptyObject) {
|
|
59
|
+
throw new BadRequestError(
|
|
60
|
+
"Theme payload was malformed (expected an object).",
|
|
61
|
+
);
|
|
62
|
+
}
|
|
63
|
+
const saved: Theme = await this.themeStore.set(sanitized ?? {});
|
|
64
|
+
return saved;
|
|
65
|
+
};
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* DELETE drops the persisted row so the next load re-seeds from
|
|
69
|
+
* publisher.config.json's `theme` block (or ends up unthemed if the
|
|
70
|
+
* config defines none). Contrast with PUT `{}`, which persists an empty
|
|
71
|
+
* row and thereby suppresses that seed instead of restoring it. See
|
|
72
|
+
* putTheme for the full divergence.
|
|
73
|
+
*/
|
|
74
|
+
resetTheme = async (): Promise<ApiTheme> => {
|
|
75
|
+
if (isPublisherConfigFrozen(this.serverRoot)) {
|
|
76
|
+
throw new FrozenConfigError(
|
|
77
|
+
'Cannot reset theme: publisher.config.json has "frozenConfig": true.',
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
const reseeded = await this.themeStore.reset();
|
|
81
|
+
return reseeded ?? {};
|
|
82
|
+
};
|
|
83
|
+
}
|
|
@@ -265,6 +265,12 @@ export class ConnectionDto implements ApiConnection {
|
|
|
265
265
|
| "motherduck"
|
|
266
266
|
| "ducklake";
|
|
267
267
|
|
|
268
|
+
// Opaque, secret-free data-identity token; carried verbatim (never parsed
|
|
269
|
+
// or re-hashed) into content-addressed source ids. See api-doc.yaml.
|
|
270
|
+
@IsOptional()
|
|
271
|
+
@IsString()
|
|
272
|
+
fingerprint?: string;
|
|
273
|
+
|
|
268
274
|
@IsOptional()
|
|
269
275
|
@ValidateNested()
|
|
270
276
|
@Type(() => PostgresConnectionDto)
|
|
@@ -131,7 +131,7 @@ export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
|
|
|
131
131
|
|
|
132
132
|
/**
|
|
133
133
|
* Record that a connection digest could not be computed during build-plan
|
|
134
|
-
* compilation because the connection failed to resolve. The resulting
|
|
134
|
+
* compilation because the connection failed to resolve. The resulting sourceEntityIds
|
|
135
135
|
* are computed without a digest, so this is a correctness signal, not just noise.
|
|
136
136
|
*/
|
|
137
137
|
export function recordConnectionDigestSkipped(): void {
|
|
@@ -81,6 +81,7 @@ import { fileURLToPath, pathToFileURL } from "url";
|
|
|
81
81
|
|
|
82
82
|
import { ModelCompilationError } from "../errors";
|
|
83
83
|
import { logger } from "../logger";
|
|
84
|
+
import type { PackageMaterializationConfig } from "../service/package_manifest";
|
|
84
85
|
import type {
|
|
85
86
|
ConnectionMetadataRequest,
|
|
86
87
|
ConnectionMetadataResponse,
|
|
@@ -233,7 +234,7 @@ export interface LoadPackageOutcome {
|
|
|
233
234
|
explores?: string[];
|
|
234
235
|
queryableSources?: "declared" | "all";
|
|
235
236
|
manifestLocation?: string | null;
|
|
236
|
-
materialization?:
|
|
237
|
+
materialization?: PackageMaterializationConfig | null;
|
|
237
238
|
};
|
|
238
239
|
models: Array<
|
|
239
240
|
Omit<SerializedModel, "modelDef" | "sourceInfos"> & {
|
|
@@ -83,7 +83,10 @@ import { HackyDataStylesAccumulator } from "../data_styles";
|
|
|
83
83
|
import { ModelCompilationError } from "../errors";
|
|
84
84
|
import { validateAuthorizeProbes } from "../service/authorize";
|
|
85
85
|
import { type FilterDefinition } from "../service/filter";
|
|
86
|
-
import {
|
|
86
|
+
import {
|
|
87
|
+
PackageMaterializationConfig,
|
|
88
|
+
parsePackageMaterialization,
|
|
89
|
+
} from "../service/package_manifest";
|
|
87
90
|
import {
|
|
88
91
|
extractQueriesFromModelDef,
|
|
89
92
|
extractSourcesFromModelDef,
|
|
@@ -383,7 +386,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
|
|
|
383
386
|
explores?: string[];
|
|
384
387
|
queryableSources?: "declared" | "all";
|
|
385
388
|
manifestLocation?: string | null;
|
|
386
|
-
materialization?:
|
|
389
|
+
materialization?: PackageMaterializationConfig | null;
|
|
387
390
|
}> {
|
|
388
391
|
const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
|
|
389
392
|
const contents = await fs.promises.readFile(manifestPath, "utf8");
|
|
@@ -65,6 +65,7 @@
|
|
|
65
65
|
*/
|
|
66
66
|
|
|
67
67
|
import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
|
|
68
|
+
import type { PackageMaterializationConfig } from "../service/package_manifest";
|
|
68
69
|
|
|
69
70
|
// ──────────────────────────────────────────────────────────────────────
|
|
70
71
|
// Direction: main ──▶ worker (load-package job)
|
|
@@ -179,7 +180,7 @@ export interface LoadPackageResult {
|
|
|
179
180
|
explores?: string[];
|
|
180
181
|
queryableSources?: "declared" | "all";
|
|
181
182
|
manifestLocation?: string | null;
|
|
182
|
-
materialization?:
|
|
183
|
+
materialization?: PackageMaterializationConfig | null;
|
|
183
184
|
};
|
|
184
185
|
models: SerializedModel[];
|
|
185
186
|
/** Wall-clock ms inside the worker for the full package load. */
|
package/src/runtime/publisher.js
CHANGED
|
@@ -79,7 +79,12 @@
|
|
|
79
79
|
"/packages/" +
|
|
80
80
|
encodeURIComponent(target.pkg) +
|
|
81
81
|
"/models/" +
|
|
82
|
-
|
|
82
|
+
// Encode the whole model path as ONE path segment (slashes -> %2F). A model
|
|
83
|
+
// in a subfolder ("model/x.malloy") must not become two URL segments: some
|
|
84
|
+
// servers only match a single-segment {path} and would 405 a multi-segment
|
|
85
|
+
// path. This matches the typed SDK (encodeURIComponent) and the server's
|
|
86
|
+
// encoded-slash handling.
|
|
87
|
+
encodeURIComponent(modelPath) +
|
|
83
88
|
"/query";
|
|
84
89
|
var body = { compactJson: compactJson };
|
|
85
90
|
if (malloyQuery) body.query = malloyQuery;
|
package/src/server.ts
CHANGED
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
BadRequestError,
|
|
21
21
|
internalErrorToHttpError,
|
|
22
22
|
NotImplementedError,
|
|
23
|
+
ServiceUnavailableError,
|
|
23
24
|
} from "./errors";
|
|
24
25
|
import {
|
|
25
26
|
drainingGuard,
|
|
@@ -38,6 +39,7 @@ import { setFilterDeprecationHeaders } from "./filter_deprecation";
|
|
|
38
39
|
import { checkHeapConfiguration } from "./heap_check";
|
|
39
40
|
import { queryConcurrency } from "./query_concurrency";
|
|
40
41
|
import { MaterializationController } from "./controller/materialization.controller";
|
|
42
|
+
import { ThemeController } from "./controller/theme.controller";
|
|
41
43
|
import { initializeMcpServer } from "./mcp/server";
|
|
42
44
|
import { startAgentMcpServer } from "./mcp/agent_server";
|
|
43
45
|
import { registerLegacyRoutes } from "./server-old";
|
|
@@ -45,6 +47,7 @@ import { EnvironmentStore } from "./service/environment_store";
|
|
|
45
47
|
import { MaterializationService } from "./service/materialization_service";
|
|
46
48
|
import { normalizeQueryArray } from "./query_param_utils";
|
|
47
49
|
import { PackageMemoryGovernor } from "./service/package_memory_governor";
|
|
50
|
+
import { ThemeStore } from "./service/theme_store";
|
|
48
51
|
import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
|
|
49
52
|
|
|
50
53
|
export { normalizeQueryArray } from "./query_param_utils";
|
|
@@ -207,6 +210,8 @@ const materializationService = new MaterializationService(environmentStore);
|
|
|
207
210
|
const materializationController = new MaterializationController(
|
|
208
211
|
materializationService,
|
|
209
212
|
);
|
|
213
|
+
const themeStore = new ThemeStore(environmentStore.storageManager, SERVER_ROOT);
|
|
214
|
+
const themeController = new ThemeController(themeStore, SERVER_ROOT);
|
|
210
215
|
|
|
211
216
|
export const mcpApp = express();
|
|
212
217
|
|
|
@@ -722,7 +727,20 @@ app.get(
|
|
|
722
727
|
app.get(`${API_PREFIX}/status`, async (_req, res) => {
|
|
723
728
|
try {
|
|
724
729
|
const status = await environmentStore.getStatus();
|
|
725
|
-
|
|
730
|
+
// Compose theme onto the status response so the SDK can read both
|
|
731
|
+
// in one round trip on app boot. ThemeStore is the source of truth;
|
|
732
|
+
// publisher.config.json is only a boot seed (see ThemeStore). The
|
|
733
|
+
// field is always present (an empty object means "no overrides
|
|
734
|
+
// yet"), so the OpenAPI shape and the runtime payload agree.
|
|
735
|
+
// The theme here is cosmetic, so during the brief window before storage
|
|
736
|
+
// initializes report no overrides rather than 500 an endpoint the
|
|
737
|
+
// control plane polls for serving state (themeStore.get() throws until
|
|
738
|
+
// storage is ready). GET /theme, the editor's authoritative load, is
|
|
739
|
+
// answered with 503 during that window instead.
|
|
740
|
+
const theme = environmentStore.storageManager.isInitialized()
|
|
741
|
+
? await themeStore.get()
|
|
742
|
+
: undefined;
|
|
743
|
+
res.status(200).json({ ...status, theme: theme ?? {} });
|
|
726
744
|
} catch (error) {
|
|
727
745
|
logger.error("Error getting status", { error });
|
|
728
746
|
const { json, status } = internalErrorToHttpError(error as Error);
|
|
@@ -730,6 +748,45 @@ app.get(`${API_PREFIX}/status`, async (_req, res) => {
|
|
|
730
748
|
}
|
|
731
749
|
});
|
|
732
750
|
|
|
751
|
+
app.get(`${API_PREFIX}/theme`, async (_req, res) => {
|
|
752
|
+
try {
|
|
753
|
+
if (!environmentStore.storageManager.isInitialized()) {
|
|
754
|
+
// Storage is still initializing. Answer 503 (not 200 with an empty
|
|
755
|
+
// theme) so the Theme Editor's load stays in an error state and
|
|
756
|
+
// never adopts {} as the authoritative saved baseline, which would
|
|
757
|
+
// let a subsequent edit auto-save {} over the real persisted theme.
|
|
758
|
+
throw new ServiceUnavailableError(
|
|
759
|
+
"Theme storage is still initializing. Retry shortly.",
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
res.status(200).json(await themeController.getTheme());
|
|
763
|
+
} catch (error) {
|
|
764
|
+
logger.error("Error getting theme", { error });
|
|
765
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
766
|
+
res.status(status).json(json);
|
|
767
|
+
}
|
|
768
|
+
});
|
|
769
|
+
|
|
770
|
+
app.put(`${API_PREFIX}/theme`, async (req, res) => {
|
|
771
|
+
try {
|
|
772
|
+
res.status(200).json(await themeController.putTheme(req.body));
|
|
773
|
+
} catch (error) {
|
|
774
|
+
logger.error("Error saving theme", { error });
|
|
775
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
776
|
+
res.status(status).json(json);
|
|
777
|
+
}
|
|
778
|
+
});
|
|
779
|
+
|
|
780
|
+
app.delete(`${API_PREFIX}/theme`, async (_req, res) => {
|
|
781
|
+
try {
|
|
782
|
+
res.status(200).json(await themeController.resetTheme());
|
|
783
|
+
} catch (error) {
|
|
784
|
+
logger.error("Error resetting theme", { error });
|
|
785
|
+
const { json, status } = internalErrorToHttpError(error as Error);
|
|
786
|
+
res.status(status).json(json);
|
|
787
|
+
}
|
|
788
|
+
});
|
|
789
|
+
|
|
733
790
|
app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);
|
|
734
791
|
app.post(`${API_PREFIX}/watch-mode/start`, watchModeController.startWatching);
|
|
735
792
|
app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
|
|
@@ -4,7 +4,7 @@ import * as sinon from "sinon";
|
|
|
4
4
|
import type { BuildGraph as MalloyBuildGraph } from "@malloydata/malloy";
|
|
5
5
|
import {
|
|
6
6
|
compilePackageBuildPlan,
|
|
7
|
-
|
|
7
|
+
computeSourceEntityId,
|
|
8
8
|
computePackageBuildPlan,
|
|
9
9
|
deriveAnnotationFields,
|
|
10
10
|
deriveBuildPlan,
|
|
@@ -27,8 +27,8 @@ describe("flattenDependsOn", () => {
|
|
|
27
27
|
|
|
28
28
|
describe("iterGraphSources", () => {
|
|
29
29
|
it("yields resolvable sources in dependency order, skipping missing ones", () => {
|
|
30
|
-
const a = fakeSource({ name: "a",
|
|
31
|
-
const b = fakeSource({ name: "b",
|
|
30
|
+
const a = fakeSource({ name: "a", sourceEntityId: "ba" });
|
|
31
|
+
const b = fakeSource({ name: "b", sourceEntityId: "bb" });
|
|
32
32
|
const graph = {
|
|
33
33
|
connectionName: "duckdb",
|
|
34
34
|
nodes: [
|
|
@@ -52,9 +52,9 @@ describe("iterGraphSources", () => {
|
|
|
52
52
|
// and every transitive persist dependency is nested in dependsOn. All
|
|
53
53
|
// three must be yielded (so all get built), leaf-first so a downstream
|
|
54
54
|
// build reads its upstream's freshly materialized table.
|
|
55
|
-
const root = fakeSource({ name: "root",
|
|
56
|
-
const mid = fakeSource({ name: "mid",
|
|
57
|
-
const leaf = fakeSource({ name: "leaf",
|
|
55
|
+
const root = fakeSource({ name: "root", sourceEntityId: "br" });
|
|
56
|
+
const mid = fakeSource({ name: "mid", sourceEntityId: "bm" });
|
|
57
|
+
const leaf = fakeSource({ name: "leaf", sourceEntityId: "bl" });
|
|
58
58
|
const graph = {
|
|
59
59
|
connectionName: "duckdb",
|
|
60
60
|
nodes: [
|
|
@@ -85,9 +85,9 @@ describe("iterGraphSources", () => {
|
|
|
85
85
|
it("deduplicates a shared (diamond) dependency across roots", () => {
|
|
86
86
|
// r1 and r2 both depend on `shared`; it must be yielded exactly once and
|
|
87
87
|
// before both dependents.
|
|
88
|
-
const r1 = fakeSource({ name: "r1",
|
|
89
|
-
const r2 = fakeSource({ name: "r2",
|
|
90
|
-
const shared = fakeSource({ name: "shared",
|
|
88
|
+
const r1 = fakeSource({ name: "r1", sourceEntityId: "b1" });
|
|
89
|
+
const r2 = fakeSource({ name: "r2", sourceEntityId: "b2" });
|
|
90
|
+
const shared = fakeSource({ name: "shared", sourceEntityId: "bs" });
|
|
91
91
|
const graph = {
|
|
92
92
|
connectionName: "duckdb",
|
|
93
93
|
nodes: [
|
|
@@ -149,7 +149,7 @@ describe("deriveAnnotationFields", () => {
|
|
|
149
149
|
});
|
|
150
150
|
});
|
|
151
151
|
|
|
152
|
-
describe("
|
|
152
|
+
describe("computeSourceEntityId", () => {
|
|
153
153
|
it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
|
|
154
154
|
const makeBuildId = sinon.stub().returns("computed-id");
|
|
155
155
|
const source = {
|
|
@@ -158,7 +158,7 @@ describe("computeBuildId", () => {
|
|
|
158
158
|
getSQL: () => "SELECT 7",
|
|
159
159
|
} as unknown as PersistSource;
|
|
160
160
|
|
|
161
|
-
const id =
|
|
161
|
+
const id = computeSourceEntityId(source, { duckdb: "dig-1" });
|
|
162
162
|
|
|
163
163
|
expect(id).toBe("computed-id");
|
|
164
164
|
expect(makeBuildId.calledOnceWithExactly("dig-1", "SELECT 7")).toBe(true);
|
|
@@ -188,7 +188,7 @@ describe("deriveBuildPlan", () => {
|
|
|
188
188
|
it("projects graphs and sources into the wire build plan", () => {
|
|
189
189
|
const orders = fakeSource({
|
|
190
190
|
name: "orders",
|
|
191
|
-
|
|
191
|
+
sourceEntityId: "bid-orders",
|
|
192
192
|
sql: "SELECT 1",
|
|
193
193
|
});
|
|
194
194
|
const plan = deriveBuildPlan(
|
|
@@ -206,15 +206,59 @@ describe("deriveBuildPlan", () => {
|
|
|
206
206
|
expect(plan.sources["orders@m"]).toMatchObject({
|
|
207
207
|
name: "orders",
|
|
208
208
|
connectionName: "duckdb",
|
|
209
|
-
|
|
209
|
+
sourceEntityId: "bid-orders",
|
|
210
210
|
sql: "SELECT 1",
|
|
211
211
|
columns: [],
|
|
212
212
|
});
|
|
213
213
|
});
|
|
214
214
|
|
|
215
|
+
it("reports declared sharing/refresh verbatim and null when unset", () => {
|
|
216
|
+
// The control plane distinguishes unset from an explicit `shared` (it
|
|
217
|
+
// applies the platform default itself), so the publisher must report
|
|
218
|
+
// the declared value verbatim — never substitute the default.
|
|
219
|
+
const declared = fakeSource({
|
|
220
|
+
name: "declared",
|
|
221
|
+
sourceEntityId: "bid-d",
|
|
222
|
+
annotationFields: {
|
|
223
|
+
name: "d_table",
|
|
224
|
+
sharing: "private",
|
|
225
|
+
refresh: "incremental",
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
const unset = fakeSource({ name: "unset", sourceEntityId: "bid-u" });
|
|
229
|
+
const plan = deriveBuildPlan(
|
|
230
|
+
[
|
|
231
|
+
{
|
|
232
|
+
connectionName: "duckdb",
|
|
233
|
+
nodes: [
|
|
234
|
+
[
|
|
235
|
+
{ sourceID: "declared@m", dependsOn: [] },
|
|
236
|
+
{ sourceID: "unset@m", dependsOn: [] },
|
|
237
|
+
],
|
|
238
|
+
],
|
|
239
|
+
},
|
|
240
|
+
] as unknown as Parameters<typeof deriveBuildPlan>[0],
|
|
241
|
+
{ "declared@m": declared, "unset@m": unset },
|
|
242
|
+
{ duckdb: "dig" },
|
|
243
|
+
);
|
|
244
|
+
|
|
245
|
+
expect(plan.sources["declared@m"].sharing).toBe("private");
|
|
246
|
+
expect(plan.sources["declared@m"].refresh).toBe("incremental");
|
|
247
|
+
// The raw annotation map still carries every field alongside the typed
|
|
248
|
+
// projections.
|
|
249
|
+
expect(plan.sources["declared@m"].annotationFields).toEqual({
|
|
250
|
+
name: "d_table",
|
|
251
|
+
sharing: "private",
|
|
252
|
+
refresh: "incremental",
|
|
253
|
+
});
|
|
254
|
+
// Unset is null — not "shared" — on the wire.
|
|
255
|
+
expect(plan.sources["unset@m"].sharing).toBeNull();
|
|
256
|
+
expect(plan.sources["unset@m"].refresh).toBeNull();
|
|
257
|
+
});
|
|
258
|
+
|
|
215
259
|
it("honors the sourceNames filter", () => {
|
|
216
|
-
const a = fakeSource({ name: "a",
|
|
217
|
-
const b = fakeSource({ name: "b",
|
|
260
|
+
const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
|
|
261
|
+
const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
|
|
218
262
|
const plan = deriveBuildPlan(
|
|
219
263
|
[
|
|
220
264
|
{
|
|
@@ -231,8 +275,8 @@ describe("deriveBuildPlan", () => {
|
|
|
231
275
|
});
|
|
232
276
|
|
|
233
277
|
it("carries the per-source package-relative modelPath", () => {
|
|
234
|
-
const a = fakeSource({ name: "a",
|
|
235
|
-
const b = fakeSource({ name: "b",
|
|
278
|
+
const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
|
|
279
|
+
const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
|
|
236
280
|
const plan = deriveBuildPlan(
|
|
237
281
|
[
|
|
238
282
|
{
|
|
@@ -112,8 +112,8 @@ export function flattenDependsOn(node: {
|
|
|
112
112
|
* persist dependency is nested under a root in its recursive {@code dependsOn}
|
|
113
113
|
* tree (and present in {@code sources}). Walking only {@code graph.nodes}
|
|
114
114
|
* silently skips every intermediate persist source, so it never gets
|
|
115
|
-
* materialized (it only gets a table by coincidence when it shares a
|
|
116
|
-
* with a root). We post-order DFS the {@code dependsOn} tree
|
|
115
|
+
* materialized (it only gets a table by coincidence when it shares a
|
|
116
|
+
* sourceEntityId with a root). We post-order DFS the {@code dependsOn} tree
|
|
117
117
|
* so dependencies are built first (a downstream build can then read its upstream
|
|
118
118
|
* source's freshly materialized table), deduplicating shared (diamond)
|
|
119
119
|
* dependencies by sourceID so each is yielded once. This mirrors the canonical
|
|
@@ -147,11 +147,23 @@ export function* iterGraphSources(
|
|
|
147
147
|
}
|
|
148
148
|
|
|
149
149
|
/**
|
|
150
|
-
* The
|
|
151
|
-
* and canonical SQL. Centralizes the
|
|
152
|
-
* planning, self-instruction,
|
|
150
|
+
* The sourceEntityId for a persist source: a stable content address of its
|
|
151
|
+
* connection identity and canonical SQL. Centralizes the
|
|
152
|
+
* (source, connectionDigests) call shape so planning, self-instruction, build,
|
|
153
|
+
* and serve-time manifest resolution all agree on the same id.
|
|
154
|
+
*
|
|
155
|
+
* <p>This is the single seam between the publisher and the Malloy compiler's
|
|
156
|
+
* source-identity recipe. Today it delegates to
|
|
157
|
+
* {@code PersistSource.makeBuildId(connectionDigest, sql)} — the compiler's
|
|
158
|
+
* current hex content hash. The connection's contribution is already
|
|
159
|
+
* fingerprint-aware: when a connection carries an API `fingerprint`, its
|
|
160
|
+
* digest IS that fingerprint verbatim (see applyConnectionFingerprint in
|
|
161
|
+
* connection.ts), so ids stay stable across credential rotation. When the
|
|
162
|
+
* compiler ships the scoped UUID5 sourceEntityId recipe (scope + connection
|
|
163
|
+
* fingerprint + canonical SQL), swap the delegation below to the new compiler
|
|
164
|
+
* API — this function is the only place the publisher derives the id.
|
|
153
165
|
*/
|
|
154
|
-
export function
|
|
166
|
+
export function computeSourceEntityId(
|
|
155
167
|
source: PersistSource,
|
|
156
168
|
connectionDigests: Record<string, string>,
|
|
157
169
|
): string {
|
|
@@ -246,9 +258,9 @@ export async function compilePackageBuildPlan(
|
|
|
246
258
|
const conn = connections.get(graph.connectionName);
|
|
247
259
|
if (!conn) {
|
|
248
260
|
// The connection failed to resolve (already warned in
|
|
249
|
-
// resolvePackageConnections). Its
|
|
250
|
-
// digest, so surface it as a discrete correctness signal
|
|
251
|
-
// skipping silently.
|
|
261
|
+
// resolvePackageConnections). Its sourceEntityIds will be computed
|
|
262
|
+
// without a digest, so surface it as a discrete correctness signal
|
|
263
|
+
// rather than skipping silently.
|
|
252
264
|
recordConnectionDigestSkipped();
|
|
253
265
|
logger.warn("Skipping connection digest; connection did not resolve", {
|
|
254
266
|
connectionName: graph.connectionName,
|
|
@@ -256,6 +268,9 @@ export async function compilePackageBuildPlan(
|
|
|
256
268
|
continue;
|
|
257
269
|
}
|
|
258
270
|
if (!connectionDigests[graph.connectionName]) {
|
|
271
|
+
// getDigest() is fingerprint-aware: a connection configured with an
|
|
272
|
+
// API `fingerprint` returns it verbatim (applyConnectionFingerprint),
|
|
273
|
+
// so a credential rotation does not re-address its sources.
|
|
259
274
|
connectionDigests[graph.connectionName] = await conn.getDigest();
|
|
260
275
|
}
|
|
261
276
|
}
|
|
@@ -292,15 +307,24 @@ export function deriveBuildPlan(
|
|
|
292
307
|
const wireSources: Record<string, WirePersistSourcePlan> = {};
|
|
293
308
|
for (const [sourceID, source] of Object.entries(sources)) {
|
|
294
309
|
if (include && !include.has(source.name)) continue;
|
|
310
|
+
const annotationFields = deriveAnnotationFields(source);
|
|
295
311
|
wireSources[sourceID] = {
|
|
296
312
|
name: source.name,
|
|
297
313
|
sourceID: source.sourceID,
|
|
298
314
|
connectionName: source.connectionName,
|
|
299
315
|
dialect: source.dialectName,
|
|
300
|
-
|
|
316
|
+
sourceEntityId: computeSourceEntityId(source, connectionDigests),
|
|
301
317
|
sql: source.getSQL(),
|
|
318
|
+
// Declared `#@ persist` scope/refresh knobs, reported VERBATIM (null
|
|
319
|
+
// = unset). The control plane applies the platform default (unset =>
|
|
320
|
+
// shared) itself, so the publisher must never substitute it — unset
|
|
321
|
+
// has to stay distinguishable from an explicit `shared`. The source's
|
|
322
|
+
// own annotation is the most-specific declaration layer, and the only
|
|
323
|
+
// one that exists today.
|
|
324
|
+
sharing: annotationFields.sharing ?? null,
|
|
325
|
+
refresh: annotationFields.refresh ?? null,
|
|
302
326
|
columns: deriveColumns(source),
|
|
303
|
-
annotationFields
|
|
327
|
+
annotationFields,
|
|
304
328
|
modelPath: sourceModelPaths?.[sourceID],
|
|
305
329
|
};
|
|
306
330
|
}
|
|
@@ -1028,6 +1028,38 @@ function getMetadataForLookup(
|
|
|
1028
1028
|
return name ? metadata.get(name) : undefined;
|
|
1029
1029
|
}
|
|
1030
1030
|
|
|
1031
|
+
/**
|
|
1032
|
+
* Make a connection's digest the caller-supplied API `fingerprint`, verbatim.
|
|
1033
|
+
*
|
|
1034
|
+
* The fingerprint is the connection's data identity as declared by the caller
|
|
1035
|
+
* (secret-free, stable across credential rotation). When set, it replaces the
|
|
1036
|
+
* locally derived `getDigest()` so it becomes the connection's contribution to
|
|
1037
|
+
* content-addressed source ids everywhere a digest is read:
|
|
1038
|
+
*
|
|
1039
|
+
* - build planning (`compilePackageBuildPlan` -> `conn.getDigest()`),
|
|
1040
|
+
* - the package-load worker's connection-metadata RPC, and
|
|
1041
|
+
* - Malloy's own serve-time manifest resolution (the runtime calls
|
|
1042
|
+
* `conn.getDigest()` internally when a build manifest is bound).
|
|
1043
|
+
*
|
|
1044
|
+
* Overriding the method on the resolved instance — inside the one
|
|
1045
|
+
* `lookupConnection` wrapper every resolution passes through — is what keeps
|
|
1046
|
+
* build-time and serve-time ids identical; applying it per call site would
|
|
1047
|
+
* invite divergence and a silent live-serving fallback. The value is used
|
|
1048
|
+
* verbatim (never re-hashed or combined with local details) and treated as an
|
|
1049
|
+
* opaque token: it is not a secret, but it is never logged or parsed.
|
|
1050
|
+
* Without a fingerprint the connection keeps its locally derived digest.
|
|
1051
|
+
*/
|
|
1052
|
+
function applyConnectionFingerprint(
|
|
1053
|
+
connection: Connection,
|
|
1054
|
+
metadata: EnvironmentConnectionMetadata | undefined,
|
|
1055
|
+
): Connection {
|
|
1056
|
+
const fingerprint = metadata?.apiConnection.fingerprint;
|
|
1057
|
+
if (fingerprint) {
|
|
1058
|
+
connection.getDigest = () => fingerprint;
|
|
1059
|
+
}
|
|
1060
|
+
return connection;
|
|
1061
|
+
}
|
|
1062
|
+
|
|
1031
1063
|
function isDuckDBConnection(
|
|
1032
1064
|
connection: Connection,
|
|
1033
1065
|
): connection is DuckDBConnection {
|
|
@@ -1095,10 +1127,11 @@ export function buildEnvironmentMalloyConfig(
|
|
|
1095
1127
|
}
|
|
1096
1128
|
|
|
1097
1129
|
malloyConfig.wrapConnections(
|
|
1098
|
-
(base: LookupConnection<Connection>): LookupConnection<Connection> =>
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1130
|
+
(base: LookupConnection<Connection>): LookupConnection<Connection> => {
|
|
1131
|
+
const resolveConnection = async (
|
|
1132
|
+
name: string | undefined,
|
|
1133
|
+
metadata: EnvironmentConnectionMetadata | undefined,
|
|
1134
|
+
): Promise<Connection> => {
|
|
1102
1135
|
if (metadata?.isDuckLake) {
|
|
1103
1136
|
let connectionPromise = duckLakeCache.get(name!);
|
|
1104
1137
|
if (!connectionPromise) {
|
|
@@ -1210,8 +1243,21 @@ export function buildEnvironmentMalloyConfig(
|
|
|
1210
1243
|
await attachOnce(connection, metadata);
|
|
1211
1244
|
}
|
|
1212
1245
|
return connection;
|
|
1213
|
-
}
|
|
1214
|
-
|
|
1246
|
+
};
|
|
1247
|
+
|
|
1248
|
+
return {
|
|
1249
|
+
lookupConnection: async (name?: string): Promise<Connection> => {
|
|
1250
|
+
const metadata = getMetadataForLookup(assembled.metadata, name);
|
|
1251
|
+
// Every resolution branch funnels through this one exit so a
|
|
1252
|
+
// configured fingerprint is applied no matter which wrapper
|
|
1253
|
+
// produced the connection (see applyConnectionFingerprint).
|
|
1254
|
+
return applyConnectionFingerprint(
|
|
1255
|
+
await resolveConnection(name, metadata),
|
|
1256
|
+
metadata,
|
|
1257
|
+
);
|
|
1258
|
+
},
|
|
1259
|
+
};
|
|
1260
|
+
},
|
|
1215
1261
|
);
|
|
1216
1262
|
|
|
1217
1263
|
return {
|