@malloy-publisher/server 0.0.219 → 0.0.221

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/app/api-doc.yaml +291 -20
  2. package/dist/app/assets/{EnvironmentPage-gehnjfC6.js → EnvironmentPage-D6G5n6mY.js} +1 -1
  3. package/dist/app/assets/{HomePage-8LQBytE4.js → HomePage-DgZluD8u.js} +1 -1
  4. package/dist/app/assets/LightMode-w9v0pEbg.js +1 -0
  5. package/dist/app/assets/MainPage-Bi_Tukvr.js +2 -0
  6. package/dist/app/assets/{MaterializationsPage-D7P1Kp6O.js → MaterializationsPage-BVERjnmG.js} +1 -1
  7. package/dist/app/assets/ModelPage-2OjS259-.js +1 -0
  8. package/dist/app/assets/{PackagePage-Cz9fVwSG.js → PackagePage-C8L2On4z.js} +1 -1
  9. package/dist/app/assets/{RouteError-UONCloyN.js → RouteError-DJoNC_Vl.js} +1 -1
  10. package/dist/app/assets/ThemeEditorPage-B-hJ1zgd.js +1 -0
  11. package/dist/app/assets/{WorkbookPage-Bhzqvbq_.js → WorkbookPage-CTbDgGeJ.js} +1 -1
  12. package/dist/app/assets/{core-BiGj7BML.es-kMHAa8tP.js → core-D9Hl0IDY.es-CmgzHn4c.js} +1 -1
  13. package/dist/app/assets/github-dark-DenFmJkN.es-DHJKELXO.js +1 -0
  14. package/dist/app/assets/index-8E2uLeV9.js +2541 -0
  15. package/dist/app/assets/index-BEjJCJjX.js +1761 -0
  16. package/dist/app/assets/index-BF8PkFm8.js +1 -0
  17. package/dist/app/assets/index-Cr-asgoV.js +527 -0
  18. package/dist/app/assets/index-DcCw_qxr.js +18 -0
  19. package/dist/app/assets/index-w_0OQJgZ.js +23 -0
  20. package/dist/app/index.html +1 -1
  21. package/dist/package_load_worker.mjs +20 -2
  22. package/dist/runtime/publisher.js +6 -1
  23. package/dist/server.mjs +412 -55
  24. package/dist/sshcrypto-8m50vnmb.node +0 -0
  25. package/package.json +12 -12
  26. package/src/config.theme.spec.ts +178 -0
  27. package/src/config.ts +179 -0
  28. package/src/controller/materialization.controller.spec.ts +4 -4
  29. package/src/controller/materialization.controller.ts +2 -2
  30. package/src/controller/package.controller.spec.ts +89 -1
  31. package/src/controller/package.controller.ts +28 -3
  32. package/src/controller/theme.controller.ts +83 -0
  33. package/src/dto/connection.dto.ts +6 -0
  34. package/src/materialization_metrics.ts +1 -1
  35. package/src/package_load/package_load_pool.ts +2 -1
  36. package/src/package_load/package_load_worker.ts +5 -2
  37. package/src/package_load/protocol.ts +2 -1
  38. package/src/runtime/publisher.js +6 -1
  39. package/src/server.ts +58 -1
  40. package/src/service/build_plan.spec.ts +61 -17
  41. package/src/service/build_plan.ts +35 -11
  42. package/src/service/connection.ts +52 -6
  43. package/src/service/connection_fingerprint.spec.ts +102 -0
  44. package/src/service/manifest_loader.spec.ts +5 -5
  45. package/src/service/manifest_loader.ts +4 -4
  46. package/src/service/materialization_cron_gate.spec.ts +137 -0
  47. package/src/service/materialization_schedule_surface.spec.ts +30 -1
  48. package/src/service/materialization_service.spec.ts +47 -37
  49. package/src/service/materialization_service.ts +59 -50
  50. package/src/service/materialization_test_fixtures.ts +22 -6
  51. package/src/service/model.ts +1 -1
  52. package/src/service/package.ts +50 -4
  53. package/src/service/package_manifest.spec.ts +42 -4
  54. package/src/service/package_manifest.ts +53 -4
  55. package/src/service/theme_store.ts +199 -0
  56. package/src/storage/DatabaseInterface.ts +1 -1
  57. package/src/storage/StorageManager.ts +17 -0
  58. package/src/storage/duckdb/schema.ts +27 -6
  59. package/src/theme_key_parity.spec.ts +57 -0
  60. package/tests/integration/materialization/manifest_binding.integration.spec.ts +1 -1
  61. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +5 -5
  62. package/tests/unit/controller/theme.controller.spec.ts +141 -0
  63. package/tests/unit/service/theme_store.spec.ts +139 -0
  64. package/dist/app/assets/MainPage-DDaZLJw-.js +0 -2
  65. package/dist/app/assets/ModelPage-Do_vhxOc.js +0 -1
  66. package/dist/app/assets/index-VzRbxcF7.js +0 -455
  67. package/dist/app/assets/index-ddq4-5hu.js +0 -1760
  68. package/dist/app/assets/index-qOQF9CXq.js +0 -40
  69. package/dist/app/assets/index.umd-B2kmxDh7.js +0 -2467
@@ -5,6 +5,30 @@ import { EnvironmentStore } from "../service/environment_store";
5
5
 
6
6
  type ApiPackage = components["schemas"]["Package"];
7
7
 
8
+ /**
9
+ * Everything that is strict-at-publish, joined into one 400 message (or
10
+ * undefined when the package is publishable): invalid explores entries plus
11
+ * the materialization cron gate (a package-level `schedule` requires every
12
+ * persist source to resolve to explicit `sharing=private` — see
13
+ * Package.scheduleWarnings). At startup/reload both are warn-only instead
14
+ * (fail-safe; see Package.loadViaWorker).
15
+ */
16
+ function formatPublishRejections(
17
+ pkg: {
18
+ formatInvalidExplores(exploresOverride?: string[]): string;
19
+ formatInvalidSchedule(): string;
20
+ },
21
+ exploresOverride?: string[],
22
+ ): string | undefined {
23
+ const message = [
24
+ pkg.formatInvalidExplores(exploresOverride),
25
+ pkg.formatInvalidSchedule(),
26
+ ]
27
+ .filter(Boolean)
28
+ .join("\n");
29
+ return message || undefined;
30
+ }
31
+
8
32
  export class PackageController {
9
33
  private environmentStore: EnvironmentStore;
10
34
 
@@ -100,7 +124,7 @@ export class PackageController {
100
124
  bodyLocation,
101
125
  stagingPath,
102
126
  ),
103
- (pkg) => pkg.formatInvalidExplores(),
127
+ (pkg) => formatPublishRejections(pkg),
104
128
  );
105
129
  } else {
106
130
  result = await environment.addPackage(packageName);
@@ -114,7 +138,7 @@ export class PackageController {
114
138
  }
115
139
 
116
140
  if (!body.location) {
117
- const invalidMsg = result.formatInvalidExplores();
141
+ const invalidMsg = formatPublishRejections(result);
118
142
  if (invalidMsg) {
119
143
  await environment.unloadPackage(packageName).catch(() => {
120
144
  /* best-effort; the package is not persisted below */
@@ -177,7 +201,8 @@ export class PackageController {
177
201
  stagingPath,
178
202
  ),
179
203
  (pkg) =>
180
- pkg.formatInvalidExplores(
204
+ formatPublishRejections(
205
+ pkg,
181
206
  body.explores?.map(normalizeModelPath),
182
207
  ),
183
208
  );
@@ -0,0 +1,83 @@
1
+ import { components } from "../api";
2
+ import { sanitizeTheme, Theme } from "../config";
3
+ import { BadRequestError, FrozenConfigError } from "../errors";
4
+ import { isPublisherConfigFrozen } from "../config";
5
+ import { ThemeStore } from "../service/theme_store";
6
+
7
+ type ApiTheme = components["schemas"]["Theme"];
8
+
9
+ /**
10
+ * Handlers for the in-app Theme Editor.
11
+ *
12
+ * GET /api/v0/theme → current saved theme (or built-in default)
13
+ * PUT /api/v0/theme → save the supplied theme
14
+ * DELETE /api/v0/theme → reset to boot seed / built-in default
15
+ */
16
+ export class ThemeController {
17
+ constructor(
18
+ private themeStore: ThemeStore,
19
+ private serverRoot: string,
20
+ ) {}
21
+
22
+ getTheme = async (): Promise<ApiTheme> => {
23
+ const theme = await this.themeStore.get();
24
+ // Always return an object so the client never has to deal with the
25
+ // null/undefined ambiguity. An empty object means "no overrides yet,
26
+ // use SDK defaults."
27
+ return theme ?? {};
28
+ };
29
+
30
+ /**
31
+ * PUT persists the supplied theme verbatim, including a literal `{}`.
32
+ * An empty `{}` is an intentional "clear all overrides" save: it writes
33
+ * an (empty) row so the boot seed from publisher.config.json's `theme`
34
+ * block is suppressed on subsequent loads. This deliberately differs
35
+ * from DELETE (resetTheme), which drops the row and lets that config
36
+ * seed re-apply. Note the persisted `"{}"` sanitizes back to `undefined`
37
+ * on reload (sanitizeTheme returns undefined for `{}`), but the row
38
+ * still exists, so the seed stays suppressed. This PUT-`{}` path is not
39
+ * reachable from the Theme Editor UI today (its clear affordance is the
40
+ * Reset button, which calls DELETE); it exists only via the API.
41
+ */
42
+ putTheme = async (body: unknown): Promise<ApiTheme> => {
43
+ if (isPublisherConfigFrozen(this.serverRoot)) {
44
+ throw new FrozenConfigError(
45
+ 'Cannot edit theme: publisher.config.json has "frozenConfig": true.',
46
+ );
47
+ }
48
+ // Treat a literal `{}` as an intentional "clear all overrides" PUT
49
+ // (the only other clear path is DELETE, which re-seeds from the
50
+ // config file). sanitizeTheme returns undefined for both `{}` and
51
+ // garbage, so we have to disambiguate here.
52
+ const isEmptyObject =
53
+ typeof body === "object" &&
54
+ body !== null &&
55
+ !Array.isArray(body) &&
56
+ Object.keys(body as Record<string, unknown>).length === 0;
57
+ const sanitized = sanitizeTheme(body, "PUT /api/v0/theme");
58
+ if (!sanitized && !isEmptyObject) {
59
+ throw new BadRequestError(
60
+ "Theme payload was malformed (expected an object).",
61
+ );
62
+ }
63
+ const saved: Theme = await this.themeStore.set(sanitized ?? {});
64
+ return saved;
65
+ };
66
+
67
+ /**
68
+ * DELETE drops the persisted row so the next load re-seeds from
69
+ * publisher.config.json's `theme` block (or ends up unthemed if the
70
+ * config defines none). Contrast with PUT `{}`, which persists an empty
71
+ * row and thereby suppresses that seed instead of restoring it. See
72
+ * putTheme for the full divergence.
73
+ */
74
+ resetTheme = async (): Promise<ApiTheme> => {
75
+ if (isPublisherConfigFrozen(this.serverRoot)) {
76
+ throw new FrozenConfigError(
77
+ 'Cannot reset theme: publisher.config.json has "frozenConfig": true.',
78
+ );
79
+ }
80
+ const reseeded = await this.themeStore.reset();
81
+ return reseeded ?? {};
82
+ };
83
+ }
@@ -265,6 +265,12 @@ export class ConnectionDto implements ApiConnection {
265
265
  | "motherduck"
266
266
  | "ducklake";
267
267
 
268
+ // Opaque, secret-free data-identity token; carried verbatim (never parsed
269
+ // or re-hashed) into content-addressed source ids. See api-doc.yaml.
270
+ @IsOptional()
271
+ @IsString()
272
+ fingerprint?: string;
273
+
268
274
  @IsOptional()
269
275
  @ValidateNested()
270
276
  @Type(() => PostgresConnectionDto)
@@ -131,7 +131,7 @@ export function recordAutoLoadOutcome(outcome: "success" | "failure"): void {
131
131
 
132
132
  /**
133
133
  * Record that a connection digest could not be computed during build-plan
134
- * compilation because the connection failed to resolve. The resulting buildIds
134
+ * compilation because the connection failed to resolve. The resulting sourceEntityIds
135
135
  * are computed without a digest, so this is a correctness signal, not just noise.
136
136
  */
137
137
  export function recordConnectionDigestSkipped(): void {
@@ -81,6 +81,7 @@ import { fileURLToPath, pathToFileURL } from "url";
81
81
 
82
82
  import { ModelCompilationError } from "../errors";
83
83
  import { logger } from "../logger";
84
+ import type { PackageMaterializationConfig } from "../service/package_manifest";
84
85
  import type {
85
86
  ConnectionMetadataRequest,
86
87
  ConnectionMetadataResponse,
@@ -233,7 +234,7 @@ export interface LoadPackageOutcome {
233
234
  explores?: string[];
234
235
  queryableSources?: "declared" | "all";
235
236
  manifestLocation?: string | null;
236
- materialization?: { schedule: string | null } | null;
237
+ materialization?: PackageMaterializationConfig | null;
237
238
  };
238
239
  models: Array<
239
240
  Omit<SerializedModel, "modelDef" | "sourceInfos"> & {
@@ -83,7 +83,10 @@ import { HackyDataStylesAccumulator } from "../data_styles";
83
83
  import { ModelCompilationError } from "../errors";
84
84
  import { validateAuthorizeProbes } from "../service/authorize";
85
85
  import { type FilterDefinition } from "../service/filter";
86
- import { parsePackageMaterialization } from "../service/package_manifest";
86
+ import {
87
+ PackageMaterializationConfig,
88
+ parsePackageMaterialization,
89
+ } from "../service/package_manifest";
87
90
  import {
88
91
  extractQueriesFromModelDef,
89
92
  extractSourcesFromModelDef,
@@ -383,7 +386,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
383
386
  explores?: string[];
384
387
  queryableSources?: "declared" | "all";
385
388
  manifestLocation?: string | null;
386
- materialization?: { schedule: string | null } | null;
389
+ materialization?: PackageMaterializationConfig | null;
387
390
  }> {
388
391
  const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
389
392
  const contents = await fs.promises.readFile(manifestPath, "utf8");
@@ -65,6 +65,7 @@
65
65
  */
66
66
 
67
67
  import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
68
+ import type { PackageMaterializationConfig } from "../service/package_manifest";
68
69
 
69
70
  // ──────────────────────────────────────────────────────────────────────
70
71
  // Direction: main ──▶ worker (load-package job)
@@ -179,7 +180,7 @@ export interface LoadPackageResult {
179
180
  explores?: string[];
180
181
  queryableSources?: "declared" | "all";
181
182
  manifestLocation?: string | null;
182
- materialization?: { schedule: string | null } | null;
183
+ materialization?: PackageMaterializationConfig | null;
183
184
  };
184
185
  models: SerializedModel[];
185
186
  /** Wall-clock ms inside the worker for the full package load. */
@@ -79,7 +79,12 @@
79
79
  "/packages/" +
80
80
  encodeURIComponent(target.pkg) +
81
81
  "/models/" +
82
- modelPath.split("/").map(encodeURIComponent).join("/") +
82
+ // Encode the whole model path as ONE path segment (slashes -> %2F). A model
83
+ // in a subfolder ("model/x.malloy") must not become two URL segments: some
84
+ // servers only match a single-segment {path} and would 405 a multi-segment
85
+ // path. This matches the typed SDK (encodeURIComponent) and the server's
86
+ // encoded-slash handling.
87
+ encodeURIComponent(modelPath) +
83
88
  "/query";
84
89
  var body = { compactJson: compactJson };
85
90
  if (malloyQuery) body.query = malloyQuery;
package/src/server.ts CHANGED
@@ -20,6 +20,7 @@ import {
20
20
  BadRequestError,
21
21
  internalErrorToHttpError,
22
22
  NotImplementedError,
23
+ ServiceUnavailableError,
23
24
  } from "./errors";
24
25
  import {
25
26
  drainingGuard,
@@ -38,6 +39,7 @@ import { setFilterDeprecationHeaders } from "./filter_deprecation";
38
39
  import { checkHeapConfiguration } from "./heap_check";
39
40
  import { queryConcurrency } from "./query_concurrency";
40
41
  import { MaterializationController } from "./controller/materialization.controller";
42
+ import { ThemeController } from "./controller/theme.controller";
41
43
  import { initializeMcpServer } from "./mcp/server";
42
44
  import { startAgentMcpServer } from "./mcp/agent_server";
43
45
  import { registerLegacyRoutes } from "./server-old";
@@ -45,6 +47,7 @@ import { EnvironmentStore } from "./service/environment_store";
45
47
  import { MaterializationService } from "./service/materialization_service";
46
48
  import { normalizeQueryArray } from "./query_param_utils";
47
49
  import { PackageMemoryGovernor } from "./service/package_memory_governor";
50
+ import { ThemeStore } from "./service/theme_store";
48
51
  import { assertSafePackageName, safeJoinUnderRoot } from "./path_safety";
49
52
 
50
53
  export { normalizeQueryArray } from "./query_param_utils";
@@ -207,6 +210,8 @@ const materializationService = new MaterializationService(environmentStore);
207
210
  const materializationController = new MaterializationController(
208
211
  materializationService,
209
212
  );
213
+ const themeStore = new ThemeStore(environmentStore.storageManager, SERVER_ROOT);
214
+ const themeController = new ThemeController(themeStore, SERVER_ROOT);
210
215
 
211
216
  export const mcpApp = express();
212
217
 
@@ -722,7 +727,20 @@ app.get(
722
727
  app.get(`${API_PREFIX}/status`, async (_req, res) => {
723
728
  try {
724
729
  const status = await environmentStore.getStatus();
725
- res.status(200).json(status);
730
+ // Compose theme onto the status response so the SDK can read both
731
+ // in one round trip on app boot. ThemeStore is the source of truth;
732
+ // publisher.config.json is only a boot seed (see ThemeStore). The
733
+ // field is always present (an empty object means "no overrides
734
+ // yet"), so the OpenAPI shape and the runtime payload agree.
735
+ // The theme here is cosmetic, so during the brief window before storage
736
+ // initializes report no overrides rather than 500 an endpoint the
737
+ // control plane polls for serving state (themeStore.get() throws until
738
+ // storage is ready). GET /theme, the editor's authoritative load, is
739
+ // answered with 503 during that window instead.
740
+ const theme = environmentStore.storageManager.isInitialized()
741
+ ? await themeStore.get()
742
+ : undefined;
743
+ res.status(200).json({ ...status, theme: theme ?? {} });
726
744
  } catch (error) {
727
745
  logger.error("Error getting status", { error });
728
746
  const { json, status } = internalErrorToHttpError(error as Error);
@@ -730,6 +748,45 @@ app.get(`${API_PREFIX}/status`, async (_req, res) => {
730
748
  }
731
749
  });
732
750
 
751
+ app.get(`${API_PREFIX}/theme`, async (_req, res) => {
752
+ try {
753
+ if (!environmentStore.storageManager.isInitialized()) {
754
+ // Storage is still initializing. Answer 503 (not 200 with an empty
755
+ // theme) so the Theme Editor's load stays in an error state and
756
+ // never adopts {} as the authoritative saved baseline, which would
757
+ // let a subsequent edit auto-save {} over the real persisted theme.
758
+ throw new ServiceUnavailableError(
759
+ "Theme storage is still initializing. Retry shortly.",
760
+ );
761
+ }
762
+ res.status(200).json(await themeController.getTheme());
763
+ } catch (error) {
764
+ logger.error("Error getting theme", { error });
765
+ const { json, status } = internalErrorToHttpError(error as Error);
766
+ res.status(status).json(json);
767
+ }
768
+ });
769
+
770
+ app.put(`${API_PREFIX}/theme`, async (req, res) => {
771
+ try {
772
+ res.status(200).json(await themeController.putTheme(req.body));
773
+ } catch (error) {
774
+ logger.error("Error saving theme", { error });
775
+ const { json, status } = internalErrorToHttpError(error as Error);
776
+ res.status(status).json(json);
777
+ }
778
+ });
779
+
780
+ app.delete(`${API_PREFIX}/theme`, async (_req, res) => {
781
+ try {
782
+ res.status(200).json(await themeController.resetTheme());
783
+ } catch (error) {
784
+ logger.error("Error resetting theme", { error });
785
+ const { json, status } = internalErrorToHttpError(error as Error);
786
+ res.status(status).json(json);
787
+ }
788
+ });
789
+
733
790
  app.get(`${API_PREFIX}/watch-mode/status`, watchModeController.getWatchStatus);
734
791
  app.post(`${API_PREFIX}/watch-mode/start`, watchModeController.startWatching);
735
792
  app.post(`${API_PREFIX}/watch-mode/stop`, watchModeController.stopWatchMode);
@@ -4,7 +4,7 @@ import * as sinon from "sinon";
4
4
  import type { BuildGraph as MalloyBuildGraph } from "@malloydata/malloy";
5
5
  import {
6
6
  compilePackageBuildPlan,
7
- computeBuildId,
7
+ computeSourceEntityId,
8
8
  computePackageBuildPlan,
9
9
  deriveAnnotationFields,
10
10
  deriveBuildPlan,
@@ -27,8 +27,8 @@ describe("flattenDependsOn", () => {
27
27
 
28
28
  describe("iterGraphSources", () => {
29
29
  it("yields resolvable sources in dependency order, skipping missing ones", () => {
30
- const a = fakeSource({ name: "a", buildId: "ba" });
31
- const b = fakeSource({ name: "b", buildId: "bb" });
30
+ const a = fakeSource({ name: "a", sourceEntityId: "ba" });
31
+ const b = fakeSource({ name: "b", sourceEntityId: "bb" });
32
32
  const graph = {
33
33
  connectionName: "duckdb",
34
34
  nodes: [
@@ -52,9 +52,9 @@ describe("iterGraphSources", () => {
52
52
  // and every transitive persist dependency is nested in dependsOn. All
53
53
  // three must be yielded (so all get built), leaf-first so a downstream
54
54
  // build reads its upstream's freshly materialized table.
55
- const root = fakeSource({ name: "root", buildId: "br" });
56
- const mid = fakeSource({ name: "mid", buildId: "bm" });
57
- const leaf = fakeSource({ name: "leaf", buildId: "bl" });
55
+ const root = fakeSource({ name: "root", sourceEntityId: "br" });
56
+ const mid = fakeSource({ name: "mid", sourceEntityId: "bm" });
57
+ const leaf = fakeSource({ name: "leaf", sourceEntityId: "bl" });
58
58
  const graph = {
59
59
  connectionName: "duckdb",
60
60
  nodes: [
@@ -85,9 +85,9 @@ describe("iterGraphSources", () => {
85
85
  it("deduplicates a shared (diamond) dependency across roots", () => {
86
86
  // r1 and r2 both depend on `shared`; it must be yielded exactly once and
87
87
  // before both dependents.
88
- const r1 = fakeSource({ name: "r1", buildId: "b1" });
89
- const r2 = fakeSource({ name: "r2", buildId: "b2" });
90
- const shared = fakeSource({ name: "shared", buildId: "bs" });
88
+ const r1 = fakeSource({ name: "r1", sourceEntityId: "b1" });
89
+ const r2 = fakeSource({ name: "r2", sourceEntityId: "b2" });
90
+ const shared = fakeSource({ name: "shared", sourceEntityId: "bs" });
91
91
  const graph = {
92
92
  connectionName: "duckdb",
93
93
  nodes: [
@@ -149,7 +149,7 @@ describe("deriveAnnotationFields", () => {
149
149
  });
150
150
  });
151
151
 
152
- describe("computeBuildId", () => {
152
+ describe("computeSourceEntityId", () => {
153
153
  it("delegates to PersistSource.makeBuildId with the connection digest and SQL", () => {
154
154
  const makeBuildId = sinon.stub().returns("computed-id");
155
155
  const source = {
@@ -158,7 +158,7 @@ describe("computeBuildId", () => {
158
158
  getSQL: () => "SELECT 7",
159
159
  } as unknown as PersistSource;
160
160
 
161
- const id = computeBuildId(source, { duckdb: "dig-1" });
161
+ const id = computeSourceEntityId(source, { duckdb: "dig-1" });
162
162
 
163
163
  expect(id).toBe("computed-id");
164
164
  expect(makeBuildId.calledOnceWithExactly("dig-1", "SELECT 7")).toBe(true);
@@ -188,7 +188,7 @@ describe("deriveBuildPlan", () => {
188
188
  it("projects graphs and sources into the wire build plan", () => {
189
189
  const orders = fakeSource({
190
190
  name: "orders",
191
- buildId: "bid-orders",
191
+ sourceEntityId: "bid-orders",
192
192
  sql: "SELECT 1",
193
193
  });
194
194
  const plan = deriveBuildPlan(
@@ -206,15 +206,59 @@ describe("deriveBuildPlan", () => {
206
206
  expect(plan.sources["orders@m"]).toMatchObject({
207
207
  name: "orders",
208
208
  connectionName: "duckdb",
209
- buildId: "bid-orders",
209
+ sourceEntityId: "bid-orders",
210
210
  sql: "SELECT 1",
211
211
  columns: [],
212
212
  });
213
213
  });
214
214
 
215
+ it("reports declared sharing/refresh verbatim and null when unset", () => {
216
+ // The control plane distinguishes unset from an explicit `shared` (it
217
+ // applies the platform default itself), so the publisher must report
218
+ // the declared value verbatim — never substitute the default.
219
+ const declared = fakeSource({
220
+ name: "declared",
221
+ sourceEntityId: "bid-d",
222
+ annotationFields: {
223
+ name: "d_table",
224
+ sharing: "private",
225
+ refresh: "incremental",
226
+ },
227
+ });
228
+ const unset = fakeSource({ name: "unset", sourceEntityId: "bid-u" });
229
+ const plan = deriveBuildPlan(
230
+ [
231
+ {
232
+ connectionName: "duckdb",
233
+ nodes: [
234
+ [
235
+ { sourceID: "declared@m", dependsOn: [] },
236
+ { sourceID: "unset@m", dependsOn: [] },
237
+ ],
238
+ ],
239
+ },
240
+ ] as unknown as Parameters<typeof deriveBuildPlan>[0],
241
+ { "declared@m": declared, "unset@m": unset },
242
+ { duckdb: "dig" },
243
+ );
244
+
245
+ expect(plan.sources["declared@m"].sharing).toBe("private");
246
+ expect(plan.sources["declared@m"].refresh).toBe("incremental");
247
+ // The raw annotation map still carries every field alongside the typed
248
+ // projections.
249
+ expect(plan.sources["declared@m"].annotationFields).toEqual({
250
+ name: "d_table",
251
+ sharing: "private",
252
+ refresh: "incremental",
253
+ });
254
+ // Unset is null — not "shared" — on the wire.
255
+ expect(plan.sources["unset@m"].sharing).toBeNull();
256
+ expect(plan.sources["unset@m"].refresh).toBeNull();
257
+ });
258
+
215
259
  it("honors the sourceNames filter", () => {
216
- const a = fakeSource({ name: "a", buildId: "bid-a" });
217
- const b = fakeSource({ name: "b", buildId: "bid-b" });
260
+ const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
261
+ const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
218
262
  const plan = deriveBuildPlan(
219
263
  [
220
264
  {
@@ -231,8 +275,8 @@ describe("deriveBuildPlan", () => {
231
275
  });
232
276
 
233
277
  it("carries the per-source package-relative modelPath", () => {
234
- const a = fakeSource({ name: "a", buildId: "bid-a" });
235
- const b = fakeSource({ name: "b", buildId: "bid-b" });
278
+ const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
279
+ const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
236
280
  const plan = deriveBuildPlan(
237
281
  [
238
282
  {
@@ -112,8 +112,8 @@ export function flattenDependsOn(node: {
112
112
  * persist dependency is nested under a root in its recursive {@code dependsOn}
113
113
  * tree (and present in {@code sources}). Walking only {@code graph.nodes}
114
114
  * silently skips every intermediate persist source, so it never gets
115
- * materialized (it only gets a table by coincidence when it shares a buildId
116
- * with a root). We post-order DFS the {@code dependsOn} tree
115
+ * materialized (it only gets a table by coincidence when it shares a
116
+ * sourceEntityId with a root). We post-order DFS the {@code dependsOn} tree
117
117
  * so dependencies are built first (a downstream build can then read its upstream
118
118
  * source's freshly materialized table), deduplicating shared (diamond)
119
119
  * dependencies by sourceID so each is yielded once. This mirrors the canonical
@@ -147,11 +147,23 @@ export function* iterGraphSources(
147
147
  }
148
148
 
149
149
  /**
150
- * The buildId for a persist source: a stable digest of its connection identity
151
- * and canonical SQL. Centralizes the (source, connectionDigests) call shape so
152
- * planning, self-instruction, and build all agree on the same id.
150
+ * The sourceEntityId for a persist source: a stable content address of its
151
+ * connection identity and canonical SQL. Centralizes the
152
+ * (source, connectionDigests) call shape so planning, self-instruction, build,
153
+ * and serve-time manifest resolution all agree on the same id.
154
+ *
155
+ * <p>This is the single seam between the publisher and the Malloy compiler's
156
+ * source-identity recipe. Today it delegates to
157
+ * {@code PersistSource.makeBuildId(connectionDigest, sql)} — the compiler's
158
+ * current hex content hash. The connection's contribution is already
159
+ * fingerprint-aware: when a connection carries an API `fingerprint`, its
160
+ * digest IS that fingerprint verbatim (see applyConnectionFingerprint in
161
+ * connection.ts), so ids stay stable across credential rotation. When the
162
+ * compiler ships the scoped UUID5 sourceEntityId recipe (scope + connection
163
+ * fingerprint + canonical SQL), swap the delegation below to the new compiler
164
+ * API — this function is the only place the publisher derives the id.
153
165
  */
154
- export function computeBuildId(
166
+ export function computeSourceEntityId(
155
167
  source: PersistSource,
156
168
  connectionDigests: Record<string, string>,
157
169
  ): string {
@@ -246,9 +258,9 @@ export async function compilePackageBuildPlan(
246
258
  const conn = connections.get(graph.connectionName);
247
259
  if (!conn) {
248
260
  // The connection failed to resolve (already warned in
249
- // resolvePackageConnections). Its buildIds will be computed without a
250
- // digest, so surface it as a discrete correctness signal rather than
251
- // skipping silently.
261
+ // resolvePackageConnections). Its sourceEntityIds will be computed
262
+ // without a digest, so surface it as a discrete correctness signal
263
+ // rather than skipping silently.
252
264
  recordConnectionDigestSkipped();
253
265
  logger.warn("Skipping connection digest; connection did not resolve", {
254
266
  connectionName: graph.connectionName,
@@ -256,6 +268,9 @@ export async function compilePackageBuildPlan(
256
268
  continue;
257
269
  }
258
270
  if (!connectionDigests[graph.connectionName]) {
271
+ // getDigest() is fingerprint-aware: a connection configured with an
272
+ // API `fingerprint` returns it verbatim (applyConnectionFingerprint),
273
+ // so a credential rotation does not re-address its sources.
259
274
  connectionDigests[graph.connectionName] = await conn.getDigest();
260
275
  }
261
276
  }
@@ -292,15 +307,24 @@ export function deriveBuildPlan(
292
307
  const wireSources: Record<string, WirePersistSourcePlan> = {};
293
308
  for (const [sourceID, source] of Object.entries(sources)) {
294
309
  if (include && !include.has(source.name)) continue;
310
+ const annotationFields = deriveAnnotationFields(source);
295
311
  wireSources[sourceID] = {
296
312
  name: source.name,
297
313
  sourceID: source.sourceID,
298
314
  connectionName: source.connectionName,
299
315
  dialect: source.dialectName,
300
- buildId: computeBuildId(source, connectionDigests),
316
+ sourceEntityId: computeSourceEntityId(source, connectionDigests),
301
317
  sql: source.getSQL(),
318
+ // Declared `#@ persist` scope/refresh knobs, reported VERBATIM (null
319
+ // = unset). The control plane applies the platform default (unset =>
320
+ // shared) itself, so the publisher must never substitute it — unset
321
+ // has to stay distinguishable from an explicit `shared`. The source's
322
+ // own annotation is the most-specific declaration layer, and the only
323
+ // one that exists today.
324
+ sharing: annotationFields.sharing ?? null,
325
+ refresh: annotationFields.refresh ?? null,
302
326
  columns: deriveColumns(source),
303
- annotationFields: deriveAnnotationFields(source),
327
+ annotationFields,
304
328
  modelPath: sourceModelPaths?.[sourceID],
305
329
  };
306
330
  }
@@ -1028,6 +1028,38 @@ function getMetadataForLookup(
1028
1028
  return name ? metadata.get(name) : undefined;
1029
1029
  }
1030
1030
 
1031
+ /**
1032
+ * Make a connection's digest the caller-supplied API `fingerprint`, verbatim.
1033
+ *
1034
+ * The fingerprint is the connection's data identity as declared by the caller
1035
+ * (secret-free, stable across credential rotation). When set, it replaces the
1036
+ * locally derived `getDigest()` so it becomes the connection's contribution to
1037
+ * content-addressed source ids everywhere a digest is read:
1038
+ *
1039
+ * - build planning (`compilePackageBuildPlan` -> `conn.getDigest()`),
1040
+ * - the package-load worker's connection-metadata RPC, and
1041
+ * - Malloy's own serve-time manifest resolution (the runtime calls
1042
+ * `conn.getDigest()` internally when a build manifest is bound).
1043
+ *
1044
+ * Overriding the method on the resolved instance — inside the one
1045
+ * `lookupConnection` wrapper every resolution passes through — is what keeps
1046
+ * build-time and serve-time ids identical; applying it per call site would
1047
+ * invite divergence and a silent live-serving fallback. The value is used
1048
+ * verbatim (never re-hashed or combined with local details) and treated as an
1049
+ * opaque token: it is not a secret, but it is never logged or parsed.
1050
+ * Without a fingerprint the connection keeps its locally derived digest.
1051
+ */
1052
+ function applyConnectionFingerprint(
1053
+ connection: Connection,
1054
+ metadata: EnvironmentConnectionMetadata | undefined,
1055
+ ): Connection {
1056
+ const fingerprint = metadata?.apiConnection.fingerprint;
1057
+ if (fingerprint) {
1058
+ connection.getDigest = () => fingerprint;
1059
+ }
1060
+ return connection;
1061
+ }
1062
+
1031
1063
  function isDuckDBConnection(
1032
1064
  connection: Connection,
1033
1065
  ): connection is DuckDBConnection {
@@ -1095,10 +1127,11 @@ export function buildEnvironmentMalloyConfig(
1095
1127
  }
1096
1128
 
1097
1129
  malloyConfig.wrapConnections(
1098
- (base: LookupConnection<Connection>): LookupConnection<Connection> => ({
1099
- lookupConnection: async (name?: string): Promise<Connection> => {
1100
- const metadata = getMetadataForLookup(assembled.metadata, name);
1101
-
1130
+ (base: LookupConnection<Connection>): LookupConnection<Connection> => {
1131
+ const resolveConnection = async (
1132
+ name: string | undefined,
1133
+ metadata: EnvironmentConnectionMetadata | undefined,
1134
+ ): Promise<Connection> => {
1102
1135
  if (metadata?.isDuckLake) {
1103
1136
  let connectionPromise = duckLakeCache.get(name!);
1104
1137
  if (!connectionPromise) {
@@ -1210,8 +1243,21 @@ export function buildEnvironmentMalloyConfig(
1210
1243
  await attachOnce(connection, metadata);
1211
1244
  }
1212
1245
  return connection;
1213
- },
1214
- }),
1246
+ };
1247
+
1248
+ return {
1249
+ lookupConnection: async (name?: string): Promise<Connection> => {
1250
+ const metadata = getMetadataForLookup(assembled.metadata, name);
1251
+ // Every resolution branch funnels through this one exit so a
1252
+ // configured fingerprint is applied no matter which wrapper
1253
+ // produced the connection (see applyConnectionFingerprint).
1254
+ return applyConnectionFingerprint(
1255
+ await resolveConnection(name, metadata),
1256
+ metadata,
1257
+ );
1258
+ },
1259
+ };
1260
+ },
1215
1261
  );
1216
1262
 
1217
1263
  return {