@malloy-publisher/server 0.0.219 → 0.0.220
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +240 -19
- package/dist/app/assets/{EnvironmentPage-gehnjfC6.js → EnvironmentPage-BqLiaatL.js} +1 -1
- package/dist/app/assets/{HomePage-8LQBytE4.js → HomePage-G_xpt9XE.js} +1 -1
- package/dist/app/assets/LightMode-s1PDRIsF.js +1 -0
- package/dist/app/assets/MainPage-C6TfpC92.js +2 -0
- package/dist/app/assets/{MaterializationsPage-D7P1Kp6O.js → MaterializationsPage-BMO1afhm.js} +1 -1
- package/dist/app/assets/ModelPage-zHOJwMnU.js +1 -0
- package/dist/app/assets/{PackagePage-Cz9fVwSG.js → PackagePage-GCaWmELQ.js} +1 -1
- package/dist/app/assets/{RouteError-UONCloyN.js → RouteError-CIC588k4.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-BdbIfIGO.js +1 -0
- package/dist/app/assets/{WorkbookPage-Bhzqvbq_.js → WorkbookPage-C8ucK_H8.js} +1 -1
- package/dist/app/assets/{core-BiGj7BML.es-kMHAa8tP.js → core-DFvqRVqM.es-NtfPVC7h.js} +1 -1
- package/dist/app/assets/github-dark-DenFmJkN.es-DHJKELXO.js +1 -0
- package/dist/app/assets/index-8E2uLeV9.js +2541 -0
- package/dist/app/assets/index-B3NcDPbp.js +18 -0
- package/dist/app/assets/index-BxDMCn3s.js +527 -0
- package/dist/app/assets/index-aYNf0kTZ.js +1 -0
- package/dist/app/assets/index-aYtt-ovi.js +1761 -0
- package/dist/app/assets/index-w_0OQJgZ.js +23 -0
- package/dist/app/index.html +1 -1
- package/dist/runtime/publisher.js +6 -1
- package/dist/server.mjs +377 -50
- package/dist/sshcrypto-8m50vnmb.node +0 -0
- package/package.json +12 -12
- package/src/config.theme.spec.ts +178 -0
- package/src/config.ts +179 -0
- package/src/controller/materialization.controller.spec.ts +4 -4
- package/src/controller/materialization.controller.ts +2 -2
- package/src/controller/theme.controller.ts +83 -0
- package/src/dto/connection.dto.ts +6 -0
- package/src/materialization_metrics.ts +1 -1
- package/src/runtime/publisher.js +6 -1
- package/src/server.ts +58 -1
- package/src/service/build_plan.spec.ts +17 -17
- package/src/service/build_plan.ts +25 -10
- package/src/service/connection.ts +52 -6
- package/src/service/connection_fingerprint.spec.ts +102 -0
- package/src/service/manifest_loader.spec.ts +5 -5
- package/src/service/manifest_loader.ts +4 -4
- package/src/service/materialization_service.spec.ts +47 -37
- package/src/service/materialization_service.ts +59 -50
- package/src/service/materialization_test_fixtures.ts +5 -5
- package/src/service/model.ts +1 -1
- package/src/service/package.ts +4 -4
- package/src/service/theme_store.ts +199 -0
- package/src/storage/DatabaseInterface.ts +1 -1
- package/src/storage/StorageManager.ts +17 -0
- package/src/storage/duckdb/schema.ts +27 -6
- package/src/theme_key_parity.spec.ts +57 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +1 -1
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +5 -5
- package/tests/unit/controller/theme.controller.spec.ts +141 -0
- package/tests/unit/service/theme_store.spec.ts +139 -0
- package/dist/app/assets/MainPage-DDaZLJw-.js +0 -2
- package/dist/app/assets/ModelPage-Do_vhxOc.js +0 -1
- package/dist/app/assets/index-VzRbxcF7.js +0 -455
- package/dist/app/assets/index-ddq4-5hu.js +0 -1760
- package/dist/app/assets/index-qOQF9CXq.js +0 -40
- package/dist/app/assets/index.umd-B2kmxDh7.js +0 -2467
|
@@ -236,13 +236,15 @@ describe("MaterializationService", () => {
|
|
|
236
236
|
).toMatchObject({ mode: "orchestrated" });
|
|
237
237
|
});
|
|
238
238
|
|
|
239
|
-
it("rejects instructions referencing an unknown
|
|
239
|
+
it("rejects instructions referencing an unknown sourceEntityId before creating", async () => {
|
|
240
240
|
setPackage(ctx.environmentStore, {
|
|
241
241
|
getBuildPlan: () => makeBuildPlan(),
|
|
242
242
|
});
|
|
243
243
|
await expect(
|
|
244
244
|
ctx.service.createMaterialization("my-env", "pkg", {
|
|
245
|
-
buildInstructions: [
|
|
245
|
+
buildInstructions: [
|
|
246
|
+
makeInstruction({ sourceEntityId: "ghost" }),
|
|
247
|
+
],
|
|
246
248
|
}),
|
|
247
249
|
).rejects.toThrow(BadRequestError);
|
|
248
250
|
expect(ctx.repository.createMaterialization.called).toBe(false);
|
|
@@ -347,7 +349,7 @@ describe("MaterializationService", () => {
|
|
|
347
349
|
strict: false,
|
|
348
350
|
entries: {
|
|
349
351
|
b1: {
|
|
350
|
-
|
|
352
|
+
sourceEntityId: "b1",
|
|
351
353
|
physicalTableName: "orders_mz",
|
|
352
354
|
connectionName: "duckdb",
|
|
353
355
|
},
|
|
@@ -383,7 +385,7 @@ describe("MaterializationService", () => {
|
|
|
383
385
|
strict: false,
|
|
384
386
|
entries: {
|
|
385
387
|
b1: {
|
|
386
|
-
|
|
388
|
+
sourceEntityId: "b1",
|
|
387
389
|
physicalTableName: "orders_mz",
|
|
388
390
|
connectionName: "duckdb",
|
|
389
391
|
},
|
|
@@ -423,7 +425,7 @@ describe("MaterializationService", () => {
|
|
|
423
425
|
strict: false,
|
|
424
426
|
entries: {
|
|
425
427
|
b1: {
|
|
426
|
-
|
|
428
|
+
sourceEntityId: "b1",
|
|
427
429
|
// Container path on a hyphenated BigQuery project: each
|
|
428
430
|
// segment must be backtick-quoted independently.
|
|
429
431
|
physicalTableName: "my-proj.ds.engaged",
|
|
@@ -460,7 +462,7 @@ describe("MaterializationService", () => {
|
|
|
460
462
|
strict: false,
|
|
461
463
|
entries: {
|
|
462
464
|
b1: {
|
|
463
|
-
|
|
465
|
+
sourceEntityId: "b1",
|
|
464
466
|
physicalTableName: "orders_mz",
|
|
465
467
|
connectionName: "duckdb",
|
|
466
468
|
},
|
|
@@ -481,7 +483,7 @@ describe("MaterializationService", () => {
|
|
|
481
483
|
});
|
|
482
484
|
|
|
483
485
|
describe("stagingSuffix", () => {
|
|
484
|
-
it("derives a short, stable suffix from the
|
|
486
|
+
it("derives a short, stable suffix from the sourceEntityId", () => {
|
|
485
487
|
expect(stagingSuffix("abcdef1234567890")).toBe("_abcdef123456");
|
|
486
488
|
});
|
|
487
489
|
});
|
|
@@ -492,17 +494,17 @@ describe("deriveSelfInstructions", () => {
|
|
|
492
494
|
ctx = createMocks();
|
|
493
495
|
});
|
|
494
496
|
|
|
495
|
-
it("carries forward unchanged
|
|
497
|
+
it("carries forward unchanged sourceEntityIds and builds the rest (deduping repeats)", () => {
|
|
496
498
|
const compiled = compiledWith(
|
|
497
499
|
{
|
|
498
|
-
s1: fakeSource({ name: "s1",
|
|
499
|
-
s2: fakeSource({ name: "s2",
|
|
500
|
+
s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
|
|
501
|
+
s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
|
|
500
502
|
},
|
|
501
503
|
[["s1", "s2"], ["s2"]],
|
|
502
504
|
);
|
|
503
505
|
const priorEntries = {
|
|
504
506
|
b1aaaaaaaaaaaaaa: {
|
|
505
|
-
|
|
507
|
+
sourceEntityId: "b1aaaaaaaaaaaaaa",
|
|
506
508
|
physicalTableName: "s1_prev",
|
|
507
509
|
connectionName: "duckdb",
|
|
508
510
|
},
|
|
@@ -519,7 +521,7 @@ describe("deriveSelfInstructions", () => {
|
|
|
519
521
|
).deriveSelfInstructions(compiled, undefined, priorEntries);
|
|
520
522
|
|
|
521
523
|
expect(instructions).toHaveLength(1);
|
|
522
|
-
expect(instructions[0].
|
|
524
|
+
expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
|
|
523
525
|
expect(instructions[0].physicalTableName).toBe("s2");
|
|
524
526
|
expect(instructions[0].realization).toBe("COPY");
|
|
525
527
|
expect(instructions[0].materializedTableId).toBe("local-b2bbbbbbbbbb");
|
|
@@ -531,8 +533,8 @@ describe("deriveSelfInstructions", () => {
|
|
|
531
533
|
it("honors the sourceNames filter (excluded sources are neither built nor carried)", () => {
|
|
532
534
|
const compiled = compiledWith(
|
|
533
535
|
{
|
|
534
|
-
s1: fakeSource({ name: "s1",
|
|
535
|
-
s2: fakeSource({ name: "s2",
|
|
536
|
+
s1: fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" }),
|
|
537
|
+
s2: fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" }),
|
|
536
538
|
},
|
|
537
539
|
[["s1", "s2"]],
|
|
538
540
|
);
|
|
@@ -547,7 +549,7 @@ describe("deriveSelfInstructions", () => {
|
|
|
547
549
|
).deriveSelfInstructions(compiled, ["s2"], {});
|
|
548
550
|
|
|
549
551
|
expect(instructions).toHaveLength(1);
|
|
550
|
-
expect(instructions[0].
|
|
552
|
+
expect(instructions[0].sourceEntityId).toBe("b2bbbbbbbbbbbbbb");
|
|
551
553
|
expect(Object.keys(carried as Record<string, unknown>)).toHaveLength(0);
|
|
552
554
|
});
|
|
553
555
|
});
|
|
@@ -561,7 +563,7 @@ describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
|
|
|
561
563
|
it("returns entries from the most recent successful run, excluding the in-flight one", async () => {
|
|
562
564
|
const entries = {
|
|
563
565
|
b1: {
|
|
564
|
-
|
|
566
|
+
sourceEntityId: "b1",
|
|
565
567
|
physicalTableName: "orders_v1",
|
|
566
568
|
connectionName: "duckdb",
|
|
567
569
|
},
|
|
@@ -599,7 +601,9 @@ describe("getMostRecentManifestEntries (skip-if-unchanged)", () => {
|
|
|
599
601
|
manifest: {
|
|
600
602
|
builtAt: "t",
|
|
601
603
|
strict: false,
|
|
602
|
-
entries: {
|
|
604
|
+
entries: {
|
|
605
|
+
b1: { sourceEntityId: "b1", physicalTableName: "t1" },
|
|
606
|
+
},
|
|
603
607
|
},
|
|
604
608
|
}),
|
|
605
609
|
]);
|
|
@@ -679,8 +683,8 @@ describe("executeInstructedBuild", () => {
|
|
|
679
683
|
it("builds instructed sources in dependency order and seeds carried entries", async () => {
|
|
680
684
|
const runSQL = sinon.stub().resolves();
|
|
681
685
|
const connection = { runSQL } as unknown as MalloyConnection;
|
|
682
|
-
const s1 = fakeSource({ name: "s1",
|
|
683
|
-
const s2 = fakeSource({ name: "s2",
|
|
686
|
+
const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
|
|
687
|
+
const s2 = fakeSource({ name: "s2", sourceEntityId: "b2bbbbbbbbbbbbbb" });
|
|
684
688
|
const compiled = compiledWith(
|
|
685
689
|
{ s1, s2 },
|
|
686
690
|
[["s1"], ["s2"]],
|
|
@@ -691,13 +695,13 @@ describe("executeInstructedBuild", () => {
|
|
|
691
695
|
compiled,
|
|
692
696
|
[
|
|
693
697
|
{
|
|
694
|
-
|
|
698
|
+
sourceEntityId: "b1aaaaaaaaaaaaaa",
|
|
695
699
|
materializedTableId: "mt-1",
|
|
696
700
|
physicalTableName: "s1_v1",
|
|
697
701
|
realization: "COPY",
|
|
698
702
|
},
|
|
699
703
|
{
|
|
700
|
-
|
|
704
|
+
sourceEntityId: "b2bbbbbbbbbbbbbb",
|
|
701
705
|
materializedTableId: "mt-2",
|
|
702
706
|
physicalTableName: "s2_v1",
|
|
703
707
|
realization: "COPY",
|
|
@@ -705,7 +709,7 @@ describe("executeInstructedBuild", () => {
|
|
|
705
709
|
],
|
|
706
710
|
{
|
|
707
711
|
carried0: {
|
|
708
|
-
|
|
712
|
+
sourceEntityId: "carried0",
|
|
709
713
|
physicalTableName: "carried_tbl",
|
|
710
714
|
connectionName: "duckdb",
|
|
711
715
|
},
|
|
@@ -725,8 +729,14 @@ describe("executeInstructedBuild", () => {
|
|
|
725
729
|
// though the caller instructed it. Both must now build, `mid` first.
|
|
726
730
|
const runSQL = sinon.stub().resolves();
|
|
727
731
|
const connection = { runSQL } as unknown as MalloyConnection;
|
|
728
|
-
const root = fakeSource({
|
|
729
|
-
|
|
732
|
+
const root = fakeSource({
|
|
733
|
+
name: "root",
|
|
734
|
+
sourceEntityId: "brootaaaaaaaaaa",
|
|
735
|
+
});
|
|
736
|
+
const mid = fakeSource({
|
|
737
|
+
name: "mid",
|
|
738
|
+
sourceEntityId: "bmidbbbbbbbbbbb",
|
|
739
|
+
});
|
|
730
740
|
const compiled = {
|
|
731
741
|
graphs: [
|
|
732
742
|
{
|
|
@@ -750,13 +760,13 @@ describe("executeInstructedBuild", () => {
|
|
|
750
760
|
compiled,
|
|
751
761
|
[
|
|
752
762
|
{
|
|
753
|
-
|
|
763
|
+
sourceEntityId: "brootaaaaaaaaaa",
|
|
754
764
|
materializedTableId: "mt-r",
|
|
755
765
|
physicalTableName: "root_v1",
|
|
756
766
|
realization: "COPY",
|
|
757
767
|
},
|
|
758
768
|
{
|
|
759
|
-
|
|
769
|
+
sourceEntityId: "bmidbbbbbbbbbbb",
|
|
760
770
|
materializedTableId: "mt-m",
|
|
761
771
|
physicalTableName: "mid_v1",
|
|
762
772
|
realization: "COPY",
|
|
@@ -779,7 +789,7 @@ describe("executeInstructedBuild", () => {
|
|
|
779
789
|
});
|
|
780
790
|
|
|
781
791
|
it("throws when an instructed graph's connection is missing", async () => {
|
|
782
|
-
const s1 = fakeSource({ name: "s1",
|
|
792
|
+
const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
|
|
783
793
|
const compiled = compiledWith({ s1 }, [["s1"]], new Map());
|
|
784
794
|
|
|
785
795
|
await expect(
|
|
@@ -787,7 +797,7 @@ describe("executeInstructedBuild", () => {
|
|
|
787
797
|
compiled,
|
|
788
798
|
[
|
|
789
799
|
{
|
|
790
|
-
|
|
800
|
+
sourceEntityId: "b1aaaaaaaaaaaaaa",
|
|
791
801
|
materializedTableId: "mt-1",
|
|
792
802
|
physicalTableName: "s1_v1",
|
|
793
803
|
realization: "COPY",
|
|
@@ -809,15 +819,15 @@ describe("buildOneSource", () => {
|
|
|
809
819
|
connection: { runSQL: sinon.SinonStub },
|
|
810
820
|
physicalTableName: string,
|
|
811
821
|
dialectName?: string,
|
|
812
|
-
): Promise<{
|
|
822
|
+
): Promise<{ sourceEntityId: string; physicalTableName: string }> {
|
|
813
823
|
const source = fakeSource({
|
|
814
824
|
name: "orders",
|
|
815
|
-
|
|
825
|
+
sourceEntityId: "abcdef1234567890",
|
|
816
826
|
sql: "SELECT * FROM t",
|
|
817
827
|
dialectName,
|
|
818
828
|
});
|
|
819
829
|
const instruction: BuildInstruction = {
|
|
820
|
-
|
|
830
|
+
sourceEntityId: "abcdef1234567890",
|
|
821
831
|
materializedTableId: "mt-1",
|
|
822
832
|
physicalTableName,
|
|
823
833
|
realization: "COPY",
|
|
@@ -830,7 +840,7 @@ describe("buildOneSource", () => {
|
|
|
830
840
|
c: unknown,
|
|
831
841
|
d: Record<string, string>,
|
|
832
842
|
m: Manifest,
|
|
833
|
-
) => Promise<{
|
|
843
|
+
) => Promise<{ sourceEntityId: string; physicalTableName: string }>;
|
|
834
844
|
}
|
|
835
845
|
).buildOneSource(
|
|
836
846
|
source,
|
|
@@ -853,7 +863,7 @@ describe("buildOneSource", () => {
|
|
|
853
863
|
'ALTER TABLE "orders_v1_abcdef123456" RENAME TO "orders_v1"',
|
|
854
864
|
]);
|
|
855
865
|
expect(entry.physicalTableName).toBe("orders_v1");
|
|
856
|
-
expect(entry.
|
|
866
|
+
expect(entry.sourceEntityId).toBe("abcdef1234567890");
|
|
857
867
|
});
|
|
858
868
|
|
|
859
869
|
it("drops the staging table and rethrows when the build SQL fails", async () => {
|
|
@@ -919,7 +929,7 @@ describe("runBuild (branch behavior)", () => {
|
|
|
919
929
|
function stubEngine(): RunBuildInternals {
|
|
920
930
|
const entries = {
|
|
921
931
|
"build-orders": {
|
|
922
|
-
|
|
932
|
+
sourceEntityId: "build-orders",
|
|
923
933
|
physicalTableName: '"orders_v1"',
|
|
924
934
|
connectionName: "duckdb",
|
|
925
935
|
},
|
|
@@ -995,11 +1005,11 @@ describe("autoLoadManifest", () => {
|
|
|
995
1005
|
const reloadAllModelsForPackage = sinon.stub().resolves();
|
|
996
1006
|
const entries = {
|
|
997
1007
|
b1: {
|
|
998
|
-
|
|
1008
|
+
sourceEntityId: "b1",
|
|
999
1009
|
physicalTableName: "orders_v1",
|
|
1000
1010
|
connectionName: "duckdb",
|
|
1001
1011
|
},
|
|
1002
|
-
b2: {
|
|
1012
|
+
b2: { sourceEntityId: "b2", physicalTableName: undefined },
|
|
1003
1013
|
};
|
|
1004
1014
|
|
|
1005
1015
|
await (
|
|
@@ -1033,7 +1043,7 @@ describe("autoLoadManifest", () => {
|
|
|
1033
1043
|
) => Promise<void>;
|
|
1034
1044
|
}
|
|
1035
1045
|
).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", {
|
|
1036
|
-
b1: {
|
|
1046
|
+
b1: { sourceEntityId: "b1", physicalTableName: "t" },
|
|
1037
1047
|
}),
|
|
1038
1048
|
).resolves.toBeUndefined();
|
|
1039
1049
|
});
|
|
@@ -34,7 +34,7 @@ import { errMessage } from "../utils";
|
|
|
34
34
|
import {
|
|
35
35
|
CompiledBuildPlan,
|
|
36
36
|
compilePackageBuildPlan,
|
|
37
|
-
|
|
37
|
+
computeSourceEntityId,
|
|
38
38
|
deriveAnnotationFields,
|
|
39
39
|
iterGraphSources,
|
|
40
40
|
} from "./build_plan";
|
|
@@ -43,16 +43,17 @@ import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
|
|
|
43
43
|
import { resolveEnvironmentId } from "./resolve_environment";
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
|
-
* Length of the
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
* tightest at 63).
|
|
46
|
+
* Length of the sourceEntityId prefix used when synthesizing staging table
|
|
47
|
+
* names. 12 hex chars is 48 bits of entropy, well inside every dialect's
|
|
48
|
+
* identifier limit (Postgres is the tightest at 63).
|
|
50
49
|
*/
|
|
51
|
-
const
|
|
50
|
+
const STAGING_ID_LEN = 12;
|
|
52
51
|
|
|
53
52
|
/** Staging suffix appended to a table name while it is being built. */
|
|
54
|
-
export function stagingSuffix(
|
|
55
|
-
|
|
53
|
+
export function stagingSuffix(sourceEntityId: string): string {
|
|
54
|
+
// Drop hyphens so the suffix stays a bare identifier fragment when the id
|
|
55
|
+
// becomes a UUID5 (a no-op for the current hex ids).
|
|
56
|
+
return `_${sourceEntityId.replace(/-/g, "").substring(0, STAGING_ID_LEN)}`;
|
|
56
57
|
}
|
|
57
58
|
|
|
58
59
|
/**
|
|
@@ -334,7 +335,7 @@ export class MaterializationService {
|
|
|
334
335
|
carried = {};
|
|
335
336
|
} else {
|
|
336
337
|
// Skip-if-unchanged: reuse tables from the most recent successful
|
|
337
|
-
// manifest for sources whose
|
|
338
|
+
// manifest for sources whose sourceEntityId is unchanged, unless
|
|
338
339
|
// forceRefresh.
|
|
339
340
|
const priorEntries = opts.forceRefresh
|
|
340
341
|
? {}
|
|
@@ -396,7 +397,7 @@ export class MaterializationService {
|
|
|
396
397
|
/**
|
|
397
398
|
* Derive the publisher's own build instructions for auto-run. Each persist
|
|
398
399
|
* source (respecting the optional sourceNames filter) gets a self-assigned
|
|
399
|
-
* physical table name and COPY realization, unless its
|
|
400
|
+
* physical table name and COPY realization, unless its sourceEntityId is unchanged
|
|
400
401
|
* since `priorEntries` — those are carried forward (reused) instead of
|
|
401
402
|
* rebuilt.
|
|
402
403
|
*/
|
|
@@ -420,24 +421,24 @@ export class MaterializationService {
|
|
|
420
421
|
)) {
|
|
421
422
|
if (include && !include.has(persistSource.name)) continue;
|
|
422
423
|
|
|
423
|
-
const
|
|
424
|
+
const sourceEntityId = computeSourceEntityId(
|
|
424
425
|
persistSource,
|
|
425
426
|
compiled.connectionDigests,
|
|
426
427
|
);
|
|
427
|
-
if (seen.has(
|
|
428
|
-
seen.add(
|
|
428
|
+
if (seen.has(sourceEntityId)) continue;
|
|
429
|
+
seen.add(sourceEntityId);
|
|
429
430
|
|
|
430
|
-
const prior = priorEntries[
|
|
431
|
+
const prior = priorEntries[sourceEntityId];
|
|
431
432
|
if (prior && prior.physicalTableName) {
|
|
432
|
-
carried[
|
|
433
|
+
carried[sourceEntityId] = prior;
|
|
433
434
|
continue;
|
|
434
435
|
}
|
|
435
436
|
|
|
436
437
|
instructions.push({
|
|
437
|
-
|
|
438
|
-
materializedTableId: `local-${
|
|
438
|
+
sourceEntityId,
|
|
439
|
+
materializedTableId: `local-${sourceEntityId.substring(
|
|
439
440
|
0,
|
|
440
|
-
|
|
441
|
+
STAGING_ID_LEN,
|
|
441
442
|
)}`,
|
|
442
443
|
physicalTableName: selfAssignTableName(persistSource),
|
|
443
444
|
realization: "COPY",
|
|
@@ -486,9 +487,11 @@ export class MaterializationService {
|
|
|
486
487
|
entries: Record<string, ManifestEntry>,
|
|
487
488
|
): Promise<void> {
|
|
488
489
|
const manifestEntries: BuildManifest["entries"] = {};
|
|
489
|
-
for (const [
|
|
490
|
+
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
490
491
|
if (entry.physicalTableName) {
|
|
491
|
-
manifestEntries[
|
|
492
|
+
manifestEntries[sourceEntityId] = {
|
|
493
|
+
tableName: entry.physicalTableName,
|
|
494
|
+
};
|
|
492
495
|
}
|
|
493
496
|
}
|
|
494
497
|
try {
|
|
@@ -512,7 +515,7 @@ export class MaterializationService {
|
|
|
512
515
|
|
|
513
516
|
/**
|
|
514
517
|
* Validate caller-supplied build instructions against the package's compiled
|
|
515
|
-
* build plan: every instructed
|
|
518
|
+
* build plan: every instructed sourceEntityId must be a planned source, and only
|
|
516
519
|
* COPY realization is supported. Throws when the package declares no persist
|
|
517
520
|
* source (no plan to build against).
|
|
518
521
|
*/
|
|
@@ -525,15 +528,15 @@ export class MaterializationService {
|
|
|
525
528
|
"Package has no persist sources; buildInstructions cannot be applied",
|
|
526
529
|
);
|
|
527
530
|
}
|
|
528
|
-
const
|
|
531
|
+
const plannedSourceEntityIds = new Set<string>();
|
|
529
532
|
for (const source of Object.values(plan.sources)) {
|
|
530
|
-
|
|
533
|
+
plannedSourceEntityIds.add(source.sourceEntityId);
|
|
531
534
|
}
|
|
532
535
|
|
|
533
536
|
for (const instruction of instructions) {
|
|
534
|
-
if (!
|
|
537
|
+
if (!plannedSourceEntityIds.has(instruction.sourceEntityId)) {
|
|
535
538
|
throw new BadRequestError(
|
|
536
|
-
`Instruction references unknown
|
|
539
|
+
`Instruction references unknown sourceEntityId '${instruction.sourceEntityId}'`,
|
|
537
540
|
);
|
|
538
541
|
}
|
|
539
542
|
// COPY-only for now; SNAPSHOT lands once clone semantics are defined.
|
|
@@ -561,20 +564,20 @@ export class MaterializationService {
|
|
|
561
564
|
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
562
565
|
|
|
563
566
|
// Index instructions by sourceID (the stable per-source handle) so the
|
|
564
|
-
// build no longer recomputes the
|
|
565
|
-
// Recomputing it here forced a caller's
|
|
566
|
-
// content hash, so a caller that derives
|
|
567
|
-
// would have its sources silently skipped (the recomputed
|
|
568
|
-
// not match the instruction).
|
|
569
|
-
// identity. A
|
|
567
|
+
// build no longer recomputes the sourceEntityId to find an instruction.
|
|
568
|
+
// Recomputing it here forced a caller's sourceEntityId to equal the publisher's
|
|
569
|
+
// content hash, so a caller that derives sourceEntityIds by any other scheme
|
|
570
|
+
// would have its sources silently skipped (the recomputed sourceEntityId would
|
|
571
|
+
// not match the instruction). sourceEntityId is treated as opaque, caller-assigned
|
|
572
|
+
// identity. A sourceEntityId index is kept as a fallback for instructions without
|
|
570
573
|
// a sourceID (e.g. standalone auto-run).
|
|
571
574
|
const bySourceID = new Map<string, BuildInstruction>();
|
|
572
|
-
const
|
|
575
|
+
const bySourceEntityId = new Map<string, BuildInstruction>();
|
|
573
576
|
for (const instruction of instructions) {
|
|
574
577
|
if (instruction.sourceID) {
|
|
575
578
|
bySourceID.set(instruction.sourceID, instruction);
|
|
576
579
|
}
|
|
577
|
-
|
|
580
|
+
bySourceEntityId.set(instruction.sourceEntityId, instruction);
|
|
578
581
|
}
|
|
579
582
|
|
|
580
583
|
// Accumulates physical names as sources are built so downstream sources
|
|
@@ -582,11 +585,13 @@ export class MaterializationService {
|
|
|
582
585
|
// it with carried-forward entries so reused upstreams resolve too.
|
|
583
586
|
const manifest = new Manifest();
|
|
584
587
|
const entries: Record<string, ManifestEntry> = {};
|
|
585
|
-
for (const [
|
|
588
|
+
for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
|
|
586
589
|
if (entry.physicalTableName) {
|
|
587
|
-
manifest.update(
|
|
590
|
+
manifest.update(sourceEntityId, {
|
|
591
|
+
tableName: entry.physicalTableName,
|
|
592
|
+
});
|
|
588
593
|
}
|
|
589
|
-
entries[
|
|
594
|
+
entries[sourceEntityId] = entry;
|
|
590
595
|
}
|
|
591
596
|
|
|
592
597
|
for (const graph of graphs) {
|
|
@@ -599,15 +604,19 @@ export class MaterializationService {
|
|
|
599
604
|
for (const persistSource of iterGraphSources(graph, sources)) {
|
|
600
605
|
if (signal.aborted) throw new Error("Build cancelled");
|
|
601
606
|
|
|
602
|
-
// The manifest is keyed by the content
|
|
607
|
+
// The manifest is keyed by the content sourceEntityId — what Malloy
|
|
603
608
|
// recomputes to resolve upstream persist references during SQL
|
|
604
|
-
// generation — independent of the instruction's identity
|
|
605
|
-
const
|
|
606
|
-
|
|
607
|
-
|
|
609
|
+
// generation — independent of the instruction's identity sourceEntityId.
|
|
610
|
+
const sourceEntityId = computeSourceEntityId(
|
|
611
|
+
persistSource,
|
|
612
|
+
connectionDigests,
|
|
613
|
+
);
|
|
614
|
+
// Prefer sourceID matching (so the caller's sourceEntityId scheme stays
|
|
615
|
+
// opaque to the build); fall back to sourceEntityId for instructions
|
|
608
616
|
// without a sourceID (auto-run).
|
|
609
617
|
const instruction =
|
|
610
|
-
bySourceID.get(persistSource.sourceID) ??
|
|
618
|
+
bySourceID.get(persistSource.sourceID) ??
|
|
619
|
+
bySourceEntityId.get(sourceEntityId);
|
|
611
620
|
if (!instruction) continue;
|
|
612
621
|
|
|
613
622
|
const entry = await this.buildOneSource(
|
|
@@ -617,7 +626,7 @@ export class MaterializationService {
|
|
|
617
626
|
connectionDigests,
|
|
618
627
|
manifest,
|
|
619
628
|
);
|
|
620
|
-
entries[
|
|
629
|
+
entries[sourceEntityId] = entry;
|
|
621
630
|
}
|
|
622
631
|
}
|
|
623
632
|
|
|
@@ -627,7 +636,7 @@ export class MaterializationService {
|
|
|
627
636
|
/**
|
|
628
637
|
* Build a single instructed source into its assigned physical table.
|
|
629
638
|
* COPY uses a staging table + atomic rename for crash-safety; the staging
|
|
630
|
-
* name derives from the
|
|
639
|
+
* name derives from the sourceEntityId. Records and returns the manifest entry.
|
|
631
640
|
*/
|
|
632
641
|
private async buildOneSource(
|
|
633
642
|
persistSource: PersistSource,
|
|
@@ -636,7 +645,7 @@ export class MaterializationService {
|
|
|
636
645
|
connectionDigests: Record<string, string>,
|
|
637
646
|
manifest: Manifest,
|
|
638
647
|
): Promise<ManifestEntry> {
|
|
639
|
-
const
|
|
648
|
+
const sourceEntityId = instruction.sourceEntityId;
|
|
640
649
|
const physicalTableName = instruction.physicalTableName;
|
|
641
650
|
const buildSQL = persistSource.getSQL({
|
|
642
651
|
buildManifest: manifest.buildManifest,
|
|
@@ -644,7 +653,7 @@ export class MaterializationService {
|
|
|
644
653
|
});
|
|
645
654
|
|
|
646
655
|
const bareName = bareTableName(physicalTableName);
|
|
647
|
-
const stagingTableName = `${physicalTableName}${stagingSuffix(
|
|
656
|
+
const stagingTableName = `${physicalTableName}${stagingSuffix(sourceEntityId)}`;
|
|
648
657
|
// The control plane sends the logical (unquoted) physical name; dialect-
|
|
649
658
|
// quote each identifier here so a container path or quote-requiring name
|
|
650
659
|
// (e.g. a hyphenated BigQuery project id) produces valid DDL. The manifest
|
|
@@ -681,7 +690,7 @@ export class MaterializationService {
|
|
|
681
690
|
}
|
|
682
691
|
|
|
683
692
|
// Make this table visible to downstream sources built later in this run.
|
|
684
|
-
manifest.update(
|
|
693
|
+
manifest.update(sourceEntityId, { tableName: physicalTableName });
|
|
685
694
|
|
|
686
695
|
const durationMs = Math.round(performance.now() - startTime);
|
|
687
696
|
recordSourceBuildDuration(durationMs);
|
|
@@ -691,7 +700,7 @@ export class MaterializationService {
|
|
|
691
700
|
});
|
|
692
701
|
|
|
693
702
|
return {
|
|
694
|
-
|
|
703
|
+
sourceEntityId,
|
|
695
704
|
sourceName: persistSource.name,
|
|
696
705
|
materializedTableId: instruction.materializedTableId,
|
|
697
706
|
physicalTableName,
|
|
@@ -799,7 +808,7 @@ export class MaterializationService {
|
|
|
799
808
|
if (!connectionName || !physicalTableName) {
|
|
800
809
|
logger.warn("Skipping manifest entry with no connection/table", {
|
|
801
810
|
materializationId: m.id,
|
|
802
|
-
|
|
811
|
+
sourceEntityId: entry.sourceEntityId,
|
|
803
812
|
});
|
|
804
813
|
continue;
|
|
805
814
|
}
|
|
@@ -825,7 +834,7 @@ export class MaterializationService {
|
|
|
825
834
|
// table behind; clean it up too while we hold the connection.
|
|
826
835
|
await connection.runSQL(
|
|
827
836
|
`DROP TABLE IF EXISTS ${quoteTablePath(
|
|
828
|
-
`${physicalTableName}${stagingSuffix(entry.
|
|
837
|
+
`${physicalTableName}${stagingSuffix(entry.sourceEntityId)}`,
|
|
829
838
|
dialect,
|
|
830
839
|
)}`,
|
|
831
840
|
);
|
|
@@ -48,7 +48,7 @@ export function makeBuildPlan(overrides: Partial<BuildPlan> = {}): BuildPlan {
|
|
|
48
48
|
sourceID: "orders@m.malloy",
|
|
49
49
|
connectionName: "duckdb",
|
|
50
50
|
dialect: "duckdb",
|
|
51
|
-
|
|
51
|
+
sourceEntityId: "build-orders",
|
|
52
52
|
sql: "SELECT 1",
|
|
53
53
|
columns: [],
|
|
54
54
|
},
|
|
@@ -62,7 +62,7 @@ export function makeInstruction(
|
|
|
62
62
|
overrides: Partial<BuildInstruction> = {},
|
|
63
63
|
): BuildInstruction {
|
|
64
64
|
return {
|
|
65
|
-
|
|
65
|
+
sourceEntityId: "build-orders",
|
|
66
66
|
materializedTableId: "mt-1",
|
|
67
67
|
physicalTableName: '"orders_v1"',
|
|
68
68
|
realization: "COPY",
|
|
@@ -72,12 +72,12 @@ export function makeInstruction(
|
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
74
|
* A minimal stand-in for a Malloy {@link PersistSource} exposing only what the
|
|
75
|
-
* build internals touch (name/id, deterministic
|
|
75
|
+
* build internals touch (name/id, deterministic sourceEntityId, SQL, and the
|
|
76
76
|
* `#@ persist name=` annotation reader, defaulted to "unset").
|
|
77
77
|
*/
|
|
78
78
|
export function fakeSource(opts: {
|
|
79
79
|
name: string;
|
|
80
|
-
|
|
80
|
+
sourceEntityId: string;
|
|
81
81
|
sql?: string;
|
|
82
82
|
connectionName?: string;
|
|
83
83
|
dialectName?: string;
|
|
@@ -87,7 +87,7 @@ export function fakeSource(opts: {
|
|
|
87
87
|
sourceID: opts.name,
|
|
88
88
|
connectionName: opts.connectionName ?? "duckdb",
|
|
89
89
|
dialectName: opts.dialectName ?? "duckdb",
|
|
90
|
-
makeBuildId: () => opts.
|
|
90
|
+
makeBuildId: () => opts.sourceEntityId,
|
|
91
91
|
getSQL: () => opts.sql ?? "SELECT 1",
|
|
92
92
|
annotations: {
|
|
93
93
|
parseAsTag: () => ({ tag: { text: () => undefined } }),
|
package/src/service/model.ts
CHANGED
|
@@ -1931,7 +1931,7 @@ function makeHydrationRuntime(
|
|
|
1931
1931
|
// (getSQL) time, gated on `prepareResultOptions.buildManifest`; without this
|
|
1932
1932
|
// the hydrated model always recomputes from the base tables even though the
|
|
1933
1933
|
// manifest was bound at load. `strict: false` keeps serving live for any
|
|
1934
|
-
// source whose
|
|
1934
|
+
// source whose sourceEntityId is absent from the manifest.
|
|
1935
1935
|
return new Runtime({
|
|
1936
1936
|
urlReader,
|
|
1937
1937
|
config,
|
package/src/service/package.ts
CHANGED
|
@@ -61,7 +61,7 @@ export class Package {
|
|
|
61
61
|
private packagePath: string;
|
|
62
62
|
private malloyConfig: MalloyConfig;
|
|
63
63
|
// Build-manifest binding state (Malloy Persistence v0). When bound, these
|
|
64
|
-
// entries (
|
|
64
|
+
// entries (sourceEntityId -> { tableName }) are what served queries use to route
|
|
65
65
|
// persist sources to their materialized physical tables; they are also reused
|
|
66
66
|
// by the /compile preview so previewed SQL matches executed SQL. Surfaced on
|
|
67
67
|
// /status (via getPackageMetadata) so the control plane can confirm a worker
|
|
@@ -72,7 +72,7 @@ export class Package {
|
|
|
72
72
|
private manifestEntryCount = 0;
|
|
73
73
|
private boundManifestUri: string | null = null;
|
|
74
74
|
// The package's persist build plan: a deterministic property of the compiled
|
|
75
|
-
// package (per-source
|
|
75
|
+
// package (per-source sourceEntityId, columns, build SQL, dependency graphs),
|
|
76
76
|
// computed once at load from the live (unbound) models so it is stable for a
|
|
77
77
|
// given (package version, connection config). Null when the package declares
|
|
78
78
|
// no persist source. Surfaced read-only on getPackageMetadata() so a caller
|
|
@@ -460,7 +460,7 @@ export class Package {
|
|
|
460
460
|
}
|
|
461
461
|
|
|
462
462
|
/**
|
|
463
|
-
* The package's persist build plan (per-source
|
|
463
|
+
* The package's persist build plan (per-source sourceEntityId, columns, build SQL,
|
|
464
464
|
* dependency graphs), or null when the package declares no persist source.
|
|
465
465
|
* A deterministic property of the compiled package; callers derive build
|
|
466
466
|
* instructions from it for an orchestrated materialization.
|
|
@@ -503,7 +503,7 @@ export class Package {
|
|
|
503
503
|
}
|
|
504
504
|
|
|
505
505
|
/**
|
|
506
|
-
* The currently-bound build-manifest entries (
|
|
506
|
+
* The currently-bound build-manifest entries (sourceEntityId -> { tableName }), or
|
|
507
507
|
* undefined when the package is serving live. Reused by the /compile preview
|
|
508
508
|
* so previewed SQL gets the same persist-source -> physical-table routing as
|
|
509
509
|
* execution.
|