@malloy-publisher/server 0.0.218 → 0.0.220
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +262 -25
- package/dist/app/assets/{EnvironmentPage-DxOiCxcd.js → EnvironmentPage-BqLiaatL.js} +1 -1
- package/dist/app/assets/{HomePage-i8qAtD6x.js → HomePage-G_xpt9XE.js} +1 -1
- package/dist/app/assets/LightMode-s1PDRIsF.js +1 -0
- package/dist/app/assets/MainPage-C6TfpC92.js +2 -0
- package/dist/app/assets/{MaterializationsPage-6OEVM543.js → MaterializationsPage-BMO1afhm.js} +1 -1
- package/dist/app/assets/ModelPage-zHOJwMnU.js +1 -0
- package/dist/app/assets/{PackagePage-BVmoVsPQ.js → PackagePage-GCaWmELQ.js} +1 -1
- package/dist/app/assets/{RouteError-DFX521_t.js → RouteError-CIC588k4.js} +1 -1
- package/dist/app/assets/ThemeEditorPage-BdbIfIGO.js +1 -0
- package/dist/app/assets/{WorkbookPage-CzucDJ-T.js → WorkbookPage-C8ucK_H8.js} +1 -1
- package/dist/app/assets/{core-B95VQkAV.es-Cbn-yKG-.js → core-DFvqRVqM.es-NtfPVC7h.js} +1 -1
- package/dist/app/assets/github-dark-DenFmJkN.es-DHJKELXO.js +1 -0
- package/dist/app/assets/index-8E2uLeV9.js +2541 -0
- package/dist/app/assets/index-B3NcDPbp.js +18 -0
- package/dist/app/assets/index-BxDMCn3s.js +527 -0
- package/dist/app/assets/index-aYNf0kTZ.js +1 -0
- package/dist/app/assets/index-aYtt-ovi.js +1761 -0
- package/dist/app/assets/index-w_0OQJgZ.js +23 -0
- package/dist/app/index.html +1 -1
- package/dist/runtime/publisher.js +6 -1
- package/dist/server.mjs +637 -263
- package/package.json +12 -12
- package/src/config.theme.spec.ts +178 -0
- package/src/config.ts +179 -0
- package/src/controller/materialization.controller.spec.ts +4 -4
- package/src/controller/materialization.controller.ts +2 -2
- package/src/controller/theme.controller.ts +83 -0
- package/src/dto/connection.dto.ts +6 -0
- package/src/materialization_metrics.ts +1 -1
- package/src/runtime/publisher.js +6 -1
- package/src/server.ts +58 -1
- package/src/service/build_plan.spec.ts +17 -17
- package/src/service/build_plan.ts +25 -10
- package/src/service/connection.spec.ts +70 -0
- package/src/service/connection.ts +102 -11
- package/src/service/connection_config.spec.ts +138 -0
- package/src/service/connection_config.ts +81 -2
- package/src/service/connection_fingerprint.spec.ts +102 -0
- package/src/service/manifest_loader.spec.ts +5 -5
- package/src/service/manifest_loader.ts +4 -4
- package/src/service/materialization_service.spec.ts +47 -37
- package/src/service/materialization_service.ts +59 -50
- package/src/service/materialization_test_fixtures.ts +5 -5
- package/src/service/model.ts +1 -1
- package/src/service/package.ts +4 -4
- package/src/service/proxy.spec.ts +82 -35
- package/src/service/proxy.ts +55 -40
- package/src/service/theme_store.ts +199 -0
- package/src/storage/DatabaseInterface.ts +1 -1
- package/src/storage/StorageManager.ts +17 -0
- package/src/storage/duckdb/schema.ts +27 -6
- package/src/theme_key_parity.spec.ts +57 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +1 -1
- package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +5 -5
- package/tests/unit/controller/theme.controller.spec.ts +141 -0
- package/tests/unit/service/theme_store.spec.ts +139 -0
- package/dist/app/assets/MainPage-B58d5pmX.js +0 -2
- package/dist/app/assets/ModelPage-8d5l7YkU.js +0 -1
- package/dist/app/assets/index--xJ1pzL-.js +0 -455
- package/dist/app/assets/index-Bxkza-hz.js +0 -40
- package/dist/app/assets/index-DxRKma36.js +0 -1760
- package/dist/app/assets/index.umd-o-yUIEtS.js +0 -2467
|
@@ -34,7 +34,7 @@ import { errMessage } from "../utils";
|
|
|
34
34
|
import {
|
|
35
35
|
CompiledBuildPlan,
|
|
36
36
|
compilePackageBuildPlan,
|
|
37
|
-
|
|
37
|
+
computeSourceEntityId,
|
|
38
38
|
deriveAnnotationFields,
|
|
39
39
|
iterGraphSources,
|
|
40
40
|
} from "./build_plan";
|
|
@@ -43,16 +43,17 @@ import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
|
|
|
43
43
|
import { resolveEnvironmentId } from "./resolve_environment";
|
|
44
44
|
|
|
45
45
|
/**
|
|
46
|
-
* Length of the
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
* tightest at 63).
|
|
46
|
+
* Length of the sourceEntityId prefix used when synthesizing staging table
|
|
47
|
+
* names. 12 hex chars is 48 bits of entropy, well inside every dialect's
|
|
48
|
+
* identifier limit (Postgres is the tightest at 63).
|
|
50
49
|
*/
|
|
51
|
-
const
|
|
50
|
+
const STAGING_ID_LEN = 12;
|
|
52
51
|
|
|
53
52
|
/** Staging suffix appended to a table name while it is being built. */
|
|
54
|
-
export function stagingSuffix(
|
|
55
|
-
|
|
53
|
+
export function stagingSuffix(sourceEntityId: string): string {
|
|
54
|
+
// Drop hyphens so the suffix stays a bare identifier fragment when the id
|
|
55
|
+
// becomes a UUID5 (a no-op for the current hex ids).
|
|
56
|
+
return `_${sourceEntityId.replace(/-/g, "").substring(0, STAGING_ID_LEN)}`;
|
|
56
57
|
}
|
|
57
58
|
|
|
58
59
|
/**
|
|
@@ -334,7 +335,7 @@ export class MaterializationService {
|
|
|
334
335
|
carried = {};
|
|
335
336
|
} else {
|
|
336
337
|
// Skip-if-unchanged: reuse tables from the most recent successful
|
|
337
|
-
// manifest for sources whose
|
|
338
|
+
// manifest for sources whose sourceEntityId is unchanged, unless
|
|
338
339
|
// forceRefresh.
|
|
339
340
|
const priorEntries = opts.forceRefresh
|
|
340
341
|
? {}
|
|
@@ -396,7 +397,7 @@ export class MaterializationService {
|
|
|
396
397
|
/**
|
|
397
398
|
* Derive the publisher's own build instructions for auto-run. Each persist
|
|
398
399
|
* source (respecting the optional sourceNames filter) gets a self-assigned
|
|
399
|
-
* physical table name and COPY realization, unless its
|
|
400
|
+
* physical table name and COPY realization, unless its sourceEntityId is unchanged
|
|
400
401
|
* since `priorEntries` — those are carried forward (reused) instead of
|
|
401
402
|
* rebuilt.
|
|
402
403
|
*/
|
|
@@ -420,24 +421,24 @@ export class MaterializationService {
|
|
|
420
421
|
)) {
|
|
421
422
|
if (include && !include.has(persistSource.name)) continue;
|
|
422
423
|
|
|
423
|
-
const
|
|
424
|
+
const sourceEntityId = computeSourceEntityId(
|
|
424
425
|
persistSource,
|
|
425
426
|
compiled.connectionDigests,
|
|
426
427
|
);
|
|
427
|
-
if (seen.has(
|
|
428
|
-
seen.add(
|
|
428
|
+
if (seen.has(sourceEntityId)) continue;
|
|
429
|
+
seen.add(sourceEntityId);
|
|
429
430
|
|
|
430
|
-
const prior = priorEntries[
|
|
431
|
+
const prior = priorEntries[sourceEntityId];
|
|
431
432
|
if (prior && prior.physicalTableName) {
|
|
432
|
-
carried[
|
|
433
|
+
carried[sourceEntityId] = prior;
|
|
433
434
|
continue;
|
|
434
435
|
}
|
|
435
436
|
|
|
436
437
|
instructions.push({
|
|
437
|
-
|
|
438
|
-
materializedTableId: `local-${
|
|
438
|
+
sourceEntityId,
|
|
439
|
+
materializedTableId: `local-${sourceEntityId.substring(
|
|
439
440
|
0,
|
|
440
|
-
|
|
441
|
+
STAGING_ID_LEN,
|
|
441
442
|
)}`,
|
|
442
443
|
physicalTableName: selfAssignTableName(persistSource),
|
|
443
444
|
realization: "COPY",
|
|
@@ -486,9 +487,11 @@ export class MaterializationService {
|
|
|
486
487
|
entries: Record<string, ManifestEntry>,
|
|
487
488
|
): Promise<void> {
|
|
488
489
|
const manifestEntries: BuildManifest["entries"] = {};
|
|
489
|
-
for (const [
|
|
490
|
+
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
490
491
|
if (entry.physicalTableName) {
|
|
491
|
-
manifestEntries[
|
|
492
|
+
manifestEntries[sourceEntityId] = {
|
|
493
|
+
tableName: entry.physicalTableName,
|
|
494
|
+
};
|
|
492
495
|
}
|
|
493
496
|
}
|
|
494
497
|
try {
|
|
@@ -512,7 +515,7 @@ export class MaterializationService {
|
|
|
512
515
|
|
|
513
516
|
/**
|
|
514
517
|
* Validate caller-supplied build instructions against the package's compiled
|
|
515
|
-
* build plan: every instructed
|
|
518
|
+
* build plan: every instructed sourceEntityId must be a planned source, and only
|
|
516
519
|
* COPY realization is supported. Throws when the package declares no persist
|
|
517
520
|
* source (no plan to build against).
|
|
518
521
|
*/
|
|
@@ -525,15 +528,15 @@ export class MaterializationService {
|
|
|
525
528
|
"Package has no persist sources; buildInstructions cannot be applied",
|
|
526
529
|
);
|
|
527
530
|
}
|
|
528
|
-
const
|
|
531
|
+
const plannedSourceEntityIds = new Set<string>();
|
|
529
532
|
for (const source of Object.values(plan.sources)) {
|
|
530
|
-
|
|
533
|
+
plannedSourceEntityIds.add(source.sourceEntityId);
|
|
531
534
|
}
|
|
532
535
|
|
|
533
536
|
for (const instruction of instructions) {
|
|
534
|
-
if (!
|
|
537
|
+
if (!plannedSourceEntityIds.has(instruction.sourceEntityId)) {
|
|
535
538
|
throw new BadRequestError(
|
|
536
|
-
`Instruction references unknown
|
|
539
|
+
`Instruction references unknown sourceEntityId '${instruction.sourceEntityId}'`,
|
|
537
540
|
);
|
|
538
541
|
}
|
|
539
542
|
// COPY-only for now; SNAPSHOT lands once clone semantics are defined.
|
|
@@ -561,20 +564,20 @@ export class MaterializationService {
|
|
|
561
564
|
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
562
565
|
|
|
563
566
|
// Index instructions by sourceID (the stable per-source handle) so the
|
|
564
|
-
// build no longer recomputes the
|
|
565
|
-
// Recomputing it here forced a caller's
|
|
566
|
-
// content hash, so a caller that derives
|
|
567
|
-
// would have its sources silently skipped (the recomputed
|
|
568
|
-
// not match the instruction).
|
|
569
|
-
// identity. A
|
|
567
|
+
// build no longer recomputes the sourceEntityId to find an instruction.
|
|
568
|
+
// Recomputing it here forced a caller's sourceEntityId to equal the publisher's
|
|
569
|
+
// content hash, so a caller that derives sourceEntityIds by any other scheme
|
|
570
|
+
// would have its sources silently skipped (the recomputed sourceEntityId would
|
|
571
|
+
// not match the instruction). sourceEntityId is treated as opaque, caller-assigned
|
|
572
|
+
// identity. A sourceEntityId index is kept as a fallback for instructions without
|
|
570
573
|
// a sourceID (e.g. standalone auto-run).
|
|
571
574
|
const bySourceID = new Map<string, BuildInstruction>();
|
|
572
|
-
const
|
|
575
|
+
const bySourceEntityId = new Map<string, BuildInstruction>();
|
|
573
576
|
for (const instruction of instructions) {
|
|
574
577
|
if (instruction.sourceID) {
|
|
575
578
|
bySourceID.set(instruction.sourceID, instruction);
|
|
576
579
|
}
|
|
577
|
-
|
|
580
|
+
bySourceEntityId.set(instruction.sourceEntityId, instruction);
|
|
578
581
|
}
|
|
579
582
|
|
|
580
583
|
// Accumulates physical names as sources are built so downstream sources
|
|
@@ -582,11 +585,13 @@ export class MaterializationService {
|
|
|
582
585
|
// it with carried-forward entries so reused upstreams resolve too.
|
|
583
586
|
const manifest = new Manifest();
|
|
584
587
|
const entries: Record<string, ManifestEntry> = {};
|
|
585
|
-
for (const [
|
|
588
|
+
for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
|
|
586
589
|
if (entry.physicalTableName) {
|
|
587
|
-
manifest.update(
|
|
590
|
+
manifest.update(sourceEntityId, {
|
|
591
|
+
tableName: entry.physicalTableName,
|
|
592
|
+
});
|
|
588
593
|
}
|
|
589
|
-
entries[
|
|
594
|
+
entries[sourceEntityId] = entry;
|
|
590
595
|
}
|
|
591
596
|
|
|
592
597
|
for (const graph of graphs) {
|
|
@@ -599,15 +604,19 @@ export class MaterializationService {
|
|
|
599
604
|
for (const persistSource of iterGraphSources(graph, sources)) {
|
|
600
605
|
if (signal.aborted) throw new Error("Build cancelled");
|
|
601
606
|
|
|
602
|
-
// The manifest is keyed by the content
|
|
607
|
+
// The manifest is keyed by the content sourceEntityId — what Malloy
|
|
603
608
|
// recomputes to resolve upstream persist references during SQL
|
|
604
|
-
// generation — independent of the instruction's identity
|
|
605
|
-
const
|
|
606
|
-
|
|
607
|
-
|
|
609
|
+
// generation — independent of the instruction's identity sourceEntityId.
|
|
610
|
+
const sourceEntityId = computeSourceEntityId(
|
|
611
|
+
persistSource,
|
|
612
|
+
connectionDigests,
|
|
613
|
+
);
|
|
614
|
+
// Prefer sourceID matching (so the caller's sourceEntityId scheme stays
|
|
615
|
+
// opaque to the build); fall back to sourceEntityId for instructions
|
|
608
616
|
// without a sourceID (auto-run).
|
|
609
617
|
const instruction =
|
|
610
|
-
bySourceID.get(persistSource.sourceID) ??
|
|
618
|
+
bySourceID.get(persistSource.sourceID) ??
|
|
619
|
+
bySourceEntityId.get(sourceEntityId);
|
|
611
620
|
if (!instruction) continue;
|
|
612
621
|
|
|
613
622
|
const entry = await this.buildOneSource(
|
|
@@ -617,7 +626,7 @@ export class MaterializationService {
|
|
|
617
626
|
connectionDigests,
|
|
618
627
|
manifest,
|
|
619
628
|
);
|
|
620
|
-
entries[
|
|
629
|
+
entries[sourceEntityId] = entry;
|
|
621
630
|
}
|
|
622
631
|
}
|
|
623
632
|
|
|
@@ -627,7 +636,7 @@ export class MaterializationService {
|
|
|
627
636
|
/**
|
|
628
637
|
* Build a single instructed source into its assigned physical table.
|
|
629
638
|
* COPY uses a staging table + atomic rename for crash-safety; the staging
|
|
630
|
-
* name derives from the
|
|
639
|
+
* name derives from the sourceEntityId. Records and returns the manifest entry.
|
|
631
640
|
*/
|
|
632
641
|
private async buildOneSource(
|
|
633
642
|
persistSource: PersistSource,
|
|
@@ -636,7 +645,7 @@ export class MaterializationService {
|
|
|
636
645
|
connectionDigests: Record<string, string>,
|
|
637
646
|
manifest: Manifest,
|
|
638
647
|
): Promise<ManifestEntry> {
|
|
639
|
-
const
|
|
648
|
+
const sourceEntityId = instruction.sourceEntityId;
|
|
640
649
|
const physicalTableName = instruction.physicalTableName;
|
|
641
650
|
const buildSQL = persistSource.getSQL({
|
|
642
651
|
buildManifest: manifest.buildManifest,
|
|
@@ -644,7 +653,7 @@ export class MaterializationService {
|
|
|
644
653
|
});
|
|
645
654
|
|
|
646
655
|
const bareName = bareTableName(physicalTableName);
|
|
647
|
-
const stagingTableName = `${physicalTableName}${stagingSuffix(
|
|
656
|
+
const stagingTableName = `${physicalTableName}${stagingSuffix(sourceEntityId)}`;
|
|
648
657
|
// The control plane sends the logical (unquoted) physical name; dialect-
|
|
649
658
|
// quote each identifier here so a container path or quote-requiring name
|
|
650
659
|
// (e.g. a hyphenated BigQuery project id) produces valid DDL. The manifest
|
|
@@ -681,7 +690,7 @@ export class MaterializationService {
|
|
|
681
690
|
}
|
|
682
691
|
|
|
683
692
|
// Make this table visible to downstream sources built later in this run.
|
|
684
|
-
manifest.update(
|
|
693
|
+
manifest.update(sourceEntityId, { tableName: physicalTableName });
|
|
685
694
|
|
|
686
695
|
const durationMs = Math.round(performance.now() - startTime);
|
|
687
696
|
recordSourceBuildDuration(durationMs);
|
|
@@ -691,7 +700,7 @@ export class MaterializationService {
|
|
|
691
700
|
});
|
|
692
701
|
|
|
693
702
|
return {
|
|
694
|
-
|
|
703
|
+
sourceEntityId,
|
|
695
704
|
sourceName: persistSource.name,
|
|
696
705
|
materializedTableId: instruction.materializedTableId,
|
|
697
706
|
physicalTableName,
|
|
@@ -799,7 +808,7 @@ export class MaterializationService {
|
|
|
799
808
|
if (!connectionName || !physicalTableName) {
|
|
800
809
|
logger.warn("Skipping manifest entry with no connection/table", {
|
|
801
810
|
materializationId: m.id,
|
|
802
|
-
|
|
811
|
+
sourceEntityId: entry.sourceEntityId,
|
|
803
812
|
});
|
|
804
813
|
continue;
|
|
805
814
|
}
|
|
@@ -825,7 +834,7 @@ export class MaterializationService {
|
|
|
825
834
|
// table behind; clean it up too while we hold the connection.
|
|
826
835
|
await connection.runSQL(
|
|
827
836
|
`DROP TABLE IF EXISTS ${quoteTablePath(
|
|
828
|
-
`${physicalTableName}${stagingSuffix(entry.
|
|
837
|
+
`${physicalTableName}${stagingSuffix(entry.sourceEntityId)}`,
|
|
829
838
|
dialect,
|
|
830
839
|
)}`,
|
|
831
840
|
);
|
|
@@ -48,7 +48,7 @@ export function makeBuildPlan(overrides: Partial<BuildPlan> = {}): BuildPlan {
|
|
|
48
48
|
sourceID: "orders@m.malloy",
|
|
49
49
|
connectionName: "duckdb",
|
|
50
50
|
dialect: "duckdb",
|
|
51
|
-
|
|
51
|
+
sourceEntityId: "build-orders",
|
|
52
52
|
sql: "SELECT 1",
|
|
53
53
|
columns: [],
|
|
54
54
|
},
|
|
@@ -62,7 +62,7 @@ export function makeInstruction(
|
|
|
62
62
|
overrides: Partial<BuildInstruction> = {},
|
|
63
63
|
): BuildInstruction {
|
|
64
64
|
return {
|
|
65
|
-
|
|
65
|
+
sourceEntityId: "build-orders",
|
|
66
66
|
materializedTableId: "mt-1",
|
|
67
67
|
physicalTableName: '"orders_v1"',
|
|
68
68
|
realization: "COPY",
|
|
@@ -72,12 +72,12 @@ export function makeInstruction(
|
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
74
|
* A minimal stand-in for a Malloy {@link PersistSource} exposing only what the
|
|
75
|
-
* build internals touch (name/id, deterministic
|
|
75
|
+
* build internals touch (name/id, deterministic sourceEntityId, SQL, and the
|
|
76
76
|
* `#@ persist name=` annotation reader, defaulted to "unset").
|
|
77
77
|
*/
|
|
78
78
|
export function fakeSource(opts: {
|
|
79
79
|
name: string;
|
|
80
|
-
|
|
80
|
+
sourceEntityId: string;
|
|
81
81
|
sql?: string;
|
|
82
82
|
connectionName?: string;
|
|
83
83
|
dialectName?: string;
|
|
@@ -87,7 +87,7 @@ export function fakeSource(opts: {
|
|
|
87
87
|
sourceID: opts.name,
|
|
88
88
|
connectionName: opts.connectionName ?? "duckdb",
|
|
89
89
|
dialectName: opts.dialectName ?? "duckdb",
|
|
90
|
-
makeBuildId: () => opts.
|
|
90
|
+
makeBuildId: () => opts.sourceEntityId,
|
|
91
91
|
getSQL: () => opts.sql ?? "SELECT 1",
|
|
92
92
|
annotations: {
|
|
93
93
|
parseAsTag: () => ({ tag: { text: () => undefined } }),
|
package/src/service/model.ts
CHANGED
|
@@ -1931,7 +1931,7 @@ function makeHydrationRuntime(
|
|
|
1931
1931
|
// (getSQL) time, gated on `prepareResultOptions.buildManifest`; without this
|
|
1932
1932
|
// the hydrated model always recomputes from the base tables even though the
|
|
1933
1933
|
// manifest was bound at load. `strict: false` keeps serving live for any
|
|
1934
|
-
// source whose
|
|
1934
|
+
// source whose sourceEntityId is absent from the manifest.
|
|
1935
1935
|
return new Runtime({
|
|
1936
1936
|
urlReader,
|
|
1937
1937
|
config,
|
package/src/service/package.ts
CHANGED
|
@@ -61,7 +61,7 @@ export class Package {
|
|
|
61
61
|
private packagePath: string;
|
|
62
62
|
private malloyConfig: MalloyConfig;
|
|
63
63
|
// Build-manifest binding state (Malloy Persistence v0). When bound, these
|
|
64
|
-
// entries (
|
|
64
|
+
// entries (sourceEntityId -> { tableName }) are what served queries use to route
|
|
65
65
|
// persist sources to their materialized physical tables; they are also reused
|
|
66
66
|
// by the /compile preview so previewed SQL matches executed SQL. Surfaced on
|
|
67
67
|
// /status (via getPackageMetadata) so the control plane can confirm a worker
|
|
@@ -72,7 +72,7 @@ export class Package {
|
|
|
72
72
|
private manifestEntryCount = 0;
|
|
73
73
|
private boundManifestUri: string | null = null;
|
|
74
74
|
// The package's persist build plan: a deterministic property of the compiled
|
|
75
|
-
// package (per-source
|
|
75
|
+
// package (per-source sourceEntityId, columns, build SQL, dependency graphs),
|
|
76
76
|
// computed once at load from the live (unbound) models so it is stable for a
|
|
77
77
|
// given (package version, connection config). Null when the package declares
|
|
78
78
|
// no persist source. Surfaced read-only on getPackageMetadata() so a caller
|
|
@@ -460,7 +460,7 @@ export class Package {
|
|
|
460
460
|
}
|
|
461
461
|
|
|
462
462
|
/**
|
|
463
|
-
* The package's persist build plan (per-source
|
|
463
|
+
* The package's persist build plan (per-source sourceEntityId, columns, build SQL,
|
|
464
464
|
* dependency graphs), or null when the package declares no persist source.
|
|
465
465
|
* A deterministic property of the compiled package; callers derive build
|
|
466
466
|
* instructions from it for an orchestrated materialization.
|
|
@@ -503,7 +503,7 @@ export class Package {
|
|
|
503
503
|
}
|
|
504
504
|
|
|
505
505
|
/**
|
|
506
|
-
* The currently-bound build-manifest entries (
|
|
506
|
+
* The currently-bound build-manifest entries (sourceEntityId -> { tableName }), or
|
|
507
507
|
* undefined when the package is serving live. Reused by the /compile preview
|
|
508
508
|
* so previewed SQL gets the same persist-source -> physical-table routing as
|
|
509
509
|
* execution.
|
|
@@ -300,39 +300,69 @@ describe("openProxy — SSH tunnel", () => {
|
|
|
300
300
|
},
|
|
301
301
|
{ host: "127.0.0.1", port: echoServer.port },
|
|
302
302
|
),
|
|
303
|
-
).rejects.toThrow(/host-key
|
|
303
|
+
).rejects.toThrow(/host-key verification failed/i);
|
|
304
304
|
});
|
|
305
305
|
|
|
306
|
-
it("
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
306
|
+
it("connects when hostKey is absent — unpinned self-service default", async () => {
|
|
307
|
+
// No hostKey → connect without host-key verification (the SSH transport is
|
|
308
|
+
// still encrypted). Matches mainstream BI tools' SSH-tunnel default.
|
|
309
|
+
const ep = await openProxy(
|
|
310
|
+
{
|
|
311
|
+
type: "ssh",
|
|
312
|
+
ssh: {
|
|
313
|
+
host: "127.0.0.1",
|
|
314
|
+
port: sshServer.port,
|
|
315
|
+
username: "testuser",
|
|
316
|
+
privateKey: clientPrivatePem,
|
|
317
|
+
},
|
|
318
|
+
},
|
|
319
|
+
{ host: "127.0.0.1", port: echoServer.port },
|
|
320
|
+
);
|
|
310
321
|
try {
|
|
311
|
-
await
|
|
312
|
-
|
|
313
|
-
{
|
|
314
|
-
type: "ssh",
|
|
315
|
-
ssh: {
|
|
316
|
-
host: "127.0.0.1",
|
|
317
|
-
port: sshServer.port,
|
|
318
|
-
username: "testuser",
|
|
319
|
-
privateKey: clientPrivatePem,
|
|
320
|
-
},
|
|
321
|
-
},
|
|
322
|
-
{ host: "127.0.0.1", port: echoServer.port },
|
|
323
|
-
),
|
|
324
|
-
).rejects.toThrow(/no hostKey provided/i);
|
|
322
|
+
const reply = await connectAndSend(ep.port, "unpinned");
|
|
323
|
+
expect(reply).toContain("unpinned");
|
|
325
324
|
} finally {
|
|
326
|
-
|
|
327
|
-
process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY = orig;
|
|
328
|
-
}
|
|
325
|
+
await closeQuietly(() => ep.close());
|
|
329
326
|
}
|
|
330
327
|
});
|
|
331
328
|
|
|
332
|
-
it("
|
|
333
|
-
|
|
329
|
+
it("accepts a multi-line hostKey listing several keys — matches any (LB bastion)", async () => {
|
|
330
|
+
// A load-balanced bastion presents a different host key per backend, so the
|
|
331
|
+
// pin lists every backend's key and any match is accepted.
|
|
332
|
+
const decoy = Buffer.alloc(32, 0xff).toString("base64");
|
|
333
|
+
const multiKey = [
|
|
334
|
+
`bastion.example.com ssh-ed25519 ${decoy}`,
|
|
335
|
+
`bastion.example.com ssh-rsa ${sshServer.hostKeyBase64}`,
|
|
336
|
+
].join("\n");
|
|
337
|
+
const ep = await openProxy(
|
|
338
|
+
{
|
|
339
|
+
type: "ssh",
|
|
340
|
+
ssh: {
|
|
341
|
+
host: "127.0.0.1",
|
|
342
|
+
port: sshServer.port,
|
|
343
|
+
username: "testuser",
|
|
344
|
+
privateKey: clientPrivatePem,
|
|
345
|
+
hostKey: multiKey,
|
|
346
|
+
},
|
|
347
|
+
},
|
|
348
|
+
{ host: "127.0.0.1", port: echoServer.port },
|
|
349
|
+
);
|
|
334
350
|
try {
|
|
335
|
-
const
|
|
351
|
+
const reply = await connectAndSend(ep.port, "multi-key");
|
|
352
|
+
expect(reply).toContain("multi-key");
|
|
353
|
+
} finally {
|
|
354
|
+
await closeQuietly(() => ep.close());
|
|
355
|
+
}
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
it("fails closed when hostKey is set but parses to zero keys (comment/blank only)", async () => {
|
|
359
|
+
// A set-but-degenerate pin must NOT fall through to the unpinned branch —
|
|
360
|
+
// the security boundary is gated on hostKey being configured, not on it
|
|
361
|
+
// parsing to >=1 key. (Config load rejects this earlier; this guards the
|
|
362
|
+
// boundary itself.)
|
|
363
|
+
const commentOnly = "# ssh-keyscan bastion.example.com timed out\n \n";
|
|
364
|
+
await expect(
|
|
365
|
+
openProxy(
|
|
336
366
|
{
|
|
337
367
|
type: "ssh",
|
|
338
368
|
ssh: {
|
|
@@ -340,19 +370,36 @@ describe("openProxy — SSH tunnel", () => {
|
|
|
340
370
|
port: sshServer.port,
|
|
341
371
|
username: "testuser",
|
|
342
372
|
privateKey: clientPrivatePem,
|
|
373
|
+
hostKey: commentOnly,
|
|
343
374
|
},
|
|
344
375
|
},
|
|
345
376
|
{ host: "127.0.0.1", port: echoServer.port },
|
|
346
|
-
)
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
377
|
+
),
|
|
378
|
+
).rejects.toThrow(/host-key verification failed/i);
|
|
379
|
+
});
|
|
380
|
+
|
|
381
|
+
it("rejects a multi-line hostKey when none of the listed keys match", async () => {
|
|
382
|
+
const decoy1 = Buffer.alloc(32, 0xff).toString("base64");
|
|
383
|
+
const decoy2 = Buffer.alloc(32, 0xaa).toString("base64");
|
|
384
|
+
const multiKey = [
|
|
385
|
+
`bastion.example.com ssh-ed25519 ${decoy1}`,
|
|
386
|
+
`bastion.example.com ssh-rsa ${decoy2}`,
|
|
387
|
+
].join("\n");
|
|
388
|
+
await expect(
|
|
389
|
+
openProxy(
|
|
390
|
+
{
|
|
391
|
+
type: "ssh",
|
|
392
|
+
ssh: {
|
|
393
|
+
host: "127.0.0.1",
|
|
394
|
+
port: sshServer.port,
|
|
395
|
+
username: "testuser",
|
|
396
|
+
privateKey: clientPrivatePem,
|
|
397
|
+
hostKey: multiKey,
|
|
398
|
+
},
|
|
399
|
+
},
|
|
400
|
+
{ host: "127.0.0.1", port: echoServer.port },
|
|
401
|
+
),
|
|
402
|
+
).rejects.toThrow(/host-key verification failed/i);
|
|
356
403
|
});
|
|
357
404
|
|
|
358
405
|
it("rejects for unsupported proxy type", async () => {
|
package/src/service/proxy.ts
CHANGED
|
@@ -8,24 +8,26 @@
|
|
|
8
8
|
* A connection proxy is a normal connection capability (authorized by whoever
|
|
9
9
|
* configures the connection); it is intentionally NOT behind an env-flag gate,
|
|
10
10
|
* and is kept separate from the `publisher` HTTP multi-hop type's
|
|
11
|
-
* PUBLISHER_ALLOW_PROXY_CONNECTIONS gate.
|
|
12
|
-
*
|
|
11
|
+
* PUBLISHER_ALLOW_PROXY_CONNECTIONS gate. When host-key pinning is configured it
|
|
12
|
+
* is fail-closed (below).
|
|
13
13
|
*
|
|
14
14
|
* # Host-key policy
|
|
15
15
|
*
|
|
16
|
-
*
|
|
17
|
-
* is
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
16
|
+
* `ssh.hostKey` is optional. When set, it pins the bastion's host key(s) and the
|
|
17
|
+
* tunnel is fail-closed on mismatch; it may list multiple known_hosts lines (a
|
|
18
|
+
* load-balanced/HA bastion presents a different key per backend), and any listed
|
|
19
|
+
* key is accepted. When absent, the tunnel connects without host-key
|
|
20
|
+
* verification — the self-service default, matching mainstream BI tools' SSH
|
|
21
|
+
* tunnels. The SSH transport is still encrypted; unpinned, a MITM on the
|
|
22
|
+
* publisher→bastion hop is possible, mitigated by the customer allowlisting our
|
|
23
|
+
* egress on the bastion.
|
|
23
24
|
*/
|
|
24
25
|
|
|
25
26
|
import net from "net";
|
|
26
27
|
import { Client as SshClient } from "ssh2";
|
|
27
28
|
import type { ConnectConfig, SyncHostVerifier } from "ssh2";
|
|
28
29
|
import { components } from "../api";
|
|
30
|
+
import { logger } from "../logger";
|
|
29
31
|
|
|
30
32
|
type ConnectionProxy = components["schemas"]["ConnectionProxy"];
|
|
31
33
|
|
|
@@ -39,17 +41,27 @@ const SSH_CONNECT_TIMEOUT_MS = 15_000;
|
|
|
39
41
|
const SSH_KEEPALIVE_INTERVAL_MS = 15_000;
|
|
40
42
|
|
|
41
43
|
/**
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
44
|
+
* Parse a stored hostKey into the set of base64 wire-format key blobs to accept.
|
|
45
|
+
* The value may hold one or more OpenSSH known_hosts entries (one per line) — a
|
|
46
|
+
* load-balanced bastion presents a different host key per backend, so pinning
|
|
47
|
+
* such a bastion means listing every backend's key. Blank lines and `#` comments
|
|
48
|
+
* are skipped. Each entry may be a full known_hosts line (`[markers] host keytype
|
|
49
|
+
* AAAA…`), a `keytype AAAA…` pair, or a bare base64 blob; SSH key blobs
|
|
50
|
+
* base64-encode to a string starting with "AAAA" (the length prefix of the
|
|
51
|
+
* key-type name), so we pick that token, else the last whitespace-delimited
|
|
52
|
+
* token. Compared against what ssh2 hands us (`key.toString("base64")`).
|
|
48
53
|
*/
|
|
49
|
-
function
|
|
50
|
-
const
|
|
51
|
-
const
|
|
52
|
-
|
|
54
|
+
export function parseHostKeys(raw: string): Set<string> {
|
|
55
|
+
const keys = new Set<string>();
|
|
56
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
57
|
+
const trimmed = line.trim();
|
|
58
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
59
|
+
const tokens = trimmed.split(/\s+/);
|
|
60
|
+
const blob = tokens.find((t) => /^AAAA[A-Za-z0-9+/]+={0,2}$/.test(t));
|
|
61
|
+
const key = blob ?? tokens[tokens.length - 1];
|
|
62
|
+
if (key) keys.add(key);
|
|
63
|
+
}
|
|
64
|
+
return keys;
|
|
53
65
|
}
|
|
54
66
|
|
|
55
67
|
/**
|
|
@@ -112,31 +124,34 @@ function openSshProxy(
|
|
|
112
124
|
hostVerifier: ((key: Buffer): boolean => {
|
|
113
125
|
// `key` is the raw Buffer of the host's public key.
|
|
114
126
|
const presented = key.toString("base64");
|
|
127
|
+
// "Pinned" is gated on hostKey being a non-empty value, not on it
|
|
128
|
+
// parsing to ≥1 key: a set-but-degenerate hostKey (only whitespace,
|
|
129
|
+
// blanks, or comments) must fail closed, never fall through to the
|
|
130
|
+
// unpinned branch. Config load (validateConnectionShape) already
|
|
131
|
+
// rejects that case; this is the security-boundary backstop for any
|
|
132
|
+
// path that reaches here directly. ("" is the unpinned signal.)
|
|
115
133
|
if (ssh.hostKey) {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
),
|
|
122
|
-
);
|
|
123
|
-
return false;
|
|
134
|
+
// Fail-closed, accepting any listed key — an LB/HA bastion presents
|
|
135
|
+
// a different host key per backend.
|
|
136
|
+
const pinned = parseHostKeys(ssh.hostKey);
|
|
137
|
+
if (pinned.has(presented)) {
|
|
138
|
+
return true;
|
|
124
139
|
}
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
140
|
+
fail(
|
|
141
|
+
new Error(
|
|
142
|
+
`SSH host-key verification failed for ${ssh.host}: the presented key ` +
|
|
143
|
+
`(${presented.slice(0, 24)}…) is not among the ${pinned.size} pinned ` +
|
|
144
|
+
`host key(s).`,
|
|
145
|
+
),
|
|
146
|
+
);
|
|
147
|
+
return false;
|
|
130
148
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
`Set hostKey to the bastion's host public key (an OpenSSH ` +
|
|
136
|
-
`known_hosts line or its base64 blob, e.g. from ssh-keyscan).`,
|
|
137
|
-
),
|
|
149
|
+
// Unpinned (no hostKey): connect without host-key verification — the
|
|
150
|
+
// self-service default (see file header).
|
|
151
|
+
logger.warn(
|
|
152
|
+
`Connecting to SSH bastion ${ssh.host} without host-key verification (no hostKey pinned).`,
|
|
138
153
|
);
|
|
139
|
-
return
|
|
154
|
+
return true;
|
|
140
155
|
}) as SyncHostVerifier,
|
|
141
156
|
};
|
|
142
157
|
|