@malloy-publisher/server 0.0.218 → 0.0.220

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/dist/app/api-doc.yaml +262 -25
  2. package/dist/app/assets/{EnvironmentPage-DxOiCxcd.js → EnvironmentPage-BqLiaatL.js} +1 -1
  3. package/dist/app/assets/{HomePage-i8qAtD6x.js → HomePage-G_xpt9XE.js} +1 -1
  4. package/dist/app/assets/LightMode-s1PDRIsF.js +1 -0
  5. package/dist/app/assets/MainPage-C6TfpC92.js +2 -0
  6. package/dist/app/assets/{MaterializationsPage-6OEVM543.js → MaterializationsPage-BMO1afhm.js} +1 -1
  7. package/dist/app/assets/ModelPage-zHOJwMnU.js +1 -0
  8. package/dist/app/assets/{PackagePage-BVmoVsPQ.js → PackagePage-GCaWmELQ.js} +1 -1
  9. package/dist/app/assets/{RouteError-DFX521_t.js → RouteError-CIC588k4.js} +1 -1
  10. package/dist/app/assets/ThemeEditorPage-BdbIfIGO.js +1 -0
  11. package/dist/app/assets/{WorkbookPage-CzucDJ-T.js → WorkbookPage-C8ucK_H8.js} +1 -1
  12. package/dist/app/assets/{core-B95VQkAV.es-Cbn-yKG-.js → core-DFvqRVqM.es-NtfPVC7h.js} +1 -1
  13. package/dist/app/assets/github-dark-DenFmJkN.es-DHJKELXO.js +1 -0
  14. package/dist/app/assets/index-8E2uLeV9.js +2541 -0
  15. package/dist/app/assets/index-B3NcDPbp.js +18 -0
  16. package/dist/app/assets/index-BxDMCn3s.js +527 -0
  17. package/dist/app/assets/index-aYNf0kTZ.js +1 -0
  18. package/dist/app/assets/index-aYtt-ovi.js +1761 -0
  19. package/dist/app/assets/index-w_0OQJgZ.js +23 -0
  20. package/dist/app/index.html +1 -1
  21. package/dist/runtime/publisher.js +6 -1
  22. package/dist/server.mjs +637 -263
  23. package/package.json +12 -12
  24. package/src/config.theme.spec.ts +178 -0
  25. package/src/config.ts +179 -0
  26. package/src/controller/materialization.controller.spec.ts +4 -4
  27. package/src/controller/materialization.controller.ts +2 -2
  28. package/src/controller/theme.controller.ts +83 -0
  29. package/src/dto/connection.dto.ts +6 -0
  30. package/src/materialization_metrics.ts +1 -1
  31. package/src/runtime/publisher.js +6 -1
  32. package/src/server.ts +58 -1
  33. package/src/service/build_plan.spec.ts +17 -17
  34. package/src/service/build_plan.ts +25 -10
  35. package/src/service/connection.spec.ts +70 -0
  36. package/src/service/connection.ts +102 -11
  37. package/src/service/connection_config.spec.ts +138 -0
  38. package/src/service/connection_config.ts +81 -2
  39. package/src/service/connection_fingerprint.spec.ts +102 -0
  40. package/src/service/manifest_loader.spec.ts +5 -5
  41. package/src/service/manifest_loader.ts +4 -4
  42. package/src/service/materialization_service.spec.ts +47 -37
  43. package/src/service/materialization_service.ts +59 -50
  44. package/src/service/materialization_test_fixtures.ts +5 -5
  45. package/src/service/model.ts +1 -1
  46. package/src/service/package.ts +4 -4
  47. package/src/service/proxy.spec.ts +82 -35
  48. package/src/service/proxy.ts +55 -40
  49. package/src/service/theme_store.ts +199 -0
  50. package/src/storage/DatabaseInterface.ts +1 -1
  51. package/src/storage/StorageManager.ts +17 -0
  52. package/src/storage/duckdb/schema.ts +27 -6
  53. package/src/theme_key_parity.spec.ts +57 -0
  54. package/tests/integration/materialization/manifest_binding.integration.spec.ts +1 -1
  55. package/tests/integration/materialization/materialization_lifecycle.integration.spec.ts +5 -5
  56. package/tests/unit/controller/theme.controller.spec.ts +141 -0
  57. package/tests/unit/service/theme_store.spec.ts +139 -0
  58. package/dist/app/assets/MainPage-B58d5pmX.js +0 -2
  59. package/dist/app/assets/ModelPage-8d5l7YkU.js +0 -1
  60. package/dist/app/assets/index--xJ1pzL-.js +0 -455
  61. package/dist/app/assets/index-Bxkza-hz.js +0 -40
  62. package/dist/app/assets/index-DxRKma36.js +0 -1760
  63. package/dist/app/assets/index.umd-o-yUIEtS.js +0 -2467
@@ -34,7 +34,7 @@ import { errMessage } from "../utils";
34
34
  import {
35
35
  CompiledBuildPlan,
36
36
  compilePackageBuildPlan,
37
- computeBuildId,
37
+ computeSourceEntityId,
38
38
  deriveAnnotationFields,
39
39
  iterGraphSources,
40
40
  } from "./build_plan";
@@ -43,16 +43,17 @@ import { bareTableName, quoteIdentifier, quoteTablePath } from "./quoting";
43
43
  import { resolveEnvironmentId } from "./resolve_environment";
44
44
 
45
45
  /**
46
- * Length of the buildId prefix used when synthesizing staging table names.
47
- * buildId is a 64-char SHA-256 hex string; 12 hex chars is 48 bits of
48
- * entropy, well inside every dialect's identifier limit (Postgres is the
49
- * tightest at 63).
46
+ * Length of the sourceEntityId prefix used when synthesizing staging table
47
+ * names. 12 hex chars is 48 bits of entropy, well inside every dialect's
48
+ * identifier limit (Postgres is the tightest at 63).
50
49
  */
51
- const STAGING_BUILD_ID_LEN = 12;
50
+ const STAGING_ID_LEN = 12;
52
51
 
53
52
  /** Staging suffix appended to a table name while it is being built. */
54
- export function stagingSuffix(buildId: string): string {
55
- return `_${buildId.substring(0, STAGING_BUILD_ID_LEN)}`;
53
+ export function stagingSuffix(sourceEntityId: string): string {
54
+ // Drop hyphens so the suffix stays a bare identifier fragment when the id
55
+ // becomes a UUID5 (a no-op for the current hex ids).
56
+ return `_${sourceEntityId.replace(/-/g, "").substring(0, STAGING_ID_LEN)}`;
56
57
  }
57
58
 
58
59
  /**
@@ -334,7 +335,7 @@ export class MaterializationService {
334
335
  carried = {};
335
336
  } else {
336
337
  // Skip-if-unchanged: reuse tables from the most recent successful
337
- // manifest for sources whose buildId is unchanged, unless
338
+ // manifest for sources whose sourceEntityId is unchanged, unless
338
339
  // forceRefresh.
339
340
  const priorEntries = opts.forceRefresh
340
341
  ? {}
@@ -396,7 +397,7 @@ export class MaterializationService {
396
397
  /**
397
398
  * Derive the publisher's own build instructions for auto-run. Each persist
398
399
  * source (respecting the optional sourceNames filter) gets a self-assigned
399
- * physical table name and COPY realization, unless its buildId is unchanged
400
+ * physical table name and COPY realization, unless its sourceEntityId is unchanged
400
401
  * since `priorEntries` — those are carried forward (reused) instead of
401
402
  * rebuilt.
402
403
  */
@@ -420,24 +421,24 @@ export class MaterializationService {
420
421
  )) {
421
422
  if (include && !include.has(persistSource.name)) continue;
422
423
 
423
- const buildId = computeBuildId(
424
+ const sourceEntityId = computeSourceEntityId(
424
425
  persistSource,
425
426
  compiled.connectionDigests,
426
427
  );
427
- if (seen.has(buildId)) continue;
428
- seen.add(buildId);
428
+ if (seen.has(sourceEntityId)) continue;
429
+ seen.add(sourceEntityId);
429
430
 
430
- const prior = priorEntries[buildId];
431
+ const prior = priorEntries[sourceEntityId];
431
432
  if (prior && prior.physicalTableName) {
432
- carried[buildId] = prior;
433
+ carried[sourceEntityId] = prior;
433
434
  continue;
434
435
  }
435
436
 
436
437
  instructions.push({
437
- buildId,
438
- materializedTableId: `local-${buildId.substring(
438
+ sourceEntityId,
439
+ materializedTableId: `local-${sourceEntityId.substring(
439
440
  0,
440
- STAGING_BUILD_ID_LEN,
441
+ STAGING_ID_LEN,
441
442
  )}`,
442
443
  physicalTableName: selfAssignTableName(persistSource),
443
444
  realization: "COPY",
@@ -486,9 +487,11 @@ export class MaterializationService {
486
487
  entries: Record<string, ManifestEntry>,
487
488
  ): Promise<void> {
488
489
  const manifestEntries: BuildManifest["entries"] = {};
489
- for (const [buildId, entry] of Object.entries(entries)) {
490
+ for (const [sourceEntityId, entry] of Object.entries(entries)) {
490
491
  if (entry.physicalTableName) {
491
- manifestEntries[buildId] = { tableName: entry.physicalTableName };
492
+ manifestEntries[sourceEntityId] = {
493
+ tableName: entry.physicalTableName,
494
+ };
492
495
  }
493
496
  }
494
497
  try {
@@ -512,7 +515,7 @@ export class MaterializationService {
512
515
 
513
516
  /**
514
517
  * Validate caller-supplied build instructions against the package's compiled
515
- * build plan: every instructed buildId must be a planned source, and only
518
+ * build plan: every instructed sourceEntityId must be a planned source, and only
516
519
  * COPY realization is supported. Throws when the package declares no persist
517
520
  * source (no plan to build against).
518
521
  */
@@ -525,15 +528,15 @@ export class MaterializationService {
525
528
  "Package has no persist sources; buildInstructions cannot be applied",
526
529
  );
527
530
  }
528
- const plannedBuildIds = new Set<string>();
531
+ const plannedSourceEntityIds = new Set<string>();
529
532
  for (const source of Object.values(plan.sources)) {
530
- plannedBuildIds.add(source.buildId);
533
+ plannedSourceEntityIds.add(source.sourceEntityId);
531
534
  }
532
535
 
533
536
  for (const instruction of instructions) {
534
- if (!plannedBuildIds.has(instruction.buildId)) {
537
+ if (!plannedSourceEntityIds.has(instruction.sourceEntityId)) {
535
538
  throw new BadRequestError(
536
- `Instruction references unknown buildId '${instruction.buildId}'`,
539
+ `Instruction references unknown sourceEntityId '${instruction.sourceEntityId}'`,
537
540
  );
538
541
  }
539
542
  // COPY-only for now; SNAPSHOT lands once clone semantics are defined.
@@ -561,20 +564,20 @@ export class MaterializationService {
561
564
  const { graphs, sources, connectionDigests, connections } = compiled;
562
565
 
563
566
  // Index instructions by sourceID (the stable per-source handle) so the
564
- // build no longer recomputes the buildId to find an instruction.
565
- // Recomputing it here forced a caller's buildId to equal the publisher's
566
- // content hash, so a caller that derives buildIds by any other scheme
567
- // would have its sources silently skipped (the recomputed buildId would
568
- // not match the instruction). buildId is treated as opaque, caller-assigned
569
- // identity. A buildId index is kept as a fallback for instructions without
567
+ // build no longer recomputes the sourceEntityId to find an instruction.
568
+ // Recomputing it here forced a caller's sourceEntityId to equal the publisher's
569
+ // content hash, so a caller that derives sourceEntityIds by any other scheme
570
+ // would have its sources silently skipped (the recomputed sourceEntityId would
571
+ // not match the instruction). sourceEntityId is treated as opaque, caller-assigned
572
+ // identity. A sourceEntityId index is kept as a fallback for instructions without
570
573
  // a sourceID (e.g. standalone auto-run).
571
574
  const bySourceID = new Map<string, BuildInstruction>();
572
- const byBuildId = new Map<string, BuildInstruction>();
575
+ const bySourceEntityId = new Map<string, BuildInstruction>();
573
576
  for (const instruction of instructions) {
574
577
  if (instruction.sourceID) {
575
578
  bySourceID.set(instruction.sourceID, instruction);
576
579
  }
577
- byBuildId.set(instruction.buildId, instruction);
580
+ bySourceEntityId.set(instruction.sourceEntityId, instruction);
578
581
  }
579
582
 
580
583
  // Accumulates physical names as sources are built so downstream sources
@@ -582,11 +585,13 @@ export class MaterializationService {
582
585
  // it with carried-forward entries so reused upstreams resolve too.
583
586
  const manifest = new Manifest();
584
587
  const entries: Record<string, ManifestEntry> = {};
585
- for (const [buildId, entry] of Object.entries(seedEntries)) {
588
+ for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
586
589
  if (entry.physicalTableName) {
587
- manifest.update(buildId, { tableName: entry.physicalTableName });
590
+ manifest.update(sourceEntityId, {
591
+ tableName: entry.physicalTableName,
592
+ });
588
593
  }
589
- entries[buildId] = entry;
594
+ entries[sourceEntityId] = entry;
590
595
  }
591
596
 
592
597
  for (const graph of graphs) {
@@ -599,15 +604,19 @@ export class MaterializationService {
599
604
  for (const persistSource of iterGraphSources(graph, sources)) {
600
605
  if (signal.aborted) throw new Error("Build cancelled");
601
606
 
602
- // The manifest is keyed by the content buildId — what Malloy
607
+ // The manifest is keyed by the content sourceEntityId — what Malloy
603
608
  // recomputes to resolve upstream persist references during SQL
604
- // generation — independent of the instruction's identity buildId.
605
- const buildId = computeBuildId(persistSource, connectionDigests);
606
- // Prefer sourceID matching (so the caller's buildId scheme stays
607
- // opaque to the build); fall back to buildId for instructions
609
+ // generation — independent of the instruction's identity sourceEntityId.
610
+ const sourceEntityId = computeSourceEntityId(
611
+ persistSource,
612
+ connectionDigests,
613
+ );
614
+ // Prefer sourceID matching (so the caller's sourceEntityId scheme stays
615
+ // opaque to the build); fall back to sourceEntityId for instructions
608
616
  // without a sourceID (auto-run).
609
617
  const instruction =
610
- bySourceID.get(persistSource.sourceID) ?? byBuildId.get(buildId);
618
+ bySourceID.get(persistSource.sourceID) ??
619
+ bySourceEntityId.get(sourceEntityId);
611
620
  if (!instruction) continue;
612
621
 
613
622
  const entry = await this.buildOneSource(
@@ -617,7 +626,7 @@ export class MaterializationService {
617
626
  connectionDigests,
618
627
  manifest,
619
628
  );
620
- entries[buildId] = entry;
629
+ entries[sourceEntityId] = entry;
621
630
  }
622
631
  }
623
632
 
@@ -627,7 +636,7 @@ export class MaterializationService {
627
636
  /**
628
637
  * Build a single instructed source into its assigned physical table.
629
638
  * COPY uses a staging table + atomic rename for crash-safety; the staging
630
- * name derives from the buildId. Records and returns the manifest entry.
639
+ * name derives from the sourceEntityId. Records and returns the manifest entry.
631
640
  */
632
641
  private async buildOneSource(
633
642
  persistSource: PersistSource,
@@ -636,7 +645,7 @@ export class MaterializationService {
636
645
  connectionDigests: Record<string, string>,
637
646
  manifest: Manifest,
638
647
  ): Promise<ManifestEntry> {
639
- const buildId = instruction.buildId;
648
+ const sourceEntityId = instruction.sourceEntityId;
640
649
  const physicalTableName = instruction.physicalTableName;
641
650
  const buildSQL = persistSource.getSQL({
642
651
  buildManifest: manifest.buildManifest,
@@ -644,7 +653,7 @@ export class MaterializationService {
644
653
  });
645
654
 
646
655
  const bareName = bareTableName(physicalTableName);
647
- const stagingTableName = `${physicalTableName}${stagingSuffix(buildId)}`;
656
+ const stagingTableName = `${physicalTableName}${stagingSuffix(sourceEntityId)}`;
648
657
  // The control plane sends the logical (unquoted) physical name; dialect-
649
658
  // quote each identifier here so a container path or quote-requiring name
650
659
  // (e.g. a hyphenated BigQuery project id) produces valid DDL. The manifest
@@ -681,7 +690,7 @@ export class MaterializationService {
681
690
  }
682
691
 
683
692
  // Make this table visible to downstream sources built later in this run.
684
- manifest.update(buildId, { tableName: physicalTableName });
693
+ manifest.update(sourceEntityId, { tableName: physicalTableName });
685
694
 
686
695
  const durationMs = Math.round(performance.now() - startTime);
687
696
  recordSourceBuildDuration(durationMs);
@@ -691,7 +700,7 @@ export class MaterializationService {
691
700
  });
692
701
 
693
702
  return {
694
- buildId,
703
+ sourceEntityId,
695
704
  sourceName: persistSource.name,
696
705
  materializedTableId: instruction.materializedTableId,
697
706
  physicalTableName,
@@ -799,7 +808,7 @@ export class MaterializationService {
799
808
  if (!connectionName || !physicalTableName) {
800
809
  logger.warn("Skipping manifest entry with no connection/table", {
801
810
  materializationId: m.id,
802
- buildId: entry.buildId,
811
+ sourceEntityId: entry.sourceEntityId,
803
812
  });
804
813
  continue;
805
814
  }
@@ -825,7 +834,7 @@ export class MaterializationService {
825
834
  // table behind; clean it up too while we hold the connection.
826
835
  await connection.runSQL(
827
836
  `DROP TABLE IF EXISTS ${quoteTablePath(
828
- `${physicalTableName}${stagingSuffix(entry.buildId)}`,
837
+ `${physicalTableName}${stagingSuffix(entry.sourceEntityId)}`,
829
838
  dialect,
830
839
  )}`,
831
840
  );
@@ -48,7 +48,7 @@ export function makeBuildPlan(overrides: Partial<BuildPlan> = {}): BuildPlan {
48
48
  sourceID: "orders@m.malloy",
49
49
  connectionName: "duckdb",
50
50
  dialect: "duckdb",
51
- buildId: "build-orders",
51
+ sourceEntityId: "build-orders",
52
52
  sql: "SELECT 1",
53
53
  columns: [],
54
54
  },
@@ -62,7 +62,7 @@ export function makeInstruction(
62
62
  overrides: Partial<BuildInstruction> = {},
63
63
  ): BuildInstruction {
64
64
  return {
65
- buildId: "build-orders",
65
+ sourceEntityId: "build-orders",
66
66
  materializedTableId: "mt-1",
67
67
  physicalTableName: '"orders_v1"',
68
68
  realization: "COPY",
@@ -72,12 +72,12 @@ export function makeInstruction(
72
72
 
73
73
  /**
74
74
  * A minimal stand-in for a Malloy {@link PersistSource} exposing only what the
75
- * build internals touch (name/id, deterministic buildId, SQL, and the
75
+ * build internals touch (name/id, deterministic sourceEntityId, SQL, and the
76
76
  * `#@ persist name=` annotation reader, defaulted to "unset").
77
77
  */
78
78
  export function fakeSource(opts: {
79
79
  name: string;
80
- buildId: string;
80
+ sourceEntityId: string;
81
81
  sql?: string;
82
82
  connectionName?: string;
83
83
  dialectName?: string;
@@ -87,7 +87,7 @@ export function fakeSource(opts: {
87
87
  sourceID: opts.name,
88
88
  connectionName: opts.connectionName ?? "duckdb",
89
89
  dialectName: opts.dialectName ?? "duckdb",
90
- makeBuildId: () => opts.buildId,
90
+ makeBuildId: () => opts.sourceEntityId,
91
91
  getSQL: () => opts.sql ?? "SELECT 1",
92
92
  annotations: {
93
93
  parseAsTag: () => ({ tag: { text: () => undefined } }),
@@ -1931,7 +1931,7 @@ function makeHydrationRuntime(
1931
1931
  // (getSQL) time, gated on `prepareResultOptions.buildManifest`; without this
1932
1932
  // the hydrated model always recomputes from the base tables even though the
1933
1933
  // manifest was bound at load. `strict: false` keeps serving live for any
1934
- // source whose buildId is absent from the manifest.
1934
+ // source whose sourceEntityId is absent from the manifest.
1935
1935
  return new Runtime({
1936
1936
  urlReader,
1937
1937
  config,
@@ -61,7 +61,7 @@ export class Package {
61
61
  private packagePath: string;
62
62
  private malloyConfig: MalloyConfig;
63
63
  // Build-manifest binding state (Malloy Persistence v0). When bound, these
64
- // entries (buildId -> { tableName }) are what served queries use to route
64
+ // entries (sourceEntityId -> { tableName }) are what served queries use to route
65
65
  // persist sources to their materialized physical tables; they are also reused
66
66
  // by the /compile preview so previewed SQL matches executed SQL. Surfaced on
67
67
  // /status (via getPackageMetadata) so the control plane can confirm a worker
@@ -72,7 +72,7 @@ export class Package {
72
72
  private manifestEntryCount = 0;
73
73
  private boundManifestUri: string | null = null;
74
74
  // The package's persist build plan: a deterministic property of the compiled
75
- // package (per-source buildId, columns, build SQL, dependency graphs),
75
+ // package (per-source sourceEntityId, columns, build SQL, dependency graphs),
76
76
  // computed once at load from the live (unbound) models so it is stable for a
77
77
  // given (package version, connection config). Null when the package declares
78
78
  // no persist source. Surfaced read-only on getPackageMetadata() so a caller
@@ -460,7 +460,7 @@ export class Package {
460
460
  }
461
461
 
462
462
  /**
463
- * The package's persist build plan (per-source buildId, columns, build SQL,
463
+ * The package's persist build plan (per-source sourceEntityId, columns, build SQL,
464
464
  * dependency graphs), or null when the package declares no persist source.
465
465
  * A deterministic property of the compiled package; callers derive build
466
466
  * instructions from it for an orchestrated materialization.
@@ -503,7 +503,7 @@ export class Package {
503
503
  }
504
504
 
505
505
  /**
506
- * The currently-bound build-manifest entries (buildId -> { tableName }), or
506
+ * The currently-bound build-manifest entries (sourceEntityId -> { tableName }), or
507
507
  * undefined when the package is serving live. Reused by the /compile preview
508
508
  * so previewed SQL gets the same persist-source -> physical-table routing as
509
509
  * execution.
@@ -300,39 +300,69 @@ describe("openProxy — SSH tunnel", () => {
300
300
  },
301
301
  { host: "127.0.0.1", port: echoServer.port },
302
302
  ),
303
- ).rejects.toThrow(/host-key mismatch/i);
303
+ ).rejects.toThrow(/host-key verification failed/i);
304
304
  });
305
305
 
306
- it("rejects when hostKey is absent and opt-out env is not set", async () => {
307
- const orig = process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY;
308
- delete process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY;
309
-
306
+ it("connects when hostKey is absent unpinned self-service default", async () => {
307
+ // No hostKey → connect without host-key verification (the SSH transport is
308
+ // still encrypted). Matches mainstream BI tools' SSH-tunnel default.
309
+ const ep = await openProxy(
310
+ {
311
+ type: "ssh",
312
+ ssh: {
313
+ host: "127.0.0.1",
314
+ port: sshServer.port,
315
+ username: "testuser",
316
+ privateKey: clientPrivatePem,
317
+ },
318
+ },
319
+ { host: "127.0.0.1", port: echoServer.port },
320
+ );
310
321
  try {
311
- await expect(
312
- openProxy(
313
- {
314
- type: "ssh",
315
- ssh: {
316
- host: "127.0.0.1",
317
- port: sshServer.port,
318
- username: "testuser",
319
- privateKey: clientPrivatePem,
320
- },
321
- },
322
- { host: "127.0.0.1", port: echoServer.port },
323
- ),
324
- ).rejects.toThrow(/no hostKey provided/i);
322
+ const reply = await connectAndSend(ep.port, "unpinned");
323
+ expect(reply).toContain("unpinned");
325
324
  } finally {
326
- if (orig !== undefined) {
327
- process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY = orig;
328
- }
325
+ await closeQuietly(() => ep.close());
329
326
  }
330
327
  });
331
328
 
332
- it("connects when hostKey is absent and opt-out env is set", async () => {
333
- process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY = "true";
329
+ it("accepts a multi-line hostKey listing several keys matches any (LB bastion)", async () => {
330
+ // A load-balanced bastion presents a different host key per backend, so the
331
+ // pin lists every backend's key and any match is accepted.
332
+ const decoy = Buffer.alloc(32, 0xff).toString("base64");
333
+ const multiKey = [
334
+ `bastion.example.com ssh-ed25519 ${decoy}`,
335
+ `bastion.example.com ssh-rsa ${sshServer.hostKeyBase64}`,
336
+ ].join("\n");
337
+ const ep = await openProxy(
338
+ {
339
+ type: "ssh",
340
+ ssh: {
341
+ host: "127.0.0.1",
342
+ port: sshServer.port,
343
+ username: "testuser",
344
+ privateKey: clientPrivatePem,
345
+ hostKey: multiKey,
346
+ },
347
+ },
348
+ { host: "127.0.0.1", port: echoServer.port },
349
+ );
334
350
  try {
335
- const ep = await openProxy(
351
+ const reply = await connectAndSend(ep.port, "multi-key");
352
+ expect(reply).toContain("multi-key");
353
+ } finally {
354
+ await closeQuietly(() => ep.close());
355
+ }
356
+ });
357
+
358
+ it("fails closed when hostKey is set but parses to zero keys (comment/blank only)", async () => {
359
+ // A set-but-degenerate pin must NOT fall through to the unpinned branch —
360
+ // the security boundary is gated on hostKey being configured, not on it
361
+ // parsing to >=1 key. (Config load rejects this earlier; this guards the
362
+ // boundary itself.)
363
+ const commentOnly = "# ssh-keyscan bastion.example.com timed out\n \n";
364
+ await expect(
365
+ openProxy(
336
366
  {
337
367
  type: "ssh",
338
368
  ssh: {
@@ -340,19 +370,36 @@ describe("openProxy — SSH tunnel", () => {
340
370
  port: sshServer.port,
341
371
  username: "testuser",
342
372
  privateKey: clientPrivatePem,
373
+ hostKey: commentOnly,
343
374
  },
344
375
  },
345
376
  { host: "127.0.0.1", port: echoServer.port },
346
- );
347
- try {
348
- const reply = await connectAndSend(ep.port, "allow-unknown");
349
- expect(reply).toContain("allow-unknown");
350
- } finally {
351
- await closeQuietly(() => ep.close());
352
- }
353
- } finally {
354
- delete process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY;
355
- }
377
+ ),
378
+ ).rejects.toThrow(/host-key verification failed/i);
379
+ });
380
+
381
+ it("rejects a multi-line hostKey when none of the listed keys match", async () => {
382
+ const decoy1 = Buffer.alloc(32, 0xff).toString("base64");
383
+ const decoy2 = Buffer.alloc(32, 0xaa).toString("base64");
384
+ const multiKey = [
385
+ `bastion.example.com ssh-ed25519 ${decoy1}`,
386
+ `bastion.example.com ssh-rsa ${decoy2}`,
387
+ ].join("\n");
388
+ await expect(
389
+ openProxy(
390
+ {
391
+ type: "ssh",
392
+ ssh: {
393
+ host: "127.0.0.1",
394
+ port: sshServer.port,
395
+ username: "testuser",
396
+ privateKey: clientPrivatePem,
397
+ hostKey: multiKey,
398
+ },
399
+ },
400
+ { host: "127.0.0.1", port: echoServer.port },
401
+ ),
402
+ ).rejects.toThrow(/host-key verification failed/i);
356
403
  });
357
404
 
358
405
  it("rejects for unsupported proxy type", async () => {
@@ -8,24 +8,26 @@
8
8
  * A connection proxy is a normal connection capability (authorized by whoever
9
9
  * configures the connection); it is intentionally NOT behind an env-flag gate,
10
10
  * and is kept separate from the `publisher` HTTP multi-hop type's
11
- * PUBLISHER_ALLOW_PROXY_CONNECTIONS gate. The trust boundary is host-key pinning
12
- * (below), which is fail-closed.
11
+ * PUBLISHER_ALLOW_PROXY_CONNECTIONS gate. When host-key pinning is configured it
12
+ * is fail-closed (below).
13
13
  *
14
14
  * # Host-key policy
15
15
  *
16
- * Publisher is fail-closed by default: if `ssh.hostKey` is absent the tunnel
17
- * is refused at connect time to prevent MITM. Set the environment variable
18
- *
19
- * PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY=true
20
- *
21
- * to disable this check for development/testing. Never set this in
22
- * production.
16
+ * `ssh.hostKey` is optional. When set, it pins the bastion's host key(s) and the
17
+ * tunnel is fail-closed on mismatch; it may list multiple known_hosts lines (a
18
+ * load-balanced/HA bastion presents a different key per backend), and any listed
19
+ * key is accepted. When absent, the tunnel connects without host-key
20
+ * verification — the self-service default, matching mainstream BI tools' SSH
21
+ * tunnels. The SSH transport is still encrypted; unpinned, a MITM on the
22
+ * publisher→bastion hop is possible, mitigated by the customer allowlisting our
23
+ * egress on the bastion.
23
24
  */
24
25
 
25
26
  import net from "net";
26
27
  import { Client as SshClient } from "ssh2";
27
28
  import type { ConnectConfig, SyncHostVerifier } from "ssh2";
28
29
  import { components } from "../api";
30
+ import { logger } from "../logger";
29
31
 
30
32
  type ConnectionProxy = components["schemas"]["ConnectionProxy"];
31
33
 
@@ -39,17 +41,27 @@ const SSH_CONNECT_TIMEOUT_MS = 15_000;
39
41
  const SSH_KEEPALIVE_INTERVAL_MS = 15_000;
40
42
 
41
43
  /**
42
- * Extract the base64 wire-format key from a stored hostKey, accepting any of:
43
- * a full known_hosts line (`[markers] host keytype AAAA…`), a `keytype AAAA…`
44
- * pair, or the bare base64 blob. SSH public-key blobs always base64-encode to a
45
- * string starting with "AAAA" (the 4-byte length prefix of the key-type name),
46
- * so we pick that token; otherwise fall back to the last whitespace-delimited
47
- * token. This matches what ssh2's hostVerifier hands us (`key.toString("base64")`).
44
+ * Parse a stored hostKey into the set of base64 wire-format key blobs to accept.
45
+ * The value may hold one or more OpenSSH known_hosts entries (one per line) a
46
+ * load-balanced bastion presents a different host key per backend, so pinning
47
+ * such a bastion means listing every backend's key. Blank lines and `#` comments
48
+ * are skipped. Each entry may be a full known_hosts line (`[markers] host keytype
49
+ * AAAA…`), a `keytype AAAA…` pair, or a bare base64 blob; SSH key blobs
50
+ * base64-encode to a string starting with "AAAA" (the length prefix of the
51
+ * key-type name), so we pick that token, else the last whitespace-delimited
52
+ * token. Compared against what ssh2 hands us (`key.toString("base64")`).
48
53
  */
49
- function extractHostKeyBase64(raw: string): string {
50
- const tokens = raw.trim().split(/\s+/);
51
- const blob = tokens.find((t) => /^AAAA[A-Za-z0-9+/]+={0,2}$/.test(t));
52
- return blob ?? tokens[tokens.length - 1] ?? "";
54
+ export function parseHostKeys(raw: string): Set<string> {
55
+ const keys = new Set<string>();
56
+ for (const line of raw.split(/\r?\n/)) {
57
+ const trimmed = line.trim();
58
+ if (!trimmed || trimmed.startsWith("#")) continue;
59
+ const tokens = trimmed.split(/\s+/);
60
+ const blob = tokens.find((t) => /^AAAA[A-Za-z0-9+/]+={0,2}$/.test(t));
61
+ const key = blob ?? tokens[tokens.length - 1];
62
+ if (key) keys.add(key);
63
+ }
64
+ return keys;
53
65
  }
54
66
 
55
67
  /**
@@ -112,31 +124,34 @@ function openSshProxy(
112
124
  hostVerifier: ((key: Buffer): boolean => {
113
125
  // `key` is the raw Buffer of the host's public key.
114
126
  const presented = key.toString("base64");
127
+ // "Pinned" is gated on hostKey being a non-empty value, not on it
128
+ // parsing to ≥1 key: a set-but-degenerate hostKey (only whitespace,
129
+ // blanks, or comments) must fail closed, never fall through to the
130
+ // unpinned branch. Config load (validateConnectionShape) already
131
+ // rejects that case; this is the security-boundary backstop for any
132
+ // path that reaches here directly. ("" is the unpinned signal.)
115
133
  if (ssh.hostKey) {
116
- const expected = extractHostKeyBase64(ssh.hostKey);
117
- if (presented !== expected) {
118
- fail(
119
- new Error(
120
- `SSH host-key mismatch for ${ssh.host}: expected ${expected.slice(0, 24)}… but got ${presented.slice(0, 24)}….`,
121
- ),
122
- );
123
- return false;
134
+ // Fail-closed, accepting any listed key — an LB/HA bastion presents
135
+ // a different host key per backend.
136
+ const pinned = parseHostKeys(ssh.hostKey);
137
+ if (pinned.has(presented)) {
138
+ return true;
124
139
  }
125
- return true;
126
- }
127
- // No hostKey provided.
128
- if (process.env.PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY === "true") {
129
- return true;
140
+ fail(
141
+ new Error(
142
+ `SSH host-key verification failed for ${ssh.host}: the presented key ` +
143
+ `(${presented.slice(0, 24)}…) is not among the ${pinned.size} pinned ` +
144
+ `host key(s).`,
145
+ ),
146
+ );
147
+ return false;
130
148
  }
131
- fail(
132
- new Error(
133
- `SSH connection to ${ssh.host} refused: no hostKey provided and ` +
134
- `PUBLISHER_SSH_ALLOW_UNKNOWN_HOSTKEY is not 'true'. ` +
135
- `Set hostKey to the bastion's host public key (an OpenSSH ` +
136
- `known_hosts line or its base64 blob, e.g. from ssh-keyscan).`,
137
- ),
149
+ // Unpinned (no hostKey): connect without host-key verification — the
150
+ // self-service default (see file header).
151
+ logger.warn(
152
+ `Connecting to SSH bastion ${ssh.host} without host-key verification (no hostKey pinned).`,
138
153
  );
139
- return false;
154
+ return true;
140
155
  }) as SyncHostVerifier,
141
156
  };
142
157