@malloy-publisher/server 0.0.230 → 0.0.232

package/src/service/environment_store_clone.spec.ts

@@ -0,0 +1,350 @@
+import {
+   afterEach,
+   beforeEach,
+   describe,
+   expect,
+   it,
+   mock,
+   spyOn,
+} from "bun:test";
+import { existsSync, mkdirSync, rmSync, writeFileSync } from "fs";
+import * as path from "path";
+import { TEMP_DIR_PATH } from "../constants";
+import { logger } from "../logger";
+
+/**
+ * Wiring coverage for the package git clone: the shallow-clone options must
+ * actually reach `simple-git`'s clone call, the factory must carry a progress
+ * handler whose output lands on stderr with the right label, and the readiness
+ * line must not print when boot fails. The pure helpers (options content,
+ * throttling, label text) are covered in environment_store.spec.ts; this file
+ * proves the plumbing.
+ *
+ * simple-git is module-mocked BEFORE the store is imported (the anchoring
+ * spec's ordering idiom). The mock leaks forward in bun's shared module cache,
+ * which is safe here: environment_store.ts is the only importer, and the only
+ * spec that performs a real clone (connection.spec.ts) is credential-gated and
+ * runs in its own workflow process.
+ */
+
+const serverRootPath = path.join(TEMP_DIR_PATH, "clone-spec-server-root");
+
+const ENV_NAME = "clone-env";
+const PKG_NAME = "pkg-a";
+
+interface RecordedClone {
+   repoUrl: string;
+   dir: string;
+   opts: Record<string, unknown>;
+}
+
+let recordedClones: RecordedClone[] = [];
+let lastFactoryOpts: {
+   progress?: (event: {
+      method: string;
+      stage: string;
+      progress: number;
+      processed: number;
+      total: number;
+   }) => void;
+} | null = null;
+let storageInitFails = false;
+let storageInitMessage = "storage init failed (test)";
+let cloneFailure: Error | null = null;
+
+mock.module("simple-git", () => ({
+   default: (factoryOpts?: typeof lastFactoryOpts) => {
+      lastFactoryOpts = factoryOpts ?? null;
+      return {
+         clone: (
+            repoUrl: string,
+            dir: string,
+            opts: Record<string, unknown>,
+            cb: (err: Error | null) => void,
+         ) => {
+            recordedClones.push({ repoUrl, dir, opts });
+            if (cloneFailure) {
+               cb(cloneFailure);
+               return;
+            }
+            // Materialize the fixture the extraction step expects: the repo
+            // contains a subdirectory per package (pkg-a plus a sibling used
+            // by the shared-clone counter test). Extraction only copies the
+            // named subdir, so an unused one is harmless to single-package
+            // tests.
+            for (const name of [PKG_NAME, "pkg-b"]) {
+               const pkgDir = path.join(dir, name);
+               mkdirSync(pkgDir, { recursive: true });
+               writeFileSync(
+                  path.join(pkgDir, "publisher.json"),
+                  JSON.stringify({ name }),
+               );
+            }
+            // Drive the progress handler the way the real plugin would.
+            lastFactoryOpts?.progress?.({
+               method: "clone",
+               stage: "receiving",
+               progress: 50,
+               processed: 1,
+               total: 2,
+            });
+            cb(null);
+         },
+      };
+   },
+}));
+
+mock.module("../storage/StorageManager", () => ({
+   StorageManager: class MockStorageManager {
+      async initialize(): Promise<void> {
+         if (storageInitFails) {
+            throw new Error(storageInitMessage);
+         }
+      }
+      getRepository() {
+         return {
+            listEnvironments: async () => [],
+            getEnvironmentByName: async () => null,
+            createEnvironment: async (data: Record<string, unknown>) => ({
+               id: "env-id",
+               name: data.name,
+               path: data.path,
+            }),
+            listPackages: async () => [],
+            getPackageByName: async () => null,
+            createPackage: async (data: Record<string, unknown>) => ({
+               id: "pkg-id",
+               name: data.name,
+            }),
+            listConnections: async () => [],
+         };
+      }
+   },
+   StorageConfig: {} as Record<string, unknown>,
+}));
+
+const { EnvironmentStore, GIT_CLONE_OPTIONS } = await import(
+   "./environment_store"
+);
+
+describe("git clone wiring", () => {
+   let stderrWrites: string[];
+   let stderrSpy: ReturnType<typeof spyOn>;
+
+   beforeEach(() => {
+      rmSync(serverRootPath, { recursive: true, force: true });
+      mkdirSync(serverRootPath, { recursive: true });
+      recordedClones = [];
+      lastFactoryOpts = null;
+      storageInitFails = false;
+      storageInitMessage = "storage init failed (test)";
+      cloneFailure = null;
+      stderrWrites = [];
+      stderrSpy = spyOn(process.stderr, "write").mockImplementation(
+         (chunk: string | Uint8Array) => {
+            stderrWrites.push(String(chunk));
+            return true;
+         },
+      );
+   });
+
+   afterEach(() => {
+      stderrSpy.mockRestore();
+      rmSync(serverRootPath, { recursive: true, force: true });
+   });
+
+   it("boot clones shallow, streams labeled progress, and mounts the package", async () => {
+      writeFileSync(
+         path.join(serverRootPath, "publisher.config.json"),
+         JSON.stringify({
+            frozenConfig: false,
+            environments: [
+               {
+                  name: ENV_NAME,
+                  packages: [
+                     {
+                        name: PKG_NAME,
+                        location: `https://github.com/example/repo/tree/main/${PKG_NAME}`,
+                     },
+                  ],
+                  connections: [],
+               },
+            ],
+         }),
+      );
+
+      const infoSpy = spyOn(logger, "info");
+      const store = new EnvironmentStore(serverRootPath);
+      await store.finishedInitialization;
+
+      // The options object reaches .clone verbatim.
+      expect(recordedClones).toHaveLength(1);
+      expect(recordedClones[0].repoUrl).toBe("https://github.com/example/repo");
+      expect(recordedClones[0].opts).toEqual(GIT_CLONE_OPTIONS);
+
+      // The extraction log carries the mounted-of-total counter.
+      const infoMessages = infoSpy.mock.calls.map((c) => String(c[0]));
+      infoSpy.mockRestore();
+      expect(
+         infoMessages.some(
+            (m) =>
+               m.includes('Extracted package "pkg-a"') && m.includes("(1/1)"),
+         ),
+      ).toBe(true);
+
+      // The progress event surfaced on stderr, labeled with env and package.
+      const output = stderrWrites.join("");
+      expect(output).toContain(
+         `[${ENV_NAME}] cloning example/repo (${PKG_NAME}): receiving 50% (1/2)`,
+      );
+
+      // The package extracted from the (fake) clone and mounted.
+      expect(
+         existsSync(
+            path.join(
+               serverRootPath,
+               "publisher_data",
+               ENV_NAME,
+               PKG_NAME,
+               "publisher.json",
+            ),
+         ),
+      ).toBe(true);
+
+      // And the boot announced itself exactly once.
+      const readyLines = output
+         .split("\n")
+         .filter((line) => line.startsWith("PUBLISHER_READY"));
+      expect(readyLines).toHaveLength(1);
+      expect(readyLines[0]).toContain("environments=1");
+      expect(readyLines[0]).toContain("packages=1");
+      expect(readyLines[0]).toContain("load_errors=0");
+   });
+
+   it("the mounted-of-total counter accumulates across packages in a shared clone", async () => {
+      // Two packages share one repo clone; the counter is one variable across
+      // every mount branch, so a single (1/2)->(2/2) run pins that it advances
+      // rather than reporting a constant.
+      writeFileSync(
+         path.join(serverRootPath, "publisher.config.json"),
+         JSON.stringify({
+            frozenConfig: false,
+            environments: [
+               {
+                  name: ENV_NAME,
+                  packages: [
+                     {
+                        name: PKG_NAME,
+                        location: `https://github.com/example/repo/tree/main/${PKG_NAME}`,
+                     },
+                     {
+                        name: "pkg-b",
+                        location: `https://github.com/example/repo/tree/main/pkg-b`,
+                     },
+                  ],
+                  connections: [],
+               },
+            ],
+         }),
+      );
+
+      const infoSpy = spyOn(logger, "info");
+      const store = new EnvironmentStore(serverRootPath);
+      await store.finishedInitialization;
+      const infoMessages = infoSpy.mock.calls.map((c) => String(c[0]));
+      infoSpy.mockRestore();
+
+      // One clone serves both, and the counter runs 1/2 then 2/2.
+      expect(recordedClones).toHaveLength(1);
+      const counters = infoMessages
+         .filter((m) => m.startsWith("Extracted package"))
+         .map((m) => m.slice(m.lastIndexOf("(")));
+      expect(counters).toEqual(["(1/2)", "(2/2)"]);
+      expect(stderrWrites.join("")).toContain("packages=2");
+   });
+
+   it("direct download extracts the subdirectory and labels by repo alone", async () => {
+      // The controller add-package path: downloadGitHubDirectory is called
+      // with the raw /tree/... location and no progress context.
+      const store = new EnvironmentStore(serverRootPath);
+      await store.finishedInitialization;
+
+      const target = path.join(serverRootPath, "publisher_data", "direct");
+      await store.downloadGitHubDirectory(
+         `https://github.com/example/repo/tree/main/${PKG_NAME}`,
+         target,
+      );
+
+      expect(recordedClones).toHaveLength(1);
+      expect(recordedClones[0].opts).toEqual(GIT_CLONE_OPTIONS);
+      // Subdirectory contents were hoisted to the target root.
+      expect(existsSync(path.join(target, "publisher.json"))).toBe(true);
+      expect(existsSync(path.join(target, PKG_NAME))).toBe(false);
+      // No environment context on this path, so the label is the repo alone.
+      expect(stderrWrites.join("")).toContain(
+         "cloning example/repo: receiving 50% (1/2)",
+      );
+   });
+
+   it("a failed clone rejects with the progress noise stripped", async () => {
+      const store = new EnvironmentStore(serverRootPath);
+      await store.finishedInitialization;
+      cloneFailure = new Error(
+         "Cloning into '/x'...\nReceiving objects: 50% (5/10)\nfatal: early EOF",
+      );
+
+      const target = path.join(serverRootPath, "publisher_data", "direct");
+      let caught: Error | undefined;
+      try {
+         await store.downloadGitHubDirectory(
+            "https://github.com/example/repo",
+            target,
+         );
+      } catch (e) {
+         caught = e as Error;
+      }
+      expect(caught).toBeDefined();
+      // The message keeps what went wrong and drops the --progress spew;
+      // the stack (which V8 seeds with the message, prefixed "Error: ") is
+      // stripped too, including the prefixed "Cloning into" first line.
+      expect(caught!.message).toBe("fatal: early EOF");
+      expect(caught!.stack ?? "").not.toContain("Receiving objects");
+      expect(caught!.stack ?? "").not.toContain("Cloning into");
+   });
+
+   it("prints the failure token instead of the readiness line when boot fails", async () => {
+      storageInitFails = true;
+      const store = new EnvironmentStore(serverRootPath);
+      // initialize() swallows the failure by design; the promise resolves.
+      await store.finishedInitialization;
+      const output = stderrWrites.join("");
+      expect(output.includes("PUBLISHER_READY")).toBe(false);
+      // A script waiting on the ready token fails fast instead of hanging.
+      const failLines = output
+         .split("\n")
+         .filter((line) => line.startsWith("PUBLISHER_INIT_FAILED"));
+      expect(failLines).toHaveLength(1);
+      expect(failLines[0]).toContain("storage init failed (test)");
+   });
+
+   it("redacts a pg password in the failure token", async () => {
+      // The init error can carry a connection string; the INIT_FAILED token
+      // must redact the keyword-form password before it reaches stderr.
+      // (The adjacent winston init-error log shares the raw-message gap with
+      // every other extractErrorDataFromError site; redacting those centrally
+      // is a separate security follow-up, tracked in npx-fast-first-boot.md.)
+      storageInitFails = true;
+      storageInitMessage =
+         "attach failed: host=db port=5432 password=supersecret dbname=x";
+      const store = new EnvironmentStore(serverRootPath);
+      await store.finishedInitialization;
+
+      const failLine = stderrWrites
+         .join("")
+         .split("\n")
+         .find((line) => line.startsWith("PUBLISHER_INIT_FAILED"));
+      expect(failLine).toBeDefined();
+      expect(failLine).toContain("password=***");
+      expect(failLine).not.toContain("supersecret");
+   });
+});

package/src/service/manifest_loader.spec.ts

@@ -24,7 +24,7 @@ describe("fetchManifestEntries", () => {
       return file;
    }
 
-   it("maps physicalTableName to the Malloy runtime tableName", async () => {
+   it("maps physicalTableName to the Malloy runtime tableName (colocated entries)", async () => {
       const file = await writeManifest({
          builtAt: new Date().toISOString(),
          strict: false,
@@ -38,17 +38,64 @@ describe("fetchManifestEntries", () => {
          },
       });
 
-      const entries = await fetchManifestEntries(file);
+      const { tableNameManifest, storageEntries } =
+         await fetchManifestEntries(file);
 
-      expect(entries).toEqual({
+      expect(tableNameManifest).toEqual({
          b1: { tableName: "schema.orders_mz" },
          // connectionName is carried through so the bind step can quote the
          // table path for that connection's dialect (Package.quoteBoundTableNames).
          b2: { tableName: "schema.daily_mz", connectionName: "bq" },
       });
+      expect(storageEntries).toEqual({});
    });
 
-   it("carries the control-plane freshness fields verbatim", async () => {
+   it("routes storage= entries (with storageConnectionName) to storageEntries, not the tableName manifest", async () => {
+      const schema = [
+         { name: "d", type: "DATE" },
+         { name: "n", type: "BIGINT" },
+      ];
+      const file = await writeManifest({
+         entries: {
+            // colocated: in-warehouse substitution
+            wh: {
+               sourceEntityId: "wh",
+               physicalTableName: "orders_v1",
+               connectionName: "bq",
+            },
+            // storage tier: cross-connection serve binding (full entry retained)
+            lake: {
+               sourceEntityId: "lake",
+               sourceName: "daily_orders",
+               physicalTableName: "daily_orders",
+               connectionName: "bq",
+               storageConnectionName: "lake",
+               schema,
+            },
+         },
+      });
+
+      const { tableNameManifest, storageEntries } =
+         await fetchManifestEntries(file);
+
+      // The storage entry must NOT enter the same-connection tableName manifest.
+      expect(tableNameManifest).toEqual({
+         wh: { tableName: "orders_v1", connectionName: "bq" },
+      });
+      // …and is preserved as a full entry (schema + sourceName + destination).
+      expect(storageEntries).toEqual({
+         lake: {
+            sourceEntityId: "lake",
+            sourceName: "daily_orders",
+            physicalTableName: "daily_orders",
+            connectionName: "bq",
+            storageConnectionName: "lake",
+            schema,
+         },
+      });
+   });
+
+   it("carries the host-supplied freshness fields verbatim (colocated)", async () => {
       const dataAsOf = "2026-07-07T00:00:00.000Z";
       const file = await writeManifest({
          entries: {
@@ -66,11 +113,11 @@ describe("fetchManifestEntries", () => {
          },
       });
 
-      const entries = await fetchManifestEntries(file);
+      const { tableNameManifest } = await fetchManifestEntries(file);
 
       // Filter-free: freshness fields are retained for the serve-path gate; a
       // window-less entry is left un-gated.
-      expect(entries).toEqual({
+      expect(tableNameManifest).toEqual({
          gated: {
             tableName: "schema.gated_mz",
             dataAsOf,
@@ -86,9 +133,11 @@ describe("fetchManifestEntries", () => {
          entries: { b1: { sourceEntityId: "b1", physicalTableName: "t1" } },
       });
 
-      const entries = await fetchManifestEntries(pathToFileURL(file).href);
+      const { tableNameManifest } = await fetchManifestEntries(
+         pathToFileURL(file).href,
+      );
 
-      expect(entries).toEqual({ b1: { tableName: "t1" } });
+      expect(tableNameManifest).toEqual({ b1: { tableName: "t1" } });
    });
 
    it("skips entries without a physicalTableName", async () => {
@@ -99,17 +148,23 @@ describe("fetchManifestEntries", () => {
          },
       });
 
-      const entries = await fetchManifestEntries(file);
+      const { tableNameManifest } = await fetchManifestEntries(file);
 
-      expect(entries).toEqual({ good: { tableName: "t1" } });
+      expect(tableNameManifest).toEqual({ good: { tableName: "t1" } });
    });
 
-   it("returns an empty map when there are no entries", async () => {
+   it("returns empty maps when there are no entries", async () => {
       const file = await writeManifest({ entries: {} });
-      expect(await fetchManifestEntries(file)).toEqual({});
+      expect(await fetchManifestEntries(file)).toEqual({
+         tableNameManifest: {},
+         storageEntries: {},
+      });
 
       const file2 = await writeManifest({});
-      expect(await fetchManifestEntries(file2)).toEqual({});
+      expect(await fetchManifestEntries(file2)).toEqual({
+         tableNameManifest: {},
+         storageEntries: {},
+      });
    });
 
    it("throws on malformed JSON", async () => {

package/src/service/manifest_loader.ts

@@ -4,10 +4,29 @@ import * as fs from "fs/promises";
 import { fileURLToPath } from "url";
 import { components } from "../api";
 import { logger } from "../logger";
-import { FreshnessManifest } from "../storage/DatabaseInterface";
+import { FreshnessManifest, ManifestEntry } from "../storage/DatabaseInterface";
 
 type WireBuildManifest = components["schemas"]["BuildManifest"];
 
+/**
+ * A fetched build manifest, split by tier. A storage-materialized entry (one
+ * that carries `storageConnectionName`) is served cross-connection via the
+ * virtual-source transform, so it becomes a serve BINDING — it must never enter
+ * the same-connection `tableName` substitution. Everything else is an
+ * colocated entry the Malloy runtime resolves by substituting its
+ * `tableName` at compile time.
+ */
+export interface FetchedManifest {
+   /** colocated entries: same-connection `tableName` substitution. */
+   tableNameManifest: FreshnessManifest;
+   /**
+    * `storage=` entries keyed by sourceEntityId, carried as full
+    * {@link ManifestEntry}s (with `storageConnectionName` + captured `schema` +
+    * `sourceName`) so the bind step can derive cross-connection serve bindings.
+    */
+   storageEntries: Record<string, ManifestEntry>;
+}
+
 // Lazily-created clients reused across fetches. Both use the ambient credential
 // chain (ADC for GCS; the default provider chain for S3) — the same auth the
 // package downloader already relies on.
@@ -57,22 +76,26 @@ async function readManifestBytes(uri: string): Promise<string> {
 }
 
 /**
- * Fetch and parse the control-plane-computed build manifest at `uri`, returning
- * the full wire binding map (`sourceEntityId -> { tableName, connectionName,
- * dataAsOf, freshnessWindowSeconds, freshnessFallback }`). The wire manifest
- * keys physical tables under `physicalTableName`; the Malloy runtime consumes
- * `tableName`, so we translate that field here but otherwise carry the
- * control-plane fields verbatim — the freshness fields so the serve path can
- * gate `age vs window` per query, and `connectionName` so the bind step can
- * quote the path for that connection's dialect (see
- * Package.quoteBoundTableNames). Entries
- * missing a physical table are skipped. This is a pure fetch + parse: it does
- * **not** filter on freshness (that happens per query on the serve path). Throws
- * if the URI can't be read or parsed.
+ * Fetch and parse the host-computed build manifest at `uri`, split by
+ * tier into {@link FetchedManifest}. The wire manifest keys physical tables
+ * under `physicalTableName`; for an colocated entry the Malloy
+ * runtime consumes `tableName`, so that field is translated here and the entry
+ * lands in `tableNameManifest`, carrying the freshness fields verbatim (so the
+ * serve path can gate `age vs window` per query) and `connectionName` (so the
+ * bind step can quote the path for that connection's dialect — see
+ * Package.quoteBoundTableNames).
+ *
+ * A `storage=`-materialized entry (one carrying `storageConnectionName`) is
+ * routed instead to `storageEntries` as its full {@link ManifestEntry}: it lives
+ * on a DIFFERENT connection and is served through the virtual-source transform
+ * from its captured `schema`, so it must never become a same-connection
+ * `tableName` substitution. Entries missing a physical table are skipped. This
+ * is a pure fetch + parse: it does **not** filter on freshness (that happens per
+ * query on the serve path). Throws if the URI can't be read or parsed.
  */
 export async function fetchManifestEntries(
    uri: string,
-): Promise<FreshnessManifest> {
+): Promise<FetchedManifest> {
    const raw = await readManifestBytes(uri);
 
    let parsed: WireBuildManifest;
@@ -86,17 +109,42 @@ export async function fetchManifestEntries(
       );
    }
 
-   const entries: FreshnessManifest = {};
-   for (const [sourceEntityId, entry] of Object.entries(parsed.entries ?? {})) {
+   return splitManifestEntries(parsed.entries ?? {}, uri);
+}
+
+/**
+ * Split an already-in-hand manifest entry map by tier into
+ * {@link FetchedManifest}, applying the same wire→runtime translation as
+ * {@link fetchManifestEntries} (physicalTableName → colocated `tableName`
+ * substitution carrying freshness + connectionName; a `storageConnectionName`
+ * entry stays a full {@link ManifestEntry} for the virtual-source transform).
+ * Pure: no I/O, no freshness filtering. Shared by the URI-fetch path (host
+ * manifest) and the local-store rebind (a package's own latest persisted
+ * materialization, re-applied on load). `source` is a label for the skip log.
+ */
+export function splitManifestEntries(
+   entries: Record<string, ManifestEntry>,
+   source: string,
+): FetchedManifest {
+   const tableNameManifest: FreshnessManifest = {};
+   const storageEntries: Record<string, ManifestEntry> = {};
+   for (const [sourceEntityId, entry] of Object.entries(entries)) {
       const physicalTableName = entry?.physicalTableName;
       if (!physicalTableName) {
          logger.warn("Manifest entry has no physicalTableName; skipping", {
-            uri,
+            source,
             sourceEntityId,
          });
          continue;
       }
-      entries[sourceEntityId] = {
+      if (entry.storageConnectionName) {
+         // Cross-connection storage tier: keep the full entry (schema +
+         // sourceName + storageConnectionName) for the serve-binding derivation;
+         // never enter the same-connection tableName manifest.
+         storageEntries[sourceEntityId] = entry;
+         continue;
+      }
+      tableNameManifest[sourceEntityId] = {
          tableName: physicalTableName,
          connectionName: entry.connectionName,
          dataAsOf: entry.dataAsOf,
@@ -104,5 +152,5 @@ export async function fetchManifestEntries(
          freshnessFallback: entry.freshnessFallback,
       };
    }
-   return entries;
+   return { tableNameManifest, storageEntries };
 }