@malloy-publisher/server 0.0.235 → 0.0.236

package/src/service/package_manifest.spec.ts

@@ -1,230 +0,0 @@
-import { describe, expect, it } from "bun:test";
-import {
-   packageMaterializationWarnings,
-   parsePackageMaterialization,
-   parsePackageScope,
-   resolvePackageScope,
-} from "./package_manifest";
-
-describe("service/package_manifest", () => {
-   describe("resolvePackageScope", () => {
-      it("reads the canonical materialization.scope with no warning", () => {
-         expect(resolvePackageScope(undefined, { scope: "version" })).toEqual({
-            scope: "version",
-            warnings: [],
-         });
-      });
-
-      it("defaults to 'package' when neither home declares it", () => {
-         expect(resolvePackageScope(undefined, undefined)).toEqual({
-            scope: "package",
-            warnings: [],
-         });
-         expect(resolvePackageScope(undefined, { freshness: {} })).toEqual({
-            scope: "package",
-            warnings: [],
-         });
-      });
-
-      it("still honors a root scope, with a deprecation warning", () => {
-         const resolved = resolvePackageScope("version", undefined);
-         expect(resolved.scope).toBe("version");
-         expect(resolved.warnings).toHaveLength(1);
-         expect(resolved.warnings[0]).toMatch(/deprecated/i);
-      });
-
-      it("accepts both homes agreeing without a warning", () => {
-         // The transition state the server writes: envelope for this build, root
-         // for an older publisher. Warning would blame the operator for it.
-         expect(resolvePackageScope("version", { scope: "version" })).toEqual({
-            scope: "version",
-            warnings: [],
-         });
-      });
-
-      it("throws when the two homes disagree rather than guessing", () => {
-         expect(() =>
-            resolvePackageScope("package", { scope: "version" }),
-         ).toThrow(/conflicting/i);
-      });
-
-      it("reports an invalid value as invalid, not as a conflict", () => {
-         expect(() =>
-            resolvePackageScope(undefined, { scope: "shared" }),
-         ).toThrow(/expected "version" or "package"/i);
-         expect(() =>
-            resolvePackageScope("shared", { scope: "version" }),
-         ).toThrow(/expected "version" or "package"/i);
-      });
-   });
-
-   describe("parsePackageScope", () => {
-      it("defaults to 'package' when absent", () => {
-         expect(parsePackageScope(undefined)).toBe("package");
-         expect(parsePackageScope(null)).toBe("package");
-      });
-
-      it("accepts the two valid modes verbatim", () => {
-         expect(parsePackageScope("version")).toBe("version");
-         expect(parsePackageScope("package")).toBe("package");
-      });
-
-      it("throws on any other value (no silent default)", () => {
-         expect(() => parsePackageScope("shared")).toThrow(/scope/i);
-         expect(() => parsePackageScope("")).toThrow(/scope/i);
-         expect(() => parsePackageScope(7)).toThrow(/scope/i);
-      });
-   });
-
-   describe("parsePackageMaterialization", () => {
-      it("extracts a string schedule", () => {
-         expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
-            { schedule: "0 6 * * *", freshness: null, queryMetadata: null },
-         );
-      });
-
-      it("returns null when the block is absent", () => {
-         expect(parsePackageMaterialization(undefined)).toBeNull();
-         expect(parsePackageMaterialization(null)).toBeNull();
-      });
-
-      it("ignores extra/unknown fields", () => {
-         expect(
-            parsePackageMaterialization({
-               schedule: "*/15 * * * *",
-               retries: 3,
-            }),
-         ).toEqual({
-            schedule: "*/15 * * * *",
-            freshness: null,
-            queryMetadata: null,
-         });
-      });
-
-      it("degrades a non-string schedule to null", () => {
-         expect(parsePackageMaterialization({ schedule: 42 })).toEqual({
-            schedule: null,
-            freshness: null,
-            queryMetadata: null,
-         });
-         expect(parsePackageMaterialization({})).toEqual({
-            schedule: null,
-            freshness: null,
-            queryMetadata: null,
-         });
-      });
-
-      it("returns null for non-object input", () => {
-         expect(parsePackageMaterialization("0 6 * * *")).toBeNull();
-         expect(parsePackageMaterialization(7)).toBeNull();
-      });
-
-      it("extracts a full freshness block verbatim", () => {
-         expect(
-            parsePackageMaterialization({
-               freshness: { window: "24h", fallback: "stale_ok" },
-            }),
-         ).toEqual({
-            schedule: null,
-            freshness: { window: "24h", fallback: "stale_ok" },
-            queryMetadata: null,
-         });
-      });
-
-      it("keeps a partial freshness block (window only)", () => {
-         expect(
-            parsePackageMaterialization({ freshness: { window: "1h" } }),
-         ).toEqual({
-            schedule: null,
-            freshness: { window: "1h" },
-            queryMetadata: null,
-         });
-      });
-
-      it("drops invalid freshness fields rather than defaulting them", () => {
-         // A bad value is reported as absent — never substituted — so the
-         // control plane sees exactly what was (validly) declared.
-         expect(
-            parsePackageMaterialization({
-               freshness: { window: 24, fallback: "retry" },
-            }),
-         ).toEqual({ schedule: null, freshness: {}, queryMetadata: null });
-      });
-
-      it("degrades a non-object freshness to null", () => {
-         expect(parsePackageMaterialization({ freshness: "24h" })).toEqual({
-            schedule: null,
-            freshness: null,
-            queryMetadata: null,
-         });
-      });
-
-      it("keeps queryMetadata properties verbatim", () => {
-         expect(
-            parsePackageMaterialization({
-               queryMetadata: { team: "finance", workload: "marts" },
-            }),
-         ).toEqual({
-            schedule: null,
-            freshness: null,
-            queryMetadata: { team: "finance", workload: "marts" },
-         });
-      });
-
-      it("keeps a contract-violating property so publish can report it", () => {
-         // Filtering here would make the author's typo disappear with nothing
-         // to point at; the validator warns and the runtime clamps instead.
-         expect(
-            parsePackageMaterialization({
-               queryMetadata: { "team.name": "finance" },
-            }),
-         ).toEqual({
-            schedule: null,
-            freshness: null,
-            queryMetadata: { "team.name": "finance" },
-         });
-      });
-
-      it("drops non-string values and empties to null", () => {
-         expect(
-            parsePackageMaterialization({
-               queryMetadata: { team: "finance", retries: 3 },
-            }),
-         ).toEqual({
-            schedule: null,
-            freshness: null,
-            queryMetadata: { team: "finance" },
-         });
-         expect(
-            parsePackageMaterialization({ queryMetadata: { retries: 3 } }),
-         ).toEqual({ schedule: null, freshness: null, queryMetadata: null });
-         expect(
-            parsePackageMaterialization({ queryMetadata: "team=finance" }),
-         ).toEqual({ schedule: null, freshness: null, queryMetadata: null });
-      });
-   });
-
-   describe("packageMaterializationWarnings", () => {
-      it("names a dropped non-string value", () => {
-         // The drop happens before the config validation that would otherwise
-         // report it, so without this an unquoted `"team": 123` — a plausible
-         // hand-edit — vanishes with nothing anywhere to point at.
-         const warnings = packageMaterializationWarnings({
-            queryMetadata: { team: "finance", retries: 3 },
-         });
-         expect(warnings).toHaveLength(1);
-         expect(warnings[0]).toContain("'retries'");
-         expect(warnings[0]).toContain("got number");
-      });
-
-      it("says nothing about a block that parses cleanly", () => {
-         expect(
-            packageMaterializationWarnings({
-               schedule: "0 6 * * *",
-               queryMetadata: { team: "finance" },
-            }),
-         ).toEqual([]);
-         expect(packageMaterializationWarnings(undefined)).toEqual([]);
-      });
-   });
-});

package/src/service/package_manifest.ts

@@ -1,241 +0,0 @@
-/**
- * Pure parsing of the package manifest's (publisher.json) `materialization`
- * block. Kept side-effect free (no fs, no worker bootstrap) so it is unit
- * testable in isolation from the package-load worker that consumes it.
- */
-
-const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
-export type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
-
-/**
- * Package-level Malloy Persistence scope mode, declared once at the manifest
- * root. `package` (the default) = artifacts are reused across the package's
- * versions when they satisfy freshness; `version` = each artifact is owned by
- * one published version (the only mode a `materialization.schedule` is legal
- * in). Applied uniformly to every persist source and index in the package.
- */
-export const PACKAGE_SCOPES = ["version", "package"] as const;
-export type PackageScope = (typeof PACKAGE_SCOPES)[number];
-
-/**
- * Read a `scope` value, defaulting to `"package"` when absent or null. Any other
- * value is a manifest error: scope is load-bearing (it decides version-owned vs
- * cross-version reuse), so a typo must fail loudly rather than silently pick a
- * default. Throws on an invalid value.
- */
-export function parsePackageScope(raw: unknown): PackageScope {
-   if (raw === undefined || raw === null) {
-      return "package";
-   }
-   if ((PACKAGE_SCOPES as readonly unknown[]).includes(raw)) {
-      return raw as PackageScope;
-   }
-   throw new Error(
-      `Invalid "scope" in the package manifest: ${JSON.stringify(raw)}. ` +
-         `Expected "version" or "package" (default "package").`,
-   );
-}
-
-/**
- * Resolve the package's scope from its two possible homes: `materialization.scope`
- * (canonical — every other build-behavioral knob lives in that block) and the
- * manifest root (the original home, deprecated).
- *
- * The root form keeps working because scope rides the published artifact: a
- * package published before the move must keep parsing until it is republished.
- * Declaring DIFFERENT values in the two homes throws, because scope decides
- * whether an artifact is version-owned and guessing which one the author meant
- * could reuse a table across versions that was never supposed to be shared.
- *
- * Only a root-ONLY declaration is deprecated. Both homes agreeing is the
- * transition state the server itself writes (see `writePackageManifest`): the
- * envelope for this build, the root for an older publisher that would otherwise
- * silently default to `package`. Warning about that would be warning an operator
- * about something the server did on their behalf and they cannot fix.
- */
-export function resolvePackageScope(
-   rootRaw: unknown,
-   materializationRaw: unknown,
-): { scope: PackageScope; warnings: string[] } {
-   const envelopeRaw =
-      materializationRaw && typeof materializationRaw === "object"
-         ? (materializationRaw as { scope?: unknown }).scope
-         : undefined;
-   const rootDeclared = rootRaw !== undefined && rootRaw !== null;
-   const envelopeDeclared = envelopeRaw !== undefined && envelopeRaw !== null;
-
-   // Validate both homes before comparing, so a typo is reported as a typo
-   // rather than as a conflict.
-   const rootScope = parsePackageScope(rootRaw);
-   const envelopeScope = parsePackageScope(envelopeRaw);
-
-   if (rootDeclared && envelopeDeclared) {
-      if (rootScope !== envelopeScope) {
-         // Names the fix, because this throw fails the package LOAD: the
-         // package is skipped and its only trace is /status loadErrors, so the
-         // message is the whole diagnosis. Guessing instead would be worse —
-         // picking the wrong one reuses a table across versions that was never
-         // meant to be shared, silently.
-         throw new Error(
-            `Conflicting "scope" in publisher.json: root "${rootScope}" vs ` +
-               `"materialization.scope" "${envelopeScope}". The package cannot ` +
-               `load until they agree. Edit "materialization": { "scope": ... } ` +
-               `to the value you want and delete the root-level "scope" (the ` +
-               `server rewrites both homes on its next manifest write).`,
-         );
-      }
-      return { scope: envelopeScope, warnings: [] };
-   }
-   if (rootDeclared) {
-      return { scope: rootScope, warnings: [SCOPE_ROOT_DEPRECATION] };
-   }
-   return { scope: envelopeScope, warnings: [] };
-}
-
-const SCOPE_ROOT_DEPRECATION =
-   `"scope" at the manifest root is deprecated: declare it as ` +
-   `"materialization": { "scope": ... } alongside the other build knobs. The ` +
-   `root form still works and will be removed in a future release; until then ` +
-   `the server keeps both homes in sync when it writes the manifest, so an ` +
-   `older publisher still reads the right value.`;
-
-/**
- * The manifest's `materialization.freshness` block, surfaced verbatim for the
- * control plane (which owns the scheduling and query-time gating logic).
- * Fields are kept only when valid — an invalid value is dropped, never
- * defaulted, so absence on the wire always means "not declared".
- */
-export interface PackageFreshnessConfig {
-   /** Maximum acceptable staleness, as a duration string (e.g. "24h"). */
-   window?: string;
-   /** Declared query-time behavior when the window is missed. */
-   fallback?: FreshnessFallback;
-}
-
-export interface PackageMaterializationConfig {
-   /**
-    * 5-field UNIX cron the control plane uses to schedule version-level
-    * re-materialization. Null when absent or not a string. Publish-gated:
-    * only valid when every persist source resolves to explicit
-    * `sharing=private` (see Package.scheduleWarnings).
-    */
-   schedule: string | null;
-   /**
-    * Freshness policy ({ window, fallback }). Null when the manifest declares
-    * none — distinct from an empty object, so the control plane can tell
-    * "no policy" from "policy with no valid fields".
-    */
-   freshness: PackageFreshnessConfig | null;
-   /**
-    * Package-level per-query metadata: the least specific model-side layer,
-    * overridden per property by a model-file, source or per-request declaration.
-    * Null when the manifest declares none.
-    *
-    * Kept VERBATIM (every string-valued property, conforming or not) rather than
-    * filtered to what Malloy will accept. A property that violates the contract
-    * has to be visible somewhere to be fixable: publish reports it as a warning
-    * from this block, and the runtime clamps it with a metric. Filtering here
-    * would make an author's typo disappear with nothing to point at.
-    */
-   queryMetadata: Record<string, string> | null;
-}
-
-function parseFreshness(raw: unknown): PackageFreshnessConfig | null {
-   if (!raw || typeof raw !== "object") {
-      return null;
-   }
-   const { window, fallback } = raw as { window?: unknown; fallback?: unknown };
-   const freshness: PackageFreshnessConfig = {};
-   if (typeof window === "string") {
-      freshness.window = window;
-   }
-   if (
-      typeof fallback === "string" &&
-      (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
-   ) {
-      freshness.fallback = fallback as FreshnessFallback;
-   }
-   return freshness;
-}
-
-/**
- * The `materialization.queryMetadata` block: string-valued properties verbatim.
- * A non-object block, or one with no string values, degrades to null so absence
- * on the wire always means "declared nothing usable". A non-string value is
- * dropped here because it cannot round-trip as a property at all; contract
- * violations of the string values are deliberately NOT dropped (see
- * {@link PackageMaterializationConfig.queryMetadata}).
- *
- * Each dropped value is reported, because the drop happens BEFORE the config
- * validation that would otherwise name it: an unquoted `"team": 123` is a
- * plausible hand-edit, and without this it disappears with nothing anywhere to
- * point at.
- */
-function parseQueryMetadata(raw: unknown): {
-   metadata: Record<string, string> | null;
-   warnings: string[];
-} {
-   if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
-      return { metadata: null, warnings: [] };
-   }
-   const out: Record<string, string> = {};
-   const warnings: string[] = [];
-   for (const [name, value] of Object.entries(raw as Record<string, unknown>)) {
-      if (typeof value === "string") {
-         out[name] = value;
-      } else {
-         warnings.push(
-            `materialization.queryMetadata: property '${name}' must be a ` +
-               `string (got ${value === null ? "null" : typeof value}); it is ` +
-               `not attached to any statement`,
-         );
-      }
-   }
-   return {
-      metadata: Object.keys(out).length > 0 ? out : null,
-      warnings,
-   };
-}
-
-/**
- * Read the manifest's `materialization` object, keeping only recognized fields.
- * Returns null when the block is absent so the API field is null rather than an
- * empty object; a non-string schedule degrades to null, and an absent or
- * non-object freshness degrades to null.
- *
- * `materialization.scope` is deliberately NOT read here: scope is a package-level
- * mode rather than a layered knob, and it has two possible homes to reconcile —
- * see {@link resolvePackageScope}, which owns that read and surfaces the
- * resolved value on the package itself.
- */
-export function parsePackageMaterialization(
-   raw: unknown,
-): PackageMaterializationConfig | null {
-   if (!raw || typeof raw !== "object") {
-      return null;
-   }
-   const { schedule, freshness, queryMetadata } = raw as {
-      schedule?: unknown;
-      freshness?: unknown;
-      queryMetadata?: unknown;
-   };
-   return {
-      schedule: typeof schedule === "string" ? schedule : null,
-      freshness: parseFreshness(freshness),
-      queryMetadata: parseQueryMetadata(queryMetadata).metadata,
-   };
-}
-
-/**
- * What the `materialization` parse tolerated but could not keep, for the
- * operator warnings array. Reads the same parse as
- * {@link parsePackageMaterialization} rather than re-deriving it, so the two
- * cannot disagree about what was dropped.
- */
-export function packageMaterializationWarnings(raw: unknown): string[] {
-   if (!raw || typeof raw !== "object") {
-      return [];
-   }
-   const { queryMetadata } = raw as { queryMetadata?: unknown };
-   return parseQueryMetadata(queryMetadata).warnings;
-}

package/src/service/package_memory_governor.spec.ts

@@ -1,173 +0,0 @@
-import { describe, expect, it } from "bun:test";
-
-import type { MemoryGovernorConfig } from "../config";
-import { PackageMemoryGovernor } from "./package_memory_governor";
-
-const ONE_GB = 1024 * 1024 * 1024;
-
-function makeConfig(
-   overrides: Partial<MemoryGovernorConfig> = {},
-): MemoryGovernorConfig {
-   return {
-      maxMemoryBytes: ONE_GB,
-      highWaterFraction: 0.8,
-      lowWaterFraction: 0.7,
-      checkIntervalMs: 5_000,
-      backpressureEnabled: true,
-      ...overrides,
-   };
-}
-
-/**
- * Test driver that lets us push a sequence of RSS values into the
- * governor and inspect the state machine's reactions deterministically
- * — no real allocations, no real timers.
- */
-class FakeRssSampler {
-   private value = 0;
-   constructor(initial = 0) {
-      this.value = initial;
-   }
-   set(value: number): void {
-      this.value = value;
-   }
-   sampler = (): number => this.value;
-}
-
-describe("PackageMemoryGovernor", () => {
-   it("does not activate back-pressure below the high-water mark", () => {
-      const rss = new FakeRssSampler(0.5 * ONE_GB);
-      const gov = new PackageMemoryGovernor(makeConfig(), rss.sampler);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(false);
-   });
-
-   it("activates back-pressure at or above the high-water mark", () => {
-      const rss = new FakeRssSampler(0);
-      const gov = new PackageMemoryGovernor(makeConfig(), rss.sampler);
-
-      rss.set(0.79 * ONE_GB);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(false);
-
-      // 0.8 * 1GB is exactly the high-water threshold; using >= so it
-      // trips on the boundary.
-      rss.set(0.8 * ONE_GB);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(true);
-   });
-
-   it("does not clear back-pressure inside the hysteresis band", () => {
-      const rss = new FakeRssSampler(0.9 * ONE_GB);
-      const gov = new PackageMemoryGovernor(makeConfig(), rss.sampler);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(true);
-
-      // Between low (0.7) and high (0.8) — must stay backpressured.
-      rss.set(0.75 * ONE_GB);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(true);
-   });
-
-   it("clears back-pressure at or below the low-water mark", () => {
-      const rss = new FakeRssSampler(0.9 * ONE_GB);
-      const gov = new PackageMemoryGovernor(makeConfig(), rss.sampler);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(true);
-
-      // The implementation floors lowWaterBytes (= 0.7 * 1GB → 751619276),
-      // so we need to feed a value at or below that integer — `0.7 * 1GB`
-      // as a float is 751619276.8 which sits just above the threshold.
-      rss.set(0.69 * ONE_GB);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(false);
-   });
-
-   it("re-activates after recovery if RSS climbs again", () => {
-      const rss = new FakeRssSampler(0);
-      const gov = new PackageMemoryGovernor(makeConfig(), rss.sampler);
-
-      rss.set(0.85 * ONE_GB);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(true);
-
-      rss.set(0.6 * ONE_GB);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(false);
-
-      rss.set(0.9 * ONE_GB);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(true);
-   });
-
-   it("samples but never flips the flag when backpressureEnabled=false", () => {
-      const rss = new FakeRssSampler(0.95 * ONE_GB);
-      const gov = new PackageMemoryGovernor(
-         makeConfig({ backpressureEnabled: false }),
-         rss.sampler,
-      );
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(false);
-      // Status still tracks RSS even though the flag is suppressed.
-      expect(gov.getStatus().rssBytes).toBe(0.95 * ONE_GB);
-   });
-
-   it("survives a throwing sampler without crashing or flipping state", () => {
-      let throwOnce = true;
-      const gov = new PackageMemoryGovernor(makeConfig(), () => {
-         if (throwOnce) {
-            throwOnce = false;
-            throw new Error("simulated sampling failure");
-         }
-         return 0.4 * ONE_GB;
-      });
-
-      // First tick: sampler throws; governor swallows it and leaves
-      // the state untouched.
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(false);
-
-      // Second tick succeeds.
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(false);
-   });
-
-   it("start() takes an immediate sample so a hot-start respects the cap", () => {
-      const rss = new FakeRssSampler(0.95 * ONE_GB);
-      const gov = new PackageMemoryGovernor(
-         // Big interval so we know the initial sample isn't from a
-         // delayed tick.
-         makeConfig({ checkIntervalMs: 60_000 }),
-         rss.sampler,
-      );
-      gov.start();
-      expect(gov.isBackpressured()).toBe(true);
-      gov.stop();
-   });
-
-   it("stop() clears back-pressure and is idempotent", () => {
-      const rss = new FakeRssSampler(0.95 * ONE_GB);
-      const gov = new PackageMemoryGovernor(makeConfig(), rss.sampler);
-      gov.tick();
-      expect(gov.isBackpressured()).toBe(true);
-
-      gov.stop();
-      expect(gov.isBackpressured()).toBe(false);
-      // Second call is a no-op (no thrown error, flag stays cleared).
-      gov.stop();
-      expect(gov.isBackpressured()).toBe(false);
-   });
-
-   it("exposes computed threshold bytes through getStatus", () => {
-      const rss = new FakeRssSampler(0.4 * ONE_GB);
-      const gov = new PackageMemoryGovernor(makeConfig(), rss.sampler);
-      gov.tick();
-      const status = gov.getStatus();
-      expect(status.maxMemoryBytes).toBe(ONE_GB);
-      expect(status.highWaterBytes).toBe(Math.floor(0.8 * ONE_GB));
-      expect(status.lowWaterBytes).toBe(Math.floor(0.7 * ONE_GB));
-      expect(status.rssBytes).toBe(0.4 * ONE_GB);
-      expect(status.backpressured).toBe(false);
-      expect(typeof status.lastSampledAt).toBe("number");
-   });
-});