@malloy-publisher/server 0.0.231 → 0.0.233

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.docker.md +4 -0
  2. package/dist/app/api-doc.yaml +135 -13
  3. package/dist/app/assets/{EnvironmentPage-wa_EPkwK.js → EnvironmentPage-DutP7T8h.js} +1 -1
  4. package/dist/app/assets/{HomePage-jnCrupQp.js → HomePage-BcxDrBfl.js} +1 -1
  5. package/dist/app/assets/{LightMode-DYbwNULZ.js → LightMode-BJukGxgz.js} +1 -1
  6. package/dist/app/assets/{MainPage-CuJLrPNI.js → MainPage-DXbwlMeF.js} +2 -2
  7. package/dist/app/assets/{MaterializationsPage-D_67x2ee.js → MaterializationsPage-BBQksmTU.js} +1 -1
  8. package/dist/app/assets/{ModelPage-D5JtAWqR.js → ModelPage-C6tK51uU.js} +1 -1
  9. package/dist/app/assets/{PackagePage-BRwtqUSG.js → PackagePage-Bo3cwwZE.js} +1 -1
  10. package/dist/app/assets/{RouteError-CBNNrnSD.js → RouteError-BufkcAKE.js} +1 -1
  11. package/dist/app/assets/{ThemeEditorPage-CTCeBneA.js → ThemeEditorPage-DICvvKpa.js} +1 -1
  12. package/dist/app/assets/{WorkbookPage-SN6f1RBm.js → WorkbookPage-Dkwt75Nj.js} +1 -1
  13. package/dist/app/assets/{core-Dp3q5Ieu.es-CD5FvM2s.js → core-C0nunIQT.es-DlMLKZBK.js} +1 -1
  14. package/dist/app/assets/{index-DU4r7GdU.js → index-BabP-V-S.js} +346 -321
  15. package/dist/app/assets/{index-BLCx1EdC.js → index-BusxL5Pt.js} +1 -1
  16. package/dist/app/assets/{index-C_tJstcx.js → index-CmEVVe-8.js} +15 -15
  17. package/dist/app/assets/{index-CfmBVB6M.js → index-Cs4WVm2z.js} +1 -1
  18. package/dist/app/assets/{index-B3Nn8Vm2.js → index-qnhU9CGo.js} +266 -265
  19. package/dist/app/index.html +1 -1
  20. package/dist/instrumentation.mjs +2 -0
  21. package/dist/package_load_worker.mjs +11 -1
  22. package/dist/server.mjs +22627 -1260
  23. package/package.json +12 -12
  24. package/scripts/bake-duckdb-extensions.js +4 -1
  25. package/src/config.spec.ts +39 -0
  26. package/src/config.ts +135 -0
  27. package/src/controller/materialization.controller.spec.ts +62 -0
  28. package/src/controller/materialization.controller.ts +15 -0
  29. package/src/controller/package.controller.spec.ts +6 -0
  30. package/src/controller/package.controller.ts +7 -2
  31. package/src/controller/query.controller.ts +26 -21
  32. package/src/errors.ts +19 -0
  33. package/src/json_utils.spec.ts +51 -0
  34. package/src/json_utils.ts +33 -0
  35. package/src/logger.spec.ts +18 -1
  36. package/src/logger.ts +3 -1
  37. package/src/materialization_metrics.spec.ts +89 -4
  38. package/src/materialization_metrics.ts +155 -5
  39. package/src/mcp/query_envelope.spec.ts +229 -0
  40. package/src/mcp/query_envelope.ts +230 -0
  41. package/src/mcp/server.protocol.spec.ts +128 -16
  42. package/src/mcp/skills/build_skills_bundle.ts +94 -4
  43. package/src/mcp/skills/skills_bundle.json +1 -1
  44. package/src/mcp/skills/skills_bundle.spec.ts +113 -4
  45. package/src/mcp/tool_response.spec.ts +108 -0
  46. package/src/mcp/tool_response.ts +138 -0
  47. package/src/mcp/tools/compile_tool.spec.ts +112 -4
  48. package/src/mcp/tools/compile_tool.ts +61 -30
  49. package/src/mcp/tools/docs_search_tool.ts +6 -16
  50. package/src/mcp/tools/embedding_index.spec.ts +1236 -0
  51. package/src/mcp/tools/embedding_index.ts +808 -0
  52. package/src/mcp/tools/execute_query_tool.spec.ts +23 -3
  53. package/src/mcp/tools/execute_query_tool.ts +90 -151
  54. package/src/mcp/tools/get_context_eval.ts +194 -45
  55. package/src/mcp/tools/get_context_tool.spec.ts +358 -5
  56. package/src/mcp/tools/get_context_tool.ts +202 -56
  57. package/src/mcp/tools/reload_package_tool.ts +3 -29
  58. package/src/pg_helpers.spec.ts +201 -0
  59. package/src/pg_helpers.ts +44 -5
  60. package/src/server.ts +24 -0
  61. package/src/service/build_plan.spec.ts +128 -2
  62. package/src/service/build_plan.ts +239 -17
  63. package/src/service/compile_fragment_techniques.spec.ts +156 -0
  64. package/src/service/connection.spec.ts +371 -1
  65. package/src/service/connection.ts +339 -20
  66. package/src/service/connection_config.spec.ts +108 -0
  67. package/src/service/connection_config.ts +47 -8
  68. package/src/service/connection_federation.spec.ts +184 -0
  69. package/src/service/duckdb_instance_isolation.spec.ts +137 -0
  70. package/src/service/embedding_provider.spec.ts +329 -0
  71. package/src/service/embedding_provider.ts +236 -0
  72. package/src/service/environment.ts +274 -12
  73. package/src/service/environment_store.spec.ts +678 -3
  74. package/src/service/environment_store.ts +449 -33
  75. package/src/service/environment_store_clone.spec.ts +350 -0
  76. package/src/service/manifest_loader.spec.ts +68 -13
  77. package/src/service/manifest_loader.ts +67 -19
  78. package/src/service/materialization_build_session.spec.ts +435 -0
  79. package/src/service/materialization_build_session.ts +681 -0
  80. package/src/service/materialization_eligibility.spec.ts +158 -0
  81. package/src/service/materialization_eligibility.ts +305 -0
  82. package/src/service/materialization_serve_transform.spec.ts +1003 -0
  83. package/src/service/materialization_serve_transform.ts +779 -0
  84. package/src/service/materialization_service.spec.ts +774 -7
  85. package/src/service/materialization_service.ts +1107 -42
  86. package/src/service/materialization_test_fixtures.ts +7 -0
  87. package/src/service/model.spec.ts +207 -0
  88. package/src/service/model.ts +569 -59
  89. package/src/service/model_limits.spec.ts +28 -0
  90. package/src/service/model_limits.ts +21 -0
  91. package/src/service/model_storage_serve.spec.ts +193 -0
  92. package/src/service/model_storage_serve_joins.spec.ts +193 -0
  93. package/src/service/package.spec.ts +196 -0
  94. package/src/service/package.ts +385 -17
  95. package/src/service/persistence_policy.spec.ts +109 -0
  96. package/src/storage/duckdb/schema.ts +37 -0
  97. package/tests/fixtures/xlsx/database.xlsx +0 -0
  98. package/tests/integration/first_boot/readiness_line.integration.spec.ts +177 -0
  99. package/tests/integration/materialization/manifest_binding.integration.spec.ts +104 -0
  100. package/tests/integration/mcp/mcp_execute_query_tool.integration.spec.ts +37 -12
  101. package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +235 -0
@@ -1,4 +1,12 @@
1
- import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test";
1
+ import {
2
+ afterEach,
3
+ beforeEach,
4
+ describe,
5
+ expect,
6
+ it,
7
+ mock,
8
+ spyOn,
9
+ } from "bun:test";
2
10
  import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
3
11
  import { promises as fsPromises } from "fs";
4
12
  import * as path from "path";
@@ -7,8 +15,18 @@ import { components } from "../api";
7
15
  import { isPublisherConfigFrozen } from "../config";
8
16
  import { TEMP_DIR_PATH } from "../constants";
9
17
  import { BadRequestError } from "../errors";
10
- import { Environment } from "./environment";
11
- import { EnvironmentStore, resolvePackageLocation } from "./environment_store";
18
+ import { _resetEmbeddingIndexStateForTests } from "../mcp/tools/embedding_index";
19
+ import { Environment, PackageStatus } from "./environment";
20
+ import {
21
+ CloneProgressReporter,
22
+ cloneProgressLabel,
23
+ EnvironmentStore,
24
+ formatReadinessLine,
25
+ GIT_CLONE_OPTIONS,
26
+ parseGitHubUrl,
27
+ resolvePackageLocation,
28
+ stripGitProgressNoise,
29
+ } from "./environment_store";
12
30
 
13
31
  type MockData = Record<string, unknown>;
14
32
 
@@ -19,6 +37,13 @@ type MockData = Record<string, unknown>;
19
37
  // so a test can assign it before constructing the store.
20
38
  let mockDbEnvironments: unknown[] = [];
21
39
 
40
+ // Recorder for the embeddings-cleanup wiring tests: every SQL statement
41
+ // the store's cleanup path issues lands here. When `blocks` is true the
42
+ // statements never resolve, which pins that the delete APIs do not await
43
+ // the cleanup (it can queue for minutes behind a bulk embed).
44
+ const embeddingCleanupRuns: Array<{ sql: string; params: unknown[] }> = [];
45
+ let embeddingCleanupBlocks = false;
46
+
22
47
  mock.module("../storage/StorageManager", () => {
23
48
  return {
24
49
  StorageManager: class MockStorageManager {
@@ -26,6 +51,19 @@ mock.module("../storage/StorageManager", () => {
26
51
  return;
27
52
  }
28
53
 
54
+ getDuckDbConnection() {
55
+ return {
56
+ run: (sql: string, params?: unknown[]) => {
57
+ embeddingCleanupRuns.push({ sql, params: params ?? [] });
58
+ return embeddingCleanupBlocks
59
+ ? new Promise<void>(() => {})
60
+ : Promise.resolve();
61
+ },
62
+ all: async () => [],
63
+ get: async () => null,
64
+ };
65
+ }
66
+
29
67
  getRepository() {
30
68
  return {
31
69
  // ===== PROJECT METHODS =====
@@ -1597,3 +1635,640 @@ describe("resolvePackageLocation", () => {
1597
1635
  );
1598
1636
  });
1599
1637
  });
1638
+
1639
+ describe("Environment.getServingPackageCount", () => {
1640
+ const envRoot = path.join(TEMP_DIR_PATH, "serving-count-env");
1641
+
1642
+ beforeEach(() => {
1643
+ rmSync(envRoot, { recursive: true, force: true });
1644
+ mkdirSync(envRoot, { recursive: true });
1645
+ });
1646
+
1647
+ afterEach(() => {
1648
+ rmSync(envRoot, { recursive: true, force: true });
1649
+ });
1650
+
1651
+ it("excludes a package that is registered SERVING but recorded as failed", async () => {
1652
+ // A mount-failed package is seeded SERVING at boot and only pruned by a
1653
+ // later side-effect load; if that prune is skipped (a transient DB or
1654
+ // memory-pressure error), packages= must still not count it, and it
1655
+ // must not overlap load_errors=. The serving count is registered-minus-
1656
+ // failed, so it holds regardless of whether the prune ran.
1657
+ //
1658
+ // Two good and one bad, not one-and-one: this catches both size-only
1659
+ // (would say 3) and a full inversion (would say 1), so the count really
1660
+ // is "registered minus failed".
1661
+ const env = await Environment.create("e", envRoot, []);
1662
+ env.setPackageStatus("good-1", PackageStatus.SERVING);
1663
+ env.setPackageStatus("good-2", PackageStatus.SERVING);
1664
+ env.setPackageStatus("bad", PackageStatus.SERVING);
1665
+ env.setPackageMountError("bad", "location does not exist");
1666
+
1667
+ expect(env.getServingPackageCount()).toBe(2);
1668
+ // Disjoint from the failure set, so packages= + load_errors= never
1669
+ // double-counts "bad".
1670
+ expect(env.getFailedPackages().has("bad")).toBe(true);
1671
+ });
1672
+ });
1673
+
1674
+ describe("GIT_CLONE_OPTIONS", () => {
1675
+ it("clones shallow and single-branch", () => {
1676
+ // The load-bearing content: packages serve the default branch's working
1677
+ // tree only, so history is pure download cost. simple-git serializes
1678
+ // this to `git clone --depth=1 --single-branch`.
1679
+ expect(GIT_CLONE_OPTIONS["--depth"]).toBe(1);
1680
+ expect(GIT_CLONE_OPTIONS).toHaveProperty("--single-branch", null);
1681
+ });
1682
+ });
1683
+
1684
+ describe("stripGitProgressNoise", () => {
1685
+ it("drops progress lines and keeps the error", () => {
1686
+ const message = [
1687
+ "Cloning into '/tmp/x'...",
1688
+ "remote: Counting objects: 45% (9/20)",
1689
+ "Receiving objects: 99% (990/1000)",
1690
+ "fatal: early EOF",
1691
+ "fatal: fetch-pack: invalid index-pack output",
1692
+ ].join("\n");
1693
+ expect(stripGitProgressNoise(message)).toBe(
1694
+ "fatal: early EOF\nfatal: fetch-pack: invalid index-pack output",
1695
+ );
1696
+ });
1697
+
1698
+ it("handles carriage-return separated progress spew", () => {
1699
+ // git rewrites progress in place with \r, so the captured stderr is one
1700
+ // long line unless split on \r too.
1701
+ const message =
1702
+ "Receiving objects: 10% (1/10)\rReceiving objects: 50% (5/10)\rfatal: the remote end hung up unexpectedly";
1703
+ expect(stripGitProgressNoise(message)).toBe(
1704
+ "fatal: the remote end hung up unexpectedly",
1705
+ );
1706
+ });
1707
+
1708
+ it("returns the input when every line is progress noise", () => {
1709
+ const message = "Receiving objects: 10% (1/10)";
1710
+ expect(stripGitProgressNoise(message)).toBe(message);
1711
+ });
1712
+ });
1713
+
1714
+ describe("cloneProgressLabel", () => {
1715
+ it("names the environment, repo, and packages", () => {
1716
+ expect(
1717
+ cloneProgressLabel("malloydata/publisher", {
1718
+ environmentName: "examples",
1719
+ packageNames: ["storefront", "governed-analytics"],
1720
+ }),
1721
+ ).toBe(
1722
+ "[examples] cloning malloydata/publisher (storefront, governed-analytics)",
1723
+ );
1724
+ });
1725
+
1726
+ it("truncates long package lists", () => {
1727
+ expect(
1728
+ cloneProgressLabel("o/r", {
1729
+ environmentName: "e",
1730
+ packageNames: ["a", "b", "c", "d", "e", "f"],
1731
+ }),
1732
+ ).toBe("[e] cloning o/r (a, b, c, +3 more)");
1733
+ });
1734
+
1735
+ it("labels by repo alone without context", () => {
1736
+ // The controller add-package path passes no context.
1737
+ expect(cloneProgressLabel("o/r")).toBe("cloning o/r");
1738
+ });
1739
+ });
1740
+
1741
+ describe("CloneProgressReporter", () => {
1742
+ const event = (stage: string, progress: number, processed = 0, total = 0) =>
1743
+ ({ method: "clone", stage, progress, processed, total }) as Parameters<
1744
+ CloneProgressReporter["onProgress"]
1745
+ >[0];
1746
+
1747
+ const fakeOut = (isTTY: boolean) => {
1748
+ const writes: string[] = [];
1749
+ const out = {
1750
+ isTTY,
1751
+ write: (chunk: string) => {
1752
+ writes.push(chunk);
1753
+ return true;
1754
+ },
1755
+ } as unknown as NodeJS.WriteStream;
1756
+ return { writes, out };
1757
+ };
1758
+
1759
+ it("prints a line per stage and per 25-point step off a TTY", () => {
1760
+ const { writes, out } = fakeOut(false);
1761
+ const reporter = new CloneProgressReporter("cloning o/r", out);
1762
+ reporter.onProgress(event("receiving", 0));
1763
+ reporter.onProgress(event("receiving", 10)); // same milestone, suppressed
1764
+ reporter.onProgress(event("receiving", 24)); // same milestone, suppressed
1765
+ reporter.onProgress(event("receiving", 25, 25, 100));
1766
+ reporter.onProgress(event("receiving", 90));
1767
+ reporter.onProgress(event("resolving", 5)); // stage change, printed
1768
+ reporter.done(); // no in-place line to finish
1769
+ expect(writes).toEqual([
1770
+ "cloning o/r: receiving 0%\n",
1771
+ "cloning o/r: receiving 25% (25/100)\n",
1772
+ "cloning o/r: receiving 90%\n",
1773
+ "cloning o/r: resolving 5%\n",
1774
+ ]);
1775
+ });
1776
+
1777
+ it("tolerates arbitrary stage strings", () => {
1778
+ // Server-side progress lines parse with stage "remote:".
1779
+ const { writes, out } = fakeOut(false);
1780
+ const reporter = new CloneProgressReporter("cloning o/r", out);
1781
+ reporter.onProgress(event("remote:", 50));
1782
+ expect(writes).toEqual(["cloning o/r: remote: 50%\n"]);
1783
+ });
1784
+
1785
+ it("rewrites one line in place on a TTY and finishes it on done", () => {
1786
+ const { writes, out } = fakeOut(true);
1787
+ const reporter = new CloneProgressReporter("cloning o/r", out);
1788
+ reporter.onProgress(event("receiving", 10, 1, 10));
1789
+ reporter.onProgress(event("receiving", 11, 2, 10));
1790
+ reporter.done();
1791
+ expect(writes).toEqual([
1792
+ "\rcloning o/r: receiving 10% (1/10)\x1b[K",
1793
+ "\rcloning o/r: receiving 11% (2/10)\x1b[K",
1794
+ "\n",
1795
+ ]);
1796
+ });
1797
+
1798
+ it("clamps the in-place line to the terminal width", () => {
1799
+ // An auto-wrapped line breaks the \r rewrite: the cursor lands on the
1800
+ // continuation row and every event scrolls a stale fragment. Too
1801
+ // narrow for even the suffix, so a plain head slice applies.
1802
+ const { writes, out } = fakeOut(true);
1803
+ (out as unknown as { columns: number }).columns = 20;
1804
+ const reporter = new CloneProgressReporter("cloning owner/repo", out);
1805
+ reporter.onProgress(event("receiving", 45, 100, 220));
1806
+ reporter.done();
1807
+ expect(writes[0]).toBe(
1808
+ "\r" + "cloning owner/repo:".slice(0, 19) + "\x1b[K",
1809
+ );
1810
+ expect(writes[0].length).toBe(1 + 19 + 3);
1811
+ });
1812
+
1813
+ it("keeps the moving percentage visible when clamping a long label", () => {
1814
+ // The default npx label is longer than an 80-column terminal; the
1815
+ // label shortens, the stage and percentage stay on screen.
1816
+ const { writes, out } = fakeOut(true);
1817
+ (out as unknown as { columns: number }).columns = 60;
1818
+ const reporter = new CloneProgressReporter(
1819
+ "[examples] cloning malloydata/publisher (storefront, governed-analytics)",
1820
+ out,
1821
+ );
1822
+ reporter.onProgress(event("receiving", 45, 100, 220));
1823
+ reporter.done();
1824
+ const visible = writes[0].slice(1, -3); // strip \r and \x1b[K
1825
+ expect(visible.length).toBe(59);
1826
+ expect(visible.endsWith(": receiving 45% (100/220)")).toBe(true);
1827
+ expect(visible).toContain("...");
1828
+ });
1829
+
1830
+ it("resets the milestone when the percentage restarts within a stage", () => {
1831
+ // git's server-side counting and compressing phases both parse as
1832
+ // stage "remote:"; the second starts back at 0%.
1833
+ const { writes, out } = fakeOut(false);
1834
+ const reporter = new CloneProgressReporter("cloning o/r", out);
1835
+ reporter.onProgress(event("remote:", 100));
1836
+ reporter.onProgress(event("remote:", 10));
1837
+ reporter.onProgress(event("remote:", 60));
1838
+ reporter.done();
1839
+ expect(writes).toEqual([
1840
+ "cloning o/r: remote: 100%\n",
1841
+ "cloning o/r: remote: 10%\n",
1842
+ "cloning o/r: remote: 60%\n",
1843
+ ]);
1844
+ });
1845
+
1846
+ it("ignores a progress event that arrives after done()", () => {
1847
+ // simple-git can deliver a trailing event after the completion
1848
+ // callback ran done(); it must not re-write the row the finishing
1849
+ // newline scrolled past, nor re-claim TTY ownership.
1850
+ const { writes, out } = fakeOut(true);
1851
+ const reporter = new CloneProgressReporter("cloning o/r", out);
1852
+ reporter.onProgress(event("receiving", 50));
1853
+ reporter.done();
1854
+ const writesAfterDone = writes.length;
1855
+ reporter.onProgress(event("receiving", 90));
1856
+ expect(writes.length).toBe(writesAfterDone);
1857
+ });
1858
+
1859
+ it("only one reporter owns a TTY's in-place line at a time", () => {
1860
+ // Environments load concurrently at boot; a second clone must not
1861
+ // rewrite the first one's line. It falls back to milestone lines.
1862
+ const { writes, out } = fakeOut(true);
1863
+ const first = new CloneProgressReporter("cloning a/a", out);
1864
+ const second = new CloneProgressReporter("cloning b/b", out);
1865
+ first.onProgress(event("receiving", 10));
1866
+ second.onProgress(event("receiving", 20));
1867
+ first.onProgress(event("receiving", 30));
1868
+ first.done();
1869
+ // After the owner finishes, the line is free to claim again.
1870
+ second.onProgress(event("receiving", 90));
1871
+ second.done();
1872
+ expect(writes).toEqual([
1873
+ "\rcloning a/a: receiving 10%\x1b[K",
1874
+ // The non-owner takes over the row cleanly before scrolling it, so
1875
+ // its milestone never concatenates onto the owner's in-place text.
1876
+ "\rcloning b/b: receiving 20%\x1b[K\n",
1877
+ "\rcloning a/a: receiving 30%\x1b[K",
1878
+ "\n",
1879
+ "\rcloning b/b: receiving 90%\x1b[K",
1880
+ "\n",
1881
+ ]);
1882
+ });
1883
+ });
1884
+
1885
+ describe("formatReadinessLine", () => {
1886
+ const SAVED = ["PUBLISHER_HOST", "PUBLISHER_PORT", "MCP_PORT"] as const;
1887
+ let saved: Record<string, string | undefined>;
1888
+
1889
+ beforeEach(() => {
1890
+ saved = {};
1891
+ for (const key of SAVED) {
1892
+ saved[key] = process.env[key];
1893
+ delete process.env[key];
1894
+ }
1895
+ });
1896
+
1897
+ afterEach(() => {
1898
+ for (const key of SAVED) {
1899
+ if (saved[key] === undefined) delete process.env[key];
1900
+ else process.env[key] = saved[key];
1901
+ }
1902
+ });
1903
+
1904
+ it("uses the server defaults and displays the wildcard bind as localhost", () => {
1905
+ expect(
1906
+ formatReadinessLine({ environments: 1, packages: 3, loadErrors: 0 }),
1907
+ ).toBe(
1908
+ "PUBLISHER_READY url=http://localhost:4000 mcp=http://localhost:4040 " +
1909
+ "environments=1 packages=3 load_errors=0",
1910
+ );
1911
+ });
1912
+
1913
+ it("brackets an IPv6 literal host so the URL is dialable", () => {
1914
+ process.env.PUBLISHER_HOST = "::1";
1915
+ expect(
1916
+ formatReadinessLine({ environments: 1, packages: 1, loadErrors: 0 }),
1917
+ ).toBe(
1918
+ "PUBLISHER_READY url=http://[::1]:4000 mcp=http://[::1]:4040 " +
1919
+ "environments=1 packages=1 load_errors=0",
1920
+ );
1921
+ });
1922
+
1923
+ it("displays the :: wildcard bind as localhost", () => {
1924
+ process.env.PUBLISHER_HOST = "::";
1925
+ expect(
1926
+ formatReadinessLine({ environments: 1, packages: 1, loadErrors: 0 }),
1927
+ ).toBe(
1928
+ "PUBLISHER_READY url=http://localhost:4000 mcp=http://localhost:4040 " +
1929
+ "environments=1 packages=1 load_errors=0",
1930
+ );
1931
+ });
1932
+
1933
+ it("respects the flag-derived env vars", () => {
1934
+ // parseArgs writes --host/--port/--mcp_port here before the store exists.
1935
+ process.env.PUBLISHER_HOST = "127.0.0.1";
1936
+ process.env.PUBLISHER_PORT = "4321";
1937
+ process.env.MCP_PORT = "4361";
1938
+ expect(
1939
+ formatReadinessLine({ environments: 2, packages: 5, loadErrors: 1 }),
1940
+ ).toBe(
1941
+ "PUBLISHER_READY url=http://127.0.0.1:4321 mcp=http://127.0.0.1:4361 " +
1942
+ "environments=2 packages=5 load_errors=1",
1943
+ );
1944
+ });
1945
+ });
1946
+
1947
+ describe("readiness line emission", () => {
1948
+ const readinessRoot = path.join(TEMP_DIR_PATH, "readiness-line-test");
1949
+ let stderrWrites: string[];
1950
+ let stderrSpy: ReturnType<typeof spyOn>;
1951
+
1952
+ const readyLines = () =>
1953
+ stderrWrites
1954
+ .join("")
1955
+ .split("\n")
1956
+ .filter((line) => line.startsWith("PUBLISHER_READY"));
1957
+
1958
+ const writePackage = (name: string) => {
1959
+ const dir = path.join(readinessRoot, "src-packages", name);
1960
+ mkdirSync(dir, { recursive: true });
1961
+ writeFileSync(path.join(dir, "publisher.json"), JSON.stringify({ name }));
1962
+ return dir;
1963
+ };
1964
+
1965
+ beforeEach(() => {
1966
+ if (existsSync(readinessRoot)) {
1967
+ rmSync(readinessRoot, { recursive: true, force: true });
1968
+ }
1969
+ mkdirSync(readinessRoot, { recursive: true });
1970
+ mock.module("../config", () => ({
1971
+ isPublisherConfigFrozen: () => false,
1972
+ }));
1973
+ stderrWrites = [];
1974
+ stderrSpy = spyOn(process.stderr, "write").mockImplementation(
1975
+ (chunk: string | Uint8Array) => {
1976
+ stderrWrites.push(String(chunk));
1977
+ return true;
1978
+ },
1979
+ );
1980
+ });
1981
+
1982
+ afterEach(() => {
1983
+ stderrSpy.mockRestore();
1984
+ rmSync(readinessRoot, { recursive: true, force: true });
1985
+ });
1986
+
1987
+ it("emits exactly one line with counts after a successful boot", async () => {
1988
+ const pkgA = writePackage("pkg-a");
1989
+ const pkgB = writePackage("pkg-b");
1990
+ writeFileSync(
1991
+ path.join(readinessRoot, "publisher.config.json"),
1992
+ JSON.stringify({
1993
+ frozenConfig: false,
1994
+ environments: [
1995
+ {
1996
+ name: "readiness-env",
1997
+ packages: [
1998
+ { name: "pkg-a", location: pkgA },
1999
+ { name: "pkg-b", location: pkgB },
2000
+ ],
2001
+ connections: [],
2002
+ },
2003
+ ],
2004
+ }),
2005
+ );
2006
+
2007
+ const store = new EnvironmentStore(readinessRoot);
2008
+ await store.finishedInitialization;
2009
+
2010
+ const lines = readyLines();
2011
+ expect(lines).toHaveLength(1);
2012
+ expect(lines[0]).toContain("environments=1");
2013
+ expect(lines[0]).toContain("packages=2");
2014
+ expect(lines[0]).toContain("load_errors=0");
2015
+ expect(lines[0]).toMatch(
2016
+ /^PUBLISHER_READY url=http:\/\/\S+ mcp=http:\/\/\S+ environments=\d+ packages=\d+ load_errors=\d+$/,
2017
+ );
2018
+ });
2019
+
2020
+ it("counts a mount failure in load_errors and still emits", async () => {
2021
+ // Post-#903 a bad location drops the package, not the boot; the line
2022
+ // must report the failure rather than stay silent or claim a clean load.
2023
+ const pkgA = writePackage("pkg-a");
2024
+ writeFileSync(
2025
+ path.join(readinessRoot, "publisher.config.json"),
2026
+ JSON.stringify({
2027
+ frozenConfig: false,
2028
+ environments: [
2029
+ {
2030
+ name: "readiness-env",
2031
+ packages: [
2032
+ { name: "pkg-a", location: pkgA },
2033
+ {
2034
+ name: "pkg-missing",
2035
+ location: path.join(readinessRoot, "does-not-exist"),
2036
+ },
2037
+ ],
2038
+ connections: [],
2039
+ },
2040
+ ],
2041
+ }),
2042
+ );
2043
+
2044
+ const store = new EnvironmentStore(readinessRoot);
2045
+ await store.finishedInitialization;
2046
+
2047
+ const lines = readyLines();
2048
+ expect(lines).toHaveLength(1);
2049
+ expect(lines[0]).toContain("environments=1");
2050
+ // packages= is what the server actually serves: the boot-time database
2051
+ // sync prunes a package whose load failed (environment.ts getPackage
2052
+ // catch), so the failed one appears in load_errors, not in packages.
2053
+ expect(lines[0]).toContain("packages=1");
2054
+ expect(lines[0]).toContain("load_errors=1");
2055
+ });
2056
+
2057
+ it("counts a failed environment in load_errors", async () => {
2058
+ // An environment that fails initialization outright lands in
2059
+ // failedEnvironments, not in environments; the line must count it
2060
+ // alongside per-package failures. Failure lever: a FILE squatting the
2061
+ // environment's publisher_data directory path makes the initial mkdir
2062
+ // throw before any package mounts.
2063
+ const pkgA = writePackage("pkg-a");
2064
+ mkdirSync(path.join(readinessRoot, "publisher_data"), {
2065
+ recursive: true,
2066
+ });
2067
+ writeFileSync(path.join(readinessRoot, "publisher_data", "bad-env"), "");
2068
+ writeFileSync(
2069
+ path.join(readinessRoot, "publisher.config.json"),
2070
+ JSON.stringify({
2071
+ frozenConfig: false,
2072
+ environments: [
2073
+ {
2074
+ name: "good-env",
2075
+ packages: [{ name: "pkg-a", location: pkgA }],
2076
+ connections: [],
2077
+ },
2078
+ {
2079
+ name: "bad-env",
2080
+ packages: [{ name: "pkg-a", location: pkgA }],
2081
+ connections: [],
2082
+ },
2083
+ ],
2084
+ }),
2085
+ );
2086
+
2087
+ const store = new EnvironmentStore(readinessRoot);
2088
+ await store.finishedInitialization;
2089
+
2090
+ const lines = readyLines();
2091
+ expect(lines).toHaveLength(1);
2092
+ expect(lines[0]).toContain("environments=1");
2093
+ expect(lines[0]).toContain("packages=1");
2094
+ expect(lines[0]).toContain("load_errors=1");
2095
+ });
2096
+ });
2097
+
2098
+ describe("parseGitHubUrl", () => {
2099
+ // The exact regexes parseGitHubUrl replaced, kept here ONLY as a
2100
+ // behavioral oracle. The product code no longer runs them (they were
2101
+ // flagged as polynomial-ReDoS); the fuzz below proves the linear
2102
+ // string parser returns identical results. Never feed these long input.
2103
+ const oldParseGitHubUrl = (
2104
+ githubUrl: string,
2105
+ ): { owner: string; repoName: string; packagePath?: string } | null => {
2106
+ const httpsRegex =
2107
+ /github\.com\/(?<owner>[^/]+)\/(?<repoName>[^/]+)(?<packagePath>\/[^/]+)*/;
2108
+ const httpsMatch = githubUrl.match(httpsRegex);
2109
+ if (httpsMatch) {
2110
+ const { owner, repoName, packagePath } = httpsMatch.groups!;
2111
+ return { owner, repoName, packagePath };
2112
+ }
2113
+ const sshRegex =
2114
+ /git@github\.com:(?<owner>[^/]+)\/(?<repoName>[^/\s]+?)(?:\.git)?(?<packagePath>\/[^/]+)*$/;
2115
+ const sshMatch = githubUrl.match(sshRegex);
2116
+ if (sshMatch) {
2117
+ const { owner, repoName, packagePath } = sshMatch.groups!;
2118
+ return { owner, repoName, packagePath };
2119
+ }
2120
+ return null;
2121
+ };
2122
+
2123
+ // Absent and undefined packagePath are the same result; normalize so the
2124
+ // comparison ignores that representational difference.
2125
+ const norm = (r: ReturnType<typeof parseGitHubUrl>) =>
2126
+ r === null
2127
+ ? null
2128
+ : {
2129
+ owner: r.owner,
2130
+ repoName: r.repoName,
2131
+ packagePath: r.packagePath ?? null,
2132
+ };
2133
+
2134
+ it("matches the documented cases", () => {
2135
+ expect(
2136
+ parseGitHubUrl(
2137
+ "https://github.com/malloydata/publisher/tree/main/examples/storefront",
2138
+ ),
2139
+ ).toEqual({
2140
+ owner: "malloydata",
2141
+ repoName: "publisher",
2142
+ // Preserved quirk: only the LAST path segment.
2143
+ packagePath: "/storefront",
2144
+ });
2145
+ expect(parseGitHubUrl("https://github.com/owner/repo")).toEqual({
2146
+ owner: "owner",
2147
+ repoName: "repo",
2148
+ });
2149
+ expect(parseGitHubUrl("git@github.com:owner/repo.git")).toEqual({
2150
+ owner: "owner",
2151
+ repoName: "repo",
2152
+ });
2153
+ expect(parseGitHubUrl("https://gitlab.com/owner/repo")).toBeNull();
2154
+ });
2155
+
2156
+ it("is identical to the old regexes across a fuzzed input space", () => {
2157
+ const prefixes = [
2158
+ "https://github.com/",
2159
+ "http://github.com/",
2160
+ "github.com/",
2161
+ "git@github.com:",
2162
+ "ssh://git@github.com:",
2163
+ "before github.com/",
2164
+ "",
2165
+ "https://gitlab.com/",
2166
+ ];
2167
+ const owners = ["owner", "o", "", "own er", "a.b"];
2168
+ const repos = [
2169
+ "repo",
2170
+ "r",
2171
+ "repo.git",
2172
+ ".git",
2173
+ "a.git",
2174
+ "re po",
2175
+ "",
2176
+ "repo.gitx",
2177
+ ];
2178
+ const tails = [
2179
+ "",
2180
+ "/tree/main/sub",
2181
+ "/tree/main",
2182
+ "/a/b/c",
2183
+ "/",
2184
+ "//",
2185
+ "/a//b",
2186
+ "/sub dir",
2187
+ "/tree/branch/a/b",
2188
+ ];
2189
+ let cases = 0;
2190
+ for (const prefix of prefixes) {
2191
+ for (const owner of owners) {
2192
+ for (const repo of repos) {
2193
+ for (const tail of tails) {
2194
+ const input = `${prefix}${owner}/${repo}${tail}`;
2195
+ expect(norm(parseGitHubUrl(input))).toEqual(
2196
+ norm(oldParseGitHubUrl(input)),
2197
+ );
2198
+ cases++;
2199
+ }
2200
+ }
2201
+ }
2202
+ }
2203
+ expect(cases).toBeGreaterThan(2000);
2204
+ });
2205
+
2206
+ it("runs in linear time on adversarial input (no ReDoS)", () => {
2207
+ // The payloads CodeQL flagged: a long run that made the old
2208
+ // `(\/[^/]+)*` regexes backtrack polynomially. The string parser is
2209
+ // linear, so this returns effectively instantly.
2210
+ const start = performance.now();
2211
+ parseGitHubUrl("git@github.com:" + ".".repeat(200000));
2212
+ parseGitHubUrl("git@github.com:" + "a/".repeat(200000));
2213
+ parseGitHubUrl(
2214
+ "git@github.com:./!/" + "git@github.com:./!/".repeat(5000),
2215
+ );
2216
+ parseGitHubUrl("https://github.com/" + "a/".repeat(200000));
2217
+ expect(performance.now() - start).toBeLessThan(500);
2218
+ });
2219
+ });
2220
+
2221
+ describe("EnvironmentStore embeddings cleanup wiring", () => {
2222
+ const serverRootPath = path.join(TEMP_DIR_PATH, "cleanup-wiring-tests");
2223
+
2224
+ beforeEach(() => {
2225
+ if (existsSync(serverRootPath)) {
2226
+ rmSync(serverRootPath, { recursive: true, force: true });
2227
+ }
2228
+ mkdirSync(serverRootPath, { recursive: true });
2229
+ mockDbEnvironments = [];
2230
+ embeddingCleanupRuns.length = 0;
2231
+ embeddingCleanupBlocks = false;
2232
+ _resetEmbeddingIndexStateForTests();
2233
+ });
2234
+
2235
+ afterEach(() => {
2236
+ embeddingCleanupBlocks = false;
2237
+ _resetEmbeddingIndexStateForTests();
2238
+ if (existsSync(serverRootPath)) {
2239
+ rmSync(serverRootPath, { recursive: true, force: true });
2240
+ }
2241
+ });
2242
+
2243
+ it("deletePackageFromDatabase fires cleanup without awaiting it, even with no metadata row", async () => {
2244
+ const store = new EnvironmentStore(serverRootPath);
2245
+ await store.finishedInitialization;
2246
+
2247
+ // The cleanup's SQL never resolves: were the store awaiting it,
2248
+ // this call would hang past the test timeout. The mock repository
2249
+ // reports no environment row, so resolving promptly ALSO pins
2250
+ // that the cleanup runs before the missing-row early return.
2251
+ embeddingCleanupBlocks = true;
2252
+ await store.deletePackageFromDatabase("wiring-env", "wiring-pkg");
2253
+
2254
+ const deletes = embeddingCleanupRuns.filter((r) =>
2255
+ r.sql.includes("DELETE FROM entity_embeddings"),
2256
+ );
2257
+ expect(deletes.length).toBe(1);
2258
+ expect(deletes[0].params).toEqual(["wiring-env", "wiring-pkg"]);
2259
+ });
2260
+
2261
+ it("deleteEnvironmentFromDatabase fires the env-wide cleanup without awaiting it", async () => {
2262
+ const store = new EnvironmentStore(serverRootPath);
2263
+ await store.finishedInitialization;
2264
+
2265
+ embeddingCleanupBlocks = true;
2266
+ await store.deleteEnvironmentFromDatabase("wiring-env");
2267
+
2268
+ const deletes = embeddingCleanupRuns.filter((r) =>
2269
+ r.sql.includes("DELETE FROM entity_embeddings"),
2270
+ );
2271
+ expect(deletes.length).toBe(1);
2272
+ expect(deletes[0].params).toEqual(["wiring-env"]);
2273
+ });
2274
+ });