@malloy-publisher/server 0.0.230 → 0.0.232
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.docker.md +4 -0
- package/dist/app/api-doc.yaml +74 -3
- package/dist/app/assets/{EnvironmentPage-wa_EPkwK.js → EnvironmentPage-DXEaZIPx.js} +1 -1
- package/dist/app/assets/{HomePage-jnCrupQp.js → HomePage-kofsqpZt.js} +1 -1
- package/dist/app/assets/{LightMode-DYbwNULZ.js → LightMode-CNhIlIlJ.js} +1 -1
- package/dist/app/assets/{MainPage-CuJLrPNI.js → MainPage-Bgqo8jCy.js} +1 -1
- package/dist/app/assets/{MaterializationsPage-D_67x2ee.js → MaterializationsPage-CgBlgGz2.js} +1 -1
- package/dist/app/assets/{ModelPage-D5JtAWqR.js → ModelPage-B0TjoDtf.js} +1 -1
- package/dist/app/assets/{PackagePage-BRwtqUSG.js → PackagePage-BL8vnFj1.js} +1 -1
- package/dist/app/assets/{RouteError-CBNNrnSD.js → RouteError-BzPby0X2.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-CTCeBneA.js → ThemeEditorPage-CTEP_9r3.js} +1 -1
- package/dist/app/assets/{WorkbookPage-SN6f1RBm.js → WorkbookPage-BwM3BmKw.js} +1 -1
- package/dist/app/assets/{core-Dp3q5Ieu.es-CD5FvM2s.js → core-CK68iv6w.es-CpRxXBt7.js} +1 -1
- package/dist/app/assets/{index-C_tJstcx.js → index-B33zGctF.js} +15 -15
- package/dist/app/assets/{index-DU4r7GdU.js → index-BabP-V-S.js} +346 -321
- package/dist/app/assets/{index-B3Nn8Vm2.js → index-BkiWKaAF.js} +266 -265
- package/dist/app/assets/{index-BLCx1EdC.js → index-CmkW1MiE.js} +1 -1
- package/dist/app/assets/{index-CfmBVB6M.js → index-tXJXwdyj.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/instrumentation.mjs +2 -0
- package/dist/package_load_worker.mjs +11 -1
- package/dist/server.mjs +3186 -433
- package/package.json +12 -12
- package/scripts/bake-duckdb-extensions.js +4 -1
- package/src/config.spec.ts +39 -0
- package/src/config.ts +135 -0
- package/src/controller/materialization.controller.spec.ts +62 -0
- package/src/controller/materialization.controller.ts +15 -0
- package/src/controller/package.controller.spec.ts +6 -0
- package/src/controller/package.controller.ts +7 -2
- package/src/errors.ts +19 -0
- package/src/logger.spec.ts +18 -1
- package/src/logger.ts +3 -1
- package/src/materialization_metrics.spec.ts +89 -4
- package/src/materialization_metrics.ts +155 -5
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/mcp/tools/embedding_index.spec.ts +1236 -0
- package/src/mcp/tools/embedding_index.ts +808 -0
- package/src/mcp/tools/get_context_eval.ts +194 -45
- package/src/mcp/tools/get_context_tool.spec.ts +295 -2
- package/src/mcp/tools/get_context_tool.ts +159 -10
- package/src/pg_helpers.spec.ts +201 -0
- package/src/pg_helpers.ts +44 -5
- package/src/server.ts +24 -0
- package/src/service/build_plan.spec.ts +128 -2
- package/src/service/build_plan.ts +239 -17
- package/src/service/connection.ts +263 -7
- package/src/service/connection_config.spec.ts +48 -0
- package/src/service/connection_config.ts +21 -8
- package/src/service/connection_federation.spec.ts +184 -0
- package/src/service/embedding_provider.spec.ts +329 -0
- package/src/service/embedding_provider.ts +236 -0
- package/src/service/environment.ts +274 -12
- package/src/service/environment_store.spec.ts +678 -3
- package/src/service/environment_store.ts +449 -33
- package/src/service/environment_store_clone.spec.ts +350 -0
- package/src/service/manifest_loader.spec.ts +68 -13
- package/src/service/manifest_loader.ts +67 -19
- package/src/service/materialization_build_session.spec.ts +435 -0
- package/src/service/materialization_build_session.ts +681 -0
- package/src/service/materialization_eligibility.spec.ts +158 -0
- package/src/service/materialization_eligibility.ts +305 -0
- package/src/service/materialization_serve_transform.spec.ts +1003 -0
- package/src/service/materialization_serve_transform.ts +779 -0
- package/src/service/materialization_service.spec.ts +774 -7
- package/src/service/materialization_service.ts +1107 -42
- package/src/service/materialization_test_fixtures.ts +7 -0
- package/src/service/model.spec.ts +207 -0
- package/src/service/model.ts +540 -52
- package/src/service/model_storage_serve.spec.ts +193 -0
- package/src/service/model_storage_serve_joins.spec.ts +193 -0
- package/src/service/package.spec.ts +196 -0
- package/src/service/package.ts +385 -17
- package/src/service/persistence_policy.spec.ts +109 -0
- package/src/storage/duckdb/schema.ts +37 -0
- package/tests/fixtures/xlsx/database.xlsx +0 -0
- package/tests/integration/first_boot/readiness_line.integration.spec.ts +177 -0
- package/tests/integration/materialization/manifest_binding.integration.spec.ts +104 -0
- package/tests/integration/mcp/mcp_get_context_semantic.integration.spec.ts +235 -0
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
afterEach,
|
|
3
|
+
beforeEach,
|
|
4
|
+
describe,
|
|
5
|
+
expect,
|
|
6
|
+
it,
|
|
7
|
+
mock,
|
|
8
|
+
spyOn,
|
|
9
|
+
} from "bun:test";
|
|
2
10
|
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
|
|
3
11
|
import { promises as fsPromises } from "fs";
|
|
4
12
|
import * as path from "path";
|
|
@@ -7,8 +15,18 @@ import { components } from "../api";
|
|
|
7
15
|
import { isPublisherConfigFrozen } from "../config";
|
|
8
16
|
import { TEMP_DIR_PATH } from "../constants";
|
|
9
17
|
import { BadRequestError } from "../errors";
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
18
|
+
import { _resetEmbeddingIndexStateForTests } from "../mcp/tools/embedding_index";
|
|
19
|
+
import { Environment, PackageStatus } from "./environment";
|
|
20
|
+
import {
|
|
21
|
+
CloneProgressReporter,
|
|
22
|
+
cloneProgressLabel,
|
|
23
|
+
EnvironmentStore,
|
|
24
|
+
formatReadinessLine,
|
|
25
|
+
GIT_CLONE_OPTIONS,
|
|
26
|
+
parseGitHubUrl,
|
|
27
|
+
resolvePackageLocation,
|
|
28
|
+
stripGitProgressNoise,
|
|
29
|
+
} from "./environment_store";
|
|
12
30
|
|
|
13
31
|
type MockData = Record<string, unknown>;
|
|
14
32
|
|
|
@@ -19,6 +37,13 @@ type MockData = Record<string, unknown>;
|
|
|
19
37
|
// so a test can assign it before constructing the store.
|
|
20
38
|
let mockDbEnvironments: unknown[] = [];
|
|
21
39
|
|
|
40
|
+
// Recorder for the embeddings-cleanup wiring tests: every SQL statement
|
|
41
|
+
// the store's cleanup path issues lands here. When `blocks` is true the
|
|
42
|
+
// statements never resolve, which pins that the delete APIs do not await
|
|
43
|
+
// the cleanup (it can queue for minutes behind a bulk embed).
|
|
44
|
+
const embeddingCleanupRuns: Array<{ sql: string; params: unknown[] }> = [];
|
|
45
|
+
let embeddingCleanupBlocks = false;
|
|
46
|
+
|
|
22
47
|
mock.module("../storage/StorageManager", () => {
|
|
23
48
|
return {
|
|
24
49
|
StorageManager: class MockStorageManager {
|
|
@@ -26,6 +51,19 @@ mock.module("../storage/StorageManager", () => {
|
|
|
26
51
|
return;
|
|
27
52
|
}
|
|
28
53
|
|
|
54
|
+
getDuckDbConnection() {
|
|
55
|
+
return {
|
|
56
|
+
run: (sql: string, params?: unknown[]) => {
|
|
57
|
+
embeddingCleanupRuns.push({ sql, params: params ?? [] });
|
|
58
|
+
return embeddingCleanupBlocks
|
|
59
|
+
? new Promise<void>(() => {})
|
|
60
|
+
: Promise.resolve();
|
|
61
|
+
},
|
|
62
|
+
all: async () => [],
|
|
63
|
+
get: async () => null,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
29
67
|
getRepository() {
|
|
30
68
|
return {
|
|
31
69
|
// ===== PROJECT METHODS =====
|
|
@@ -1597,3 +1635,640 @@ describe("resolvePackageLocation", () => {
|
|
|
1597
1635
|
);
|
|
1598
1636
|
});
|
|
1599
1637
|
});
|
|
1638
|
+
|
|
1639
|
+
describe("Environment.getServingPackageCount", () => {
|
|
1640
|
+
const envRoot = path.join(TEMP_DIR_PATH, "serving-count-env");
|
|
1641
|
+
|
|
1642
|
+
beforeEach(() => {
|
|
1643
|
+
rmSync(envRoot, { recursive: true, force: true });
|
|
1644
|
+
mkdirSync(envRoot, { recursive: true });
|
|
1645
|
+
});
|
|
1646
|
+
|
|
1647
|
+
afterEach(() => {
|
|
1648
|
+
rmSync(envRoot, { recursive: true, force: true });
|
|
1649
|
+
});
|
|
1650
|
+
|
|
1651
|
+
it("excludes a package that is registered SERVING but recorded as failed", async () => {
|
|
1652
|
+
// A mount-failed package is seeded SERVING at boot and only pruned by a
|
|
1653
|
+
// later side-effect load; if that prune is skipped (a transient DB or
|
|
1654
|
+
// memory-pressure error), packages= must still not count it, and it
|
|
1655
|
+
// must not overlap load_errors=. The serving count is registered-minus-
|
|
1656
|
+
// failed, so it holds regardless of whether the prune ran.
|
|
1657
|
+
//
|
|
1658
|
+
// Two good and one bad, not one-and-one: this catches both size-only
|
|
1659
|
+
// (would say 3) and a full inversion (would say 1), so the count really
|
|
1660
|
+
// is "registered minus failed".
|
|
1661
|
+
const env = await Environment.create("e", envRoot, []);
|
|
1662
|
+
env.setPackageStatus("good-1", PackageStatus.SERVING);
|
|
1663
|
+
env.setPackageStatus("good-2", PackageStatus.SERVING);
|
|
1664
|
+
env.setPackageStatus("bad", PackageStatus.SERVING);
|
|
1665
|
+
env.setPackageMountError("bad", "location does not exist");
|
|
1666
|
+
|
|
1667
|
+
expect(env.getServingPackageCount()).toBe(2);
|
|
1668
|
+
// Disjoint from the failure set, so packages= + load_errors= never
|
|
1669
|
+
// double-counts "bad".
|
|
1670
|
+
expect(env.getFailedPackages().has("bad")).toBe(true);
|
|
1671
|
+
});
|
|
1672
|
+
});
|
|
1673
|
+
|
|
1674
|
+
describe("GIT_CLONE_OPTIONS", () => {
|
|
1675
|
+
it("clones shallow and single-branch", () => {
|
|
1676
|
+
// The load-bearing content: packages serve the default branch's working
|
|
1677
|
+
// tree only, so history is pure download cost. simple-git serializes
|
|
1678
|
+
// this to `git clone --depth=1 --single-branch`.
|
|
1679
|
+
expect(GIT_CLONE_OPTIONS["--depth"]).toBe(1);
|
|
1680
|
+
expect(GIT_CLONE_OPTIONS).toHaveProperty("--single-branch", null);
|
|
1681
|
+
});
|
|
1682
|
+
});
|
|
1683
|
+
|
|
1684
|
+
describe("stripGitProgressNoise", () => {
|
|
1685
|
+
it("drops progress lines and keeps the error", () => {
|
|
1686
|
+
const message = [
|
|
1687
|
+
"Cloning into '/tmp/x'...",
|
|
1688
|
+
"remote: Counting objects: 45% (9/20)",
|
|
1689
|
+
"Receiving objects: 99% (990/1000)",
|
|
1690
|
+
"fatal: early EOF",
|
|
1691
|
+
"fatal: fetch-pack: invalid index-pack output",
|
|
1692
|
+
].join("\n");
|
|
1693
|
+
expect(stripGitProgressNoise(message)).toBe(
|
|
1694
|
+
"fatal: early EOF\nfatal: fetch-pack: invalid index-pack output",
|
|
1695
|
+
);
|
|
1696
|
+
});
|
|
1697
|
+
|
|
1698
|
+
it("handles carriage-return separated progress spew", () => {
|
|
1699
|
+
// git rewrites progress in place with \r, so the captured stderr is one
|
|
1700
|
+
// long line unless split on \r too.
|
|
1701
|
+
const message =
|
|
1702
|
+
"Receiving objects: 10% (1/10)\rReceiving objects: 50% (5/10)\rfatal: the remote end hung up unexpectedly";
|
|
1703
|
+
expect(stripGitProgressNoise(message)).toBe(
|
|
1704
|
+
"fatal: the remote end hung up unexpectedly",
|
|
1705
|
+
);
|
|
1706
|
+
});
|
|
1707
|
+
|
|
1708
|
+
it("returns the input when every line is progress noise", () => {
|
|
1709
|
+
const message = "Receiving objects: 10% (1/10)";
|
|
1710
|
+
expect(stripGitProgressNoise(message)).toBe(message);
|
|
1711
|
+
});
|
|
1712
|
+
});
|
|
1713
|
+
|
|
1714
|
+
describe("cloneProgressLabel", () => {
|
|
1715
|
+
it("names the environment, repo, and packages", () => {
|
|
1716
|
+
expect(
|
|
1717
|
+
cloneProgressLabel("malloydata/publisher", {
|
|
1718
|
+
environmentName: "examples",
|
|
1719
|
+
packageNames: ["storefront", "governed-analytics"],
|
|
1720
|
+
}),
|
|
1721
|
+
).toBe(
|
|
1722
|
+
"[examples] cloning malloydata/publisher (storefront, governed-analytics)",
|
|
1723
|
+
);
|
|
1724
|
+
});
|
|
1725
|
+
|
|
1726
|
+
it("truncates long package lists", () => {
|
|
1727
|
+
expect(
|
|
1728
|
+
cloneProgressLabel("o/r", {
|
|
1729
|
+
environmentName: "e",
|
|
1730
|
+
packageNames: ["a", "b", "c", "d", "e", "f"],
|
|
1731
|
+
}),
|
|
1732
|
+
).toBe("[e] cloning o/r (a, b, c, +3 more)");
|
|
1733
|
+
});
|
|
1734
|
+
|
|
1735
|
+
it("labels by repo alone without context", () => {
|
|
1736
|
+
// The controller add-package path passes no context.
|
|
1737
|
+
expect(cloneProgressLabel("o/r")).toBe("cloning o/r");
|
|
1738
|
+
});
|
|
1739
|
+
});
|
|
1740
|
+
|
|
1741
|
+
describe("CloneProgressReporter", () => {
|
|
1742
|
+
const event = (stage: string, progress: number, processed = 0, total = 0) =>
|
|
1743
|
+
({ method: "clone", stage, progress, processed, total }) as Parameters<
|
|
1744
|
+
CloneProgressReporter["onProgress"]
|
|
1745
|
+
>[0];
|
|
1746
|
+
|
|
1747
|
+
const fakeOut = (isTTY: boolean) => {
|
|
1748
|
+
const writes: string[] = [];
|
|
1749
|
+
const out = {
|
|
1750
|
+
isTTY,
|
|
1751
|
+
write: (chunk: string) => {
|
|
1752
|
+
writes.push(chunk);
|
|
1753
|
+
return true;
|
|
1754
|
+
},
|
|
1755
|
+
} as unknown as NodeJS.WriteStream;
|
|
1756
|
+
return { writes, out };
|
|
1757
|
+
};
|
|
1758
|
+
|
|
1759
|
+
it("prints a line per stage and per 25-point step off a TTY", () => {
|
|
1760
|
+
const { writes, out } = fakeOut(false);
|
|
1761
|
+
const reporter = new CloneProgressReporter("cloning o/r", out);
|
|
1762
|
+
reporter.onProgress(event("receiving", 0));
|
|
1763
|
+
reporter.onProgress(event("receiving", 10)); // same milestone, suppressed
|
|
1764
|
+
reporter.onProgress(event("receiving", 24)); // same milestone, suppressed
|
|
1765
|
+
reporter.onProgress(event("receiving", 25, 25, 100));
|
|
1766
|
+
reporter.onProgress(event("receiving", 90));
|
|
1767
|
+
reporter.onProgress(event("resolving", 5)); // stage change, printed
|
|
1768
|
+
reporter.done(); // no in-place line to finish
|
|
1769
|
+
expect(writes).toEqual([
|
|
1770
|
+
"cloning o/r: receiving 0%\n",
|
|
1771
|
+
"cloning o/r: receiving 25% (25/100)\n",
|
|
1772
|
+
"cloning o/r: receiving 90%\n",
|
|
1773
|
+
"cloning o/r: resolving 5%\n",
|
|
1774
|
+
]);
|
|
1775
|
+
});
|
|
1776
|
+
|
|
1777
|
+
it("tolerates arbitrary stage strings", () => {
|
|
1778
|
+
// Server-side progress lines parse with stage "remote:".
|
|
1779
|
+
const { writes, out } = fakeOut(false);
|
|
1780
|
+
const reporter = new CloneProgressReporter("cloning o/r", out);
|
|
1781
|
+
reporter.onProgress(event("remote:", 50));
|
|
1782
|
+
expect(writes).toEqual(["cloning o/r: remote: 50%\n"]);
|
|
1783
|
+
});
|
|
1784
|
+
|
|
1785
|
+
it("rewrites one line in place on a TTY and finishes it on done", () => {
|
|
1786
|
+
const { writes, out } = fakeOut(true);
|
|
1787
|
+
const reporter = new CloneProgressReporter("cloning o/r", out);
|
|
1788
|
+
reporter.onProgress(event("receiving", 10, 1, 10));
|
|
1789
|
+
reporter.onProgress(event("receiving", 11, 2, 10));
|
|
1790
|
+
reporter.done();
|
|
1791
|
+
expect(writes).toEqual([
|
|
1792
|
+
"\rcloning o/r: receiving 10% (1/10)\x1b[K",
|
|
1793
|
+
"\rcloning o/r: receiving 11% (2/10)\x1b[K",
|
|
1794
|
+
"\n",
|
|
1795
|
+
]);
|
|
1796
|
+
});
|
|
1797
|
+
|
|
1798
|
+
it("clamps the in-place line to the terminal width", () => {
|
|
1799
|
+
// An auto-wrapped line breaks the \r rewrite: the cursor lands on the
|
|
1800
|
+
// continuation row and every event scrolls a stale fragment. Too
|
|
1801
|
+
// narrow for even the suffix, so a plain head slice applies.
|
|
1802
|
+
const { writes, out } = fakeOut(true);
|
|
1803
|
+
(out as unknown as { columns: number }).columns = 20;
|
|
1804
|
+
const reporter = new CloneProgressReporter("cloning owner/repo", out);
|
|
1805
|
+
reporter.onProgress(event("receiving", 45, 100, 220));
|
|
1806
|
+
reporter.done();
|
|
1807
|
+
expect(writes[0]).toBe(
|
|
1808
|
+
"\r" + "cloning owner/repo:".slice(0, 19) + "\x1b[K",
|
|
1809
|
+
);
|
|
1810
|
+
expect(writes[0].length).toBe(1 + 19 + 3);
|
|
1811
|
+
});
|
|
1812
|
+
|
|
1813
|
+
it("keeps the moving percentage visible when clamping a long label", () => {
|
|
1814
|
+
// The default npx label is longer than an 80-column terminal; the
|
|
1815
|
+
// label shortens, the stage and percentage stay on screen.
|
|
1816
|
+
const { writes, out } = fakeOut(true);
|
|
1817
|
+
(out as unknown as { columns: number }).columns = 60;
|
|
1818
|
+
const reporter = new CloneProgressReporter(
|
|
1819
|
+
"[examples] cloning malloydata/publisher (storefront, governed-analytics)",
|
|
1820
|
+
out,
|
|
1821
|
+
);
|
|
1822
|
+
reporter.onProgress(event("receiving", 45, 100, 220));
|
|
1823
|
+
reporter.done();
|
|
1824
|
+
const visible = writes[0].slice(1, -3); // strip \r and \x1b[K
|
|
1825
|
+
expect(visible.length).toBe(59);
|
|
1826
|
+
expect(visible.endsWith(": receiving 45% (100/220)")).toBe(true);
|
|
1827
|
+
expect(visible).toContain("...");
|
|
1828
|
+
});
|
|
1829
|
+
|
|
1830
|
+
it("resets the milestone when the percentage restarts within a stage", () => {
|
|
1831
|
+
// git's server-side counting and compressing phases both parse as
|
|
1832
|
+
// stage "remote:"; the second starts back at 0%.
|
|
1833
|
+
const { writes, out } = fakeOut(false);
|
|
1834
|
+
const reporter = new CloneProgressReporter("cloning o/r", out);
|
|
1835
|
+
reporter.onProgress(event("remote:", 100));
|
|
1836
|
+
reporter.onProgress(event("remote:", 10));
|
|
1837
|
+
reporter.onProgress(event("remote:", 60));
|
|
1838
|
+
reporter.done();
|
|
1839
|
+
expect(writes).toEqual([
|
|
1840
|
+
"cloning o/r: remote: 100%\n",
|
|
1841
|
+
"cloning o/r: remote: 10%\n",
|
|
1842
|
+
"cloning o/r: remote: 60%\n",
|
|
1843
|
+
]);
|
|
1844
|
+
});
|
|
1845
|
+
|
|
1846
|
+
it("ignores a progress event that arrives after done()", () => {
|
|
1847
|
+
// simple-git can deliver a trailing event after the completion
|
|
1848
|
+
// callback ran done(); it must not re-write the row the finishing
|
|
1849
|
+
// newline scrolled past, nor re-claim TTY ownership.
|
|
1850
|
+
const { writes, out } = fakeOut(true);
|
|
1851
|
+
const reporter = new CloneProgressReporter("cloning o/r", out);
|
|
1852
|
+
reporter.onProgress(event("receiving", 50));
|
|
1853
|
+
reporter.done();
|
|
1854
|
+
const writesAfterDone = writes.length;
|
|
1855
|
+
reporter.onProgress(event("receiving", 90));
|
|
1856
|
+
expect(writes.length).toBe(writesAfterDone);
|
|
1857
|
+
});
|
|
1858
|
+
|
|
1859
|
+
it("only one reporter owns a TTY's in-place line at a time", () => {
|
|
1860
|
+
// Environments load concurrently at boot; a second clone must not
|
|
1861
|
+
// rewrite the first one's line. It falls back to milestone lines.
|
|
1862
|
+
const { writes, out } = fakeOut(true);
|
|
1863
|
+
const first = new CloneProgressReporter("cloning a/a", out);
|
|
1864
|
+
const second = new CloneProgressReporter("cloning b/b", out);
|
|
1865
|
+
first.onProgress(event("receiving", 10));
|
|
1866
|
+
second.onProgress(event("receiving", 20));
|
|
1867
|
+
first.onProgress(event("receiving", 30));
|
|
1868
|
+
first.done();
|
|
1869
|
+
// After the owner finishes, the line is free to claim again.
|
|
1870
|
+
second.onProgress(event("receiving", 90));
|
|
1871
|
+
second.done();
|
|
1872
|
+
expect(writes).toEqual([
|
|
1873
|
+
"\rcloning a/a: receiving 10%\x1b[K",
|
|
1874
|
+
// The non-owner takes over the row cleanly before scrolling it, so
|
|
1875
|
+
// its milestone never concatenates onto the owner's in-place text.
|
|
1876
|
+
"\rcloning b/b: receiving 20%\x1b[K\n",
|
|
1877
|
+
"\rcloning a/a: receiving 30%\x1b[K",
|
|
1878
|
+
"\n",
|
|
1879
|
+
"\rcloning b/b: receiving 90%\x1b[K",
|
|
1880
|
+
"\n",
|
|
1881
|
+
]);
|
|
1882
|
+
});
|
|
1883
|
+
});
|
|
1884
|
+
|
|
1885
|
+
describe("formatReadinessLine", () => {
|
|
1886
|
+
const SAVED = ["PUBLISHER_HOST", "PUBLISHER_PORT", "MCP_PORT"] as const;
|
|
1887
|
+
let saved: Record<string, string | undefined>;
|
|
1888
|
+
|
|
1889
|
+
beforeEach(() => {
|
|
1890
|
+
saved = {};
|
|
1891
|
+
for (const key of SAVED) {
|
|
1892
|
+
saved[key] = process.env[key];
|
|
1893
|
+
delete process.env[key];
|
|
1894
|
+
}
|
|
1895
|
+
});
|
|
1896
|
+
|
|
1897
|
+
afterEach(() => {
|
|
1898
|
+
for (const key of SAVED) {
|
|
1899
|
+
if (saved[key] === undefined) delete process.env[key];
|
|
1900
|
+
else process.env[key] = saved[key];
|
|
1901
|
+
}
|
|
1902
|
+
});
|
|
1903
|
+
|
|
1904
|
+
it("uses the server defaults and displays the wildcard bind as localhost", () => {
|
|
1905
|
+
expect(
|
|
1906
|
+
formatReadinessLine({ environments: 1, packages: 3, loadErrors: 0 }),
|
|
1907
|
+
).toBe(
|
|
1908
|
+
"PUBLISHER_READY url=http://localhost:4000 mcp=http://localhost:4040 " +
|
|
1909
|
+
"environments=1 packages=3 load_errors=0",
|
|
1910
|
+
);
|
|
1911
|
+
});
|
|
1912
|
+
|
|
1913
|
+
it("brackets an IPv6 literal host so the URL is dialable", () => {
|
|
1914
|
+
process.env.PUBLISHER_HOST = "::1";
|
|
1915
|
+
expect(
|
|
1916
|
+
formatReadinessLine({ environments: 1, packages: 1, loadErrors: 0 }),
|
|
1917
|
+
).toBe(
|
|
1918
|
+
"PUBLISHER_READY url=http://[::1]:4000 mcp=http://[::1]:4040 " +
|
|
1919
|
+
"environments=1 packages=1 load_errors=0",
|
|
1920
|
+
);
|
|
1921
|
+
});
|
|
1922
|
+
|
|
1923
|
+
it("displays the :: wildcard bind as localhost", () => {
|
|
1924
|
+
process.env.PUBLISHER_HOST = "::";
|
|
1925
|
+
expect(
|
|
1926
|
+
formatReadinessLine({ environments: 1, packages: 1, loadErrors: 0 }),
|
|
1927
|
+
).toBe(
|
|
1928
|
+
"PUBLISHER_READY url=http://localhost:4000 mcp=http://localhost:4040 " +
|
|
1929
|
+
"environments=1 packages=1 load_errors=0",
|
|
1930
|
+
);
|
|
1931
|
+
});
|
|
1932
|
+
|
|
1933
|
+
it("respects the flag-derived env vars", () => {
|
|
1934
|
+
// parseArgs writes --host/--port/--mcp_port here before the store exists.
|
|
1935
|
+
process.env.PUBLISHER_HOST = "127.0.0.1";
|
|
1936
|
+
process.env.PUBLISHER_PORT = "4321";
|
|
1937
|
+
process.env.MCP_PORT = "4361";
|
|
1938
|
+
expect(
|
|
1939
|
+
formatReadinessLine({ environments: 2, packages: 5, loadErrors: 1 }),
|
|
1940
|
+
).toBe(
|
|
1941
|
+
"PUBLISHER_READY url=http://127.0.0.1:4321 mcp=http://127.0.0.1:4361 " +
|
|
1942
|
+
"environments=2 packages=5 load_errors=1",
|
|
1943
|
+
);
|
|
1944
|
+
});
|
|
1945
|
+
});
|
|
1946
|
+
|
|
1947
|
+
describe("readiness line emission", () => {
|
|
1948
|
+
const readinessRoot = path.join(TEMP_DIR_PATH, "readiness-line-test");
|
|
1949
|
+
let stderrWrites: string[];
|
|
1950
|
+
let stderrSpy: ReturnType<typeof spyOn>;
|
|
1951
|
+
|
|
1952
|
+
const readyLines = () =>
|
|
1953
|
+
stderrWrites
|
|
1954
|
+
.join("")
|
|
1955
|
+
.split("\n")
|
|
1956
|
+
.filter((line) => line.startsWith("PUBLISHER_READY"));
|
|
1957
|
+
|
|
1958
|
+
const writePackage = (name: string) => {
|
|
1959
|
+
const dir = path.join(readinessRoot, "src-packages", name);
|
|
1960
|
+
mkdirSync(dir, { recursive: true });
|
|
1961
|
+
writeFileSync(path.join(dir, "publisher.json"), JSON.stringify({ name }));
|
|
1962
|
+
return dir;
|
|
1963
|
+
};
|
|
1964
|
+
|
|
1965
|
+
beforeEach(() => {
|
|
1966
|
+
if (existsSync(readinessRoot)) {
|
|
1967
|
+
rmSync(readinessRoot, { recursive: true, force: true });
|
|
1968
|
+
}
|
|
1969
|
+
mkdirSync(readinessRoot, { recursive: true });
|
|
1970
|
+
mock.module("../config", () => ({
|
|
1971
|
+
isPublisherConfigFrozen: () => false,
|
|
1972
|
+
}));
|
|
1973
|
+
stderrWrites = [];
|
|
1974
|
+
stderrSpy = spyOn(process.stderr, "write").mockImplementation(
|
|
1975
|
+
(chunk: string | Uint8Array) => {
|
|
1976
|
+
stderrWrites.push(String(chunk));
|
|
1977
|
+
return true;
|
|
1978
|
+
},
|
|
1979
|
+
);
|
|
1980
|
+
});
|
|
1981
|
+
|
|
1982
|
+
afterEach(() => {
|
|
1983
|
+
stderrSpy.mockRestore();
|
|
1984
|
+
rmSync(readinessRoot, { recursive: true, force: true });
|
|
1985
|
+
});
|
|
1986
|
+
|
|
1987
|
+
it("emits exactly one line with counts after a successful boot", async () => {
|
|
1988
|
+
const pkgA = writePackage("pkg-a");
|
|
1989
|
+
const pkgB = writePackage("pkg-b");
|
|
1990
|
+
writeFileSync(
|
|
1991
|
+
path.join(readinessRoot, "publisher.config.json"),
|
|
1992
|
+
JSON.stringify({
|
|
1993
|
+
frozenConfig: false,
|
|
1994
|
+
environments: [
|
|
1995
|
+
{
|
|
1996
|
+
name: "readiness-env",
|
|
1997
|
+
packages: [
|
|
1998
|
+
{ name: "pkg-a", location: pkgA },
|
|
1999
|
+
{ name: "pkg-b", location: pkgB },
|
|
2000
|
+
],
|
|
2001
|
+
connections: [],
|
|
2002
|
+
},
|
|
2003
|
+
],
|
|
2004
|
+
}),
|
|
2005
|
+
);
|
|
2006
|
+
|
|
2007
|
+
const store = new EnvironmentStore(readinessRoot);
|
|
2008
|
+
await store.finishedInitialization;
|
|
2009
|
+
|
|
2010
|
+
const lines = readyLines();
|
|
2011
|
+
expect(lines).toHaveLength(1);
|
|
2012
|
+
expect(lines[0]).toContain("environments=1");
|
|
2013
|
+
expect(lines[0]).toContain("packages=2");
|
|
2014
|
+
expect(lines[0]).toContain("load_errors=0");
|
|
2015
|
+
expect(lines[0]).toMatch(
|
|
2016
|
+
/^PUBLISHER_READY url=http:\/\/\S+ mcp=http:\/\/\S+ environments=\d+ packages=\d+ load_errors=\d+$/,
|
|
2017
|
+
);
|
|
2018
|
+
});
|
|
2019
|
+
|
|
2020
|
+
it("counts a mount failure in load_errors and still emits", async () => {
|
|
2021
|
+
// Post-#903 a bad location drops the package, not the boot; the line
|
|
2022
|
+
// must report the failure rather than stay silent or claim a clean load.
|
|
2023
|
+
const pkgA = writePackage("pkg-a");
|
|
2024
|
+
writeFileSync(
|
|
2025
|
+
path.join(readinessRoot, "publisher.config.json"),
|
|
2026
|
+
JSON.stringify({
|
|
2027
|
+
frozenConfig: false,
|
|
2028
|
+
environments: [
|
|
2029
|
+
{
|
|
2030
|
+
name: "readiness-env",
|
|
2031
|
+
packages: [
|
|
2032
|
+
{ name: "pkg-a", location: pkgA },
|
|
2033
|
+
{
|
|
2034
|
+
name: "pkg-missing",
|
|
2035
|
+
location: path.join(readinessRoot, "does-not-exist"),
|
|
2036
|
+
},
|
|
2037
|
+
],
|
|
2038
|
+
connections: [],
|
|
2039
|
+
},
|
|
2040
|
+
],
|
|
2041
|
+
}),
|
|
2042
|
+
);
|
|
2043
|
+
|
|
2044
|
+
const store = new EnvironmentStore(readinessRoot);
|
|
2045
|
+
await store.finishedInitialization;
|
|
2046
|
+
|
|
2047
|
+
const lines = readyLines();
|
|
2048
|
+
expect(lines).toHaveLength(1);
|
|
2049
|
+
expect(lines[0]).toContain("environments=1");
|
|
2050
|
+
// packages= is what the server actually serves: the boot-time database
|
|
2051
|
+
// sync prunes a package whose load failed (environment.ts getPackage
|
|
2052
|
+
// catch), so the failed one appears in load_errors, not in packages.
|
|
2053
|
+
expect(lines[0]).toContain("packages=1");
|
|
2054
|
+
expect(lines[0]).toContain("load_errors=1");
|
|
2055
|
+
});
|
|
2056
|
+
|
|
2057
|
+
it("counts a failed environment in load_errors", async () => {
|
|
2058
|
+
// An environment that fails initialization outright lands in
|
|
2059
|
+
// failedEnvironments, not in environments; the line must count it
|
|
2060
|
+
// alongside per-package failures. Failure lever: a FILE squatting the
|
|
2061
|
+
// environment's publisher_data directory path makes the initial mkdir
|
|
2062
|
+
// throw before any package mounts.
|
|
2063
|
+
const pkgA = writePackage("pkg-a");
|
|
2064
|
+
mkdirSync(path.join(readinessRoot, "publisher_data"), {
|
|
2065
|
+
recursive: true,
|
|
2066
|
+
});
|
|
2067
|
+
writeFileSync(path.join(readinessRoot, "publisher_data", "bad-env"), "");
|
|
2068
|
+
writeFileSync(
|
|
2069
|
+
path.join(readinessRoot, "publisher.config.json"),
|
|
2070
|
+
JSON.stringify({
|
|
2071
|
+
frozenConfig: false,
|
|
2072
|
+
environments: [
|
|
2073
|
+
{
|
|
2074
|
+
name: "good-env",
|
|
2075
|
+
packages: [{ name: "pkg-a", location: pkgA }],
|
|
2076
|
+
connections: [],
|
|
2077
|
+
},
|
|
2078
|
+
{
|
|
2079
|
+
name: "bad-env",
|
|
2080
|
+
packages: [{ name: "pkg-a", location: pkgA }],
|
|
2081
|
+
connections: [],
|
|
2082
|
+
},
|
|
2083
|
+
],
|
|
2084
|
+
}),
|
|
2085
|
+
);
|
|
2086
|
+
|
|
2087
|
+
const store = new EnvironmentStore(readinessRoot);
|
|
2088
|
+
await store.finishedInitialization;
|
|
2089
|
+
|
|
2090
|
+
const lines = readyLines();
|
|
2091
|
+
expect(lines).toHaveLength(1);
|
|
2092
|
+
expect(lines[0]).toContain("environments=1");
|
|
2093
|
+
expect(lines[0]).toContain("packages=1");
|
|
2094
|
+
expect(lines[0]).toContain("load_errors=1");
|
|
2095
|
+
});
|
|
2096
|
+
});
|
|
2097
|
+
|
|
2098
|
+
describe("parseGitHubUrl", () => {
|
|
2099
|
+
// The exact regexes parseGitHubUrl replaced, kept here ONLY as a
|
|
2100
|
+
// behavioral oracle. The product code no longer runs them (they were
|
|
2101
|
+
// flagged as polynomial-ReDoS); the fuzz below proves the linear
|
|
2102
|
+
// string parser returns identical results. Never feed these long input.
|
|
2103
|
+
const oldParseGitHubUrl = (
|
|
2104
|
+
githubUrl: string,
|
|
2105
|
+
): { owner: string; repoName: string; packagePath?: string } | null => {
|
|
2106
|
+
const httpsRegex =
|
|
2107
|
+
/github\.com\/(?<owner>[^/]+)\/(?<repoName>[^/]+)(?<packagePath>\/[^/]+)*/;
|
|
2108
|
+
const httpsMatch = githubUrl.match(httpsRegex);
|
|
2109
|
+
if (httpsMatch) {
|
|
2110
|
+
const { owner, repoName, packagePath } = httpsMatch.groups!;
|
|
2111
|
+
return { owner, repoName, packagePath };
|
|
2112
|
+
}
|
|
2113
|
+
const sshRegex =
|
|
2114
|
+
/git@github\.com:(?<owner>[^/]+)\/(?<repoName>[^/\s]+?)(?:\.git)?(?<packagePath>\/[^/]+)*$/;
|
|
2115
|
+
const sshMatch = githubUrl.match(sshRegex);
|
|
2116
|
+
if (sshMatch) {
|
|
2117
|
+
const { owner, repoName, packagePath } = sshMatch.groups!;
|
|
2118
|
+
return { owner, repoName, packagePath };
|
|
2119
|
+
}
|
|
2120
|
+
return null;
|
|
2121
|
+
};
|
|
2122
|
+
|
|
2123
|
+
// Absent and undefined packagePath are the same result; normalize so the
|
|
2124
|
+
// comparison ignores that representational difference.
|
|
2125
|
+
const norm = (r: ReturnType<typeof parseGitHubUrl>) =>
|
|
2126
|
+
r === null
|
|
2127
|
+
? null
|
|
2128
|
+
: {
|
|
2129
|
+
owner: r.owner,
|
|
2130
|
+
repoName: r.repoName,
|
|
2131
|
+
packagePath: r.packagePath ?? null,
|
|
2132
|
+
};
|
|
2133
|
+
|
|
2134
|
+
it("matches the documented cases", () => {
|
|
2135
|
+
expect(
|
|
2136
|
+
parseGitHubUrl(
|
|
2137
|
+
"https://github.com/malloydata/publisher/tree/main/examples/storefront",
|
|
2138
|
+
),
|
|
2139
|
+
).toEqual({
|
|
2140
|
+
owner: "malloydata",
|
|
2141
|
+
repoName: "publisher",
|
|
2142
|
+
// Preserved quirk: only the LAST path segment.
|
|
2143
|
+
packagePath: "/storefront",
|
|
2144
|
+
});
|
|
2145
|
+
expect(parseGitHubUrl("https://github.com/owner/repo")).toEqual({
|
|
2146
|
+
owner: "owner",
|
|
2147
|
+
repoName: "repo",
|
|
2148
|
+
});
|
|
2149
|
+
expect(parseGitHubUrl("git@github.com:owner/repo.git")).toEqual({
|
|
2150
|
+
owner: "owner",
|
|
2151
|
+
repoName: "repo",
|
|
2152
|
+
});
|
|
2153
|
+
expect(parseGitHubUrl("https://gitlab.com/owner/repo")).toBeNull();
|
|
2154
|
+
});
|
|
2155
|
+
|
|
2156
|
+
it("is identical to the old regexes across a fuzzed input space", () => {
|
|
2157
|
+
const prefixes = [
|
|
2158
|
+
"https://github.com/",
|
|
2159
|
+
"http://github.com/",
|
|
2160
|
+
"github.com/",
|
|
2161
|
+
"git@github.com:",
|
|
2162
|
+
"ssh://git@github.com:",
|
|
2163
|
+
"before github.com/",
|
|
2164
|
+
"",
|
|
2165
|
+
"https://gitlab.com/",
|
|
2166
|
+
];
|
|
2167
|
+
const owners = ["owner", "o", "", "own er", "a.b"];
|
|
2168
|
+
const repos = [
|
|
2169
|
+
"repo",
|
|
2170
|
+
"r",
|
|
2171
|
+
"repo.git",
|
|
2172
|
+
".git",
|
|
2173
|
+
"a.git",
|
|
2174
|
+
"re po",
|
|
2175
|
+
"",
|
|
2176
|
+
"repo.gitx",
|
|
2177
|
+
];
|
|
2178
|
+
const tails = [
|
|
2179
|
+
"",
|
|
2180
|
+
"/tree/main/sub",
|
|
2181
|
+
"/tree/main",
|
|
2182
|
+
"/a/b/c",
|
|
2183
|
+
"/",
|
|
2184
|
+
"//",
|
|
2185
|
+
"/a//b",
|
|
2186
|
+
"/sub dir",
|
|
2187
|
+
"/tree/branch/a/b",
|
|
2188
|
+
];
|
|
2189
|
+
let cases = 0;
|
|
2190
|
+
for (const prefix of prefixes) {
|
|
2191
|
+
for (const owner of owners) {
|
|
2192
|
+
for (const repo of repos) {
|
|
2193
|
+
for (const tail of tails) {
|
|
2194
|
+
const input = `${prefix}${owner}/${repo}${tail}`;
|
|
2195
|
+
expect(norm(parseGitHubUrl(input))).toEqual(
|
|
2196
|
+
norm(oldParseGitHubUrl(input)),
|
|
2197
|
+
);
|
|
2198
|
+
cases++;
|
|
2199
|
+
}
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
expect(cases).toBeGreaterThan(2000);
|
|
2204
|
+
});
|
|
2205
|
+
|
|
2206
|
+
it("runs in linear time on adversarial input (no ReDoS)", () => {
|
|
2207
|
+
// The payloads CodeQL flagged: a long run that made the old
|
|
2208
|
+
// `(\/[^/]+)*` regexes backtrack polynomially. The string parser is
|
|
2209
|
+
// linear, so this returns effectively instantly.
|
|
2210
|
+
const start = performance.now();
|
|
2211
|
+
parseGitHubUrl("git@github.com:" + ".".repeat(200000));
|
|
2212
|
+
parseGitHubUrl("git@github.com:" + "a/".repeat(200000));
|
|
2213
|
+
parseGitHubUrl(
|
|
2214
|
+
"git@github.com:./!/" + "git@github.com:./!/".repeat(5000),
|
|
2215
|
+
);
|
|
2216
|
+
parseGitHubUrl("https://github.com/" + "a/".repeat(200000));
|
|
2217
|
+
expect(performance.now() - start).toBeLessThan(500);
|
|
2218
|
+
});
|
|
2219
|
+
});
|
|
2220
|
+
|
|
2221
|
+
describe("EnvironmentStore embeddings cleanup wiring", () => {
|
|
2222
|
+
const serverRootPath = path.join(TEMP_DIR_PATH, "cleanup-wiring-tests");
|
|
2223
|
+
|
|
2224
|
+
beforeEach(() => {
|
|
2225
|
+
if (existsSync(serverRootPath)) {
|
|
2226
|
+
rmSync(serverRootPath, { recursive: true, force: true });
|
|
2227
|
+
}
|
|
2228
|
+
mkdirSync(serverRootPath, { recursive: true });
|
|
2229
|
+
mockDbEnvironments = [];
|
|
2230
|
+
embeddingCleanupRuns.length = 0;
|
|
2231
|
+
embeddingCleanupBlocks = false;
|
|
2232
|
+
_resetEmbeddingIndexStateForTests();
|
|
2233
|
+
});
|
|
2234
|
+
|
|
2235
|
+
afterEach(() => {
|
|
2236
|
+
embeddingCleanupBlocks = false;
|
|
2237
|
+
_resetEmbeddingIndexStateForTests();
|
|
2238
|
+
if (existsSync(serverRootPath)) {
|
|
2239
|
+
rmSync(serverRootPath, { recursive: true, force: true });
|
|
2240
|
+
}
|
|
2241
|
+
});
|
|
2242
|
+
|
|
2243
|
+
it("deletePackageFromDatabase fires cleanup without awaiting it, even with no metadata row", async () => {
|
|
2244
|
+
const store = new EnvironmentStore(serverRootPath);
|
|
2245
|
+
await store.finishedInitialization;
|
|
2246
|
+
|
|
2247
|
+
// The cleanup's SQL never resolves: were the store awaiting it,
|
|
2248
|
+
// this call would hang past the test timeout. The mock repository
|
|
2249
|
+
// reports no environment row, so resolving promptly ALSO pins
|
|
2250
|
+
// that the cleanup runs before the missing-row early return.
|
|
2251
|
+
embeddingCleanupBlocks = true;
|
|
2252
|
+
await store.deletePackageFromDatabase("wiring-env", "wiring-pkg");
|
|
2253
|
+
|
|
2254
|
+
const deletes = embeddingCleanupRuns.filter((r) =>
|
|
2255
|
+
r.sql.includes("DELETE FROM entity_embeddings"),
|
|
2256
|
+
);
|
|
2257
|
+
expect(deletes.length).toBe(1);
|
|
2258
|
+
expect(deletes[0].params).toEqual(["wiring-env", "wiring-pkg"]);
|
|
2259
|
+
});
|
|
2260
|
+
|
|
2261
|
+
it("deleteEnvironmentFromDatabase fires the env-wide cleanup without awaiting it", async () => {
|
|
2262
|
+
const store = new EnvironmentStore(serverRootPath);
|
|
2263
|
+
await store.finishedInitialization;
|
|
2264
|
+
|
|
2265
|
+
embeddingCleanupBlocks = true;
|
|
2266
|
+
await store.deleteEnvironmentFromDatabase("wiring-env");
|
|
2267
|
+
|
|
2268
|
+
const deletes = embeddingCleanupRuns.filter((r) =>
|
|
2269
|
+
r.sql.includes("DELETE FROM entity_embeddings"),
|
|
2270
|
+
);
|
|
2271
|
+
expect(deletes.length).toBe(1);
|
|
2272
|
+
expect(deletes[0].params).toEqual(["wiring-env"]);
|
|
2273
|
+
});
|
|
2274
|
+
});
|