ltcai 11.5.2 → 11.6.0

package/scripts/release_screen_claims.json

@@ -21,6 +21,27 @@
     "specific, not less."
   ],
 
+  "11.6.0": {
+    "_why": [
+      "One Door release: the product server moved from Python/FastAPI to the",
+      "Rust lattice-host, every write moved into the native KG write engine,",
+      "and the Python package became a 28-route AI worker. The frontend",
+      "bundle is untouched — the same React sources, the same routes, the",
+      "same copy — so no captured screen changes on purpose, and all",
+      "thirteen are measured but not gated.",
+      "",
+      "Worth stating because it is the one thing that could surprise a",
+      "reader: the screens are served by a different process now, and the",
+      "capture still runs against tests/visual/mock_server.cjs, so these",
+      "shots prove the frontend did not move, not that the new server",
+      "renders them. The live proof for that is the gateway integration",
+      "test and the release smoke, not this gate.",
+      "",
+      "Known jitter: 05-memory-graph moves ~2-3% between captures from the",
+      "cytoscape layout (11.3.0-11.5.2 observed)."
+    ]
+  },
+
   "11.5.2": {
     "_why": [
       "Tight Ship release: deletions, consolidation and front-door fixes —",

package/scripts/run_integration_tests.mjs

@@ -1,6 +1,19 @@
 #!/usr/bin/env node
-import { spawn } from "node:child_process";
-import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
+/**
+ * Start an isolated Lattice product server and run the live-server integration
+ * suite (`tests/integration/`) against it.
+ *
+ * v11.6.0 "One Door": the front door is `lattice-host`, which supervises the
+ * Python AI worker (`latticeai.worker_app`) on an internal port behind it.
+ * This runner used to start `uvicorn server:app` — a root `server.py` the
+ * release deleted — so it now boots the same binary the product ships, exactly
+ * the way scripts/run_sidecar_e2e.mjs does.
+ *
+ * Isolation rule, unchanged: never touch the developer's real HOME / ~/.ltcai /
+ * Brain vault / agent workspace.
+ */
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
@@ -16,7 +29,60 @@ if (!/^\d+$/.test(port) || Number(port) < 1 || Number(port) > 65535) {
 // supplied live server through LTCAI_TEST_BASE_URL.
 const baseUrl = `http://${host}:${port}`;
 const venvPython = join(process.cwd(), ".venv", "bin", "python");
-const python = process.env.PYTHON || (existsSync(venvPython) ? venvPython : "python");
+const requestedPython = process.env.PYTHON || (existsSync(venvPython) ? venvPython : "python3");
+
+/**
+ * The interpreter, as an absolute path.
+ *
+ * `LTCAI_PYTHON` is handed to `lattice-host`, which spawns the worker with a
+ * `PATH` **prefixed** by `/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin`
+ * (supervisor/worker_env.rs — GUI processes inherit a minimal PATH). A bare
+ * name like `python3` is therefore looked up against a different PATH than the
+ * one this runner validated it on: on GitHub Actions the interpreter that
+ * `pip install -e "."` populated lives under /opt/hostedtoolcache, but the
+ * child finds /usr/bin/python3 first and dies with "No module named uvicorn"
+ * — the worker exit 1 that turned every proxied route into a 502. Resolving to
+ * `sys.executable` means the interpreter cannot change identity when it
+ * crosses the process boundary.
+ */
+function resolveInterpreter(candidate) {
+  const probe = spawnSync(candidate, ["-c", "import sys; print(sys.executable)"], {
+    encoding: "utf8",
+  });
+  const resolved = probe.status === 0 ? String(probe.stdout || "").trim() : "";
+  // An unresolvable candidate is passed through untouched so the supervisor
+  // reports the failure in its own words rather than this helper's.
+  return resolved && existsSync(resolved) ? resolved : candidate;
+}
+
+const python = resolveInterpreter(requestedPython);
+
+/**
+ * The `lattice-host` binary this run should exercise.
+ *
+ * An explicit pin wins (same names bin/ltcai.js accepts). Otherwise the
+ * current tree is built: a stale `rust/target/release/lattice-host` left over
+ * from an older version would otherwise be tested instead of this checkout.
+ * `cargo build` is a no-op when the tree is already built.
+ */
+function resolveHostBinary() {
+  const pinned = process.env.LATTICEAI_HOST_BIN || process.env.LTCAI_HOST;
+  if (pinned) {
+    if (!existsSync(pinned)) throw new Error(`lattice-host not found at ${pinned}`);
+    return pinned;
+  }
+  const built = join(process.cwd(), "rust", "target", "debug", "lattice-host");
+  if (process.env.LTCAI_SKIP_HOST_BUILD !== "1") {
+    console.log("[integration] cargo build -p lattice-host");
+    const build = spawnSync("cargo", ["build", "-p", "lattice-host"], {
+      stdio: "inherit",
+      cwd: join(process.cwd(), "rust"),
+    });
+    if (build.status !== 0) throw new Error("cargo build -p lattice-host failed");
+  }
+  if (!existsSync(built)) throw new Error(`lattice-host missing: ${built}`);
+  return built;
+}
 
 // Integration tests must never discover or mutate the developer's real HOME,
 // ~/.ltcai, Brain vault, or agent workspace. Keep every user-state path under
@@ -52,6 +118,9 @@ const isolatedEnv = {
   LATTICEAI_BRAIN_DIR: sandbox.brain,
   LATTICEAI_AGENT_ROOT: sandbox.agent,
   LATTICEAI_OBSIDIAN_VAULT_DIR: sandbox.vault,
+  // The supervisor resolves the worker interpreter itself; naming it keeps
+  // the choice the same one this runner made (repo .venv, else python3).
+  LTCAI_PYTHON: python,
   LATTICEAI_STORAGE_ENGINE: "sqlite",
   LATTICEAI_POSTGRES_DSN: "",
   LATTICEAI_REQUIRE_AUTH: "false",
@@ -74,6 +143,32 @@ function cleanupSandbox() {
 }
 process.once("exit", cleanupSandbox);
 
+/**
+ * The supervised worker's own stdout/stderr.
+ *
+ * `lattice-host` redirects them into `$HOME/.ltcai/desktop-sidecar.log` and
+ * `.err.log` (supervisor/worker_env.rs), and this runner rewrites HOME into a
+ * sandbox it deletes on the way out — so a worker that dies takes the only
+ * explanation with it and CI shows nothing but a 502. Print the tail before
+ * cleanup: those lines are the difference between "the gateway returned 502"
+ * and "that interpreter has no uvicorn".
+ */
+function dumpWorkerLogs() {
+  for (const name of ["desktop-sidecar.err.log", "desktop-sidecar.log"]) {
+    const path = join(sandbox.home, ".ltcai", name);
+    if (!existsSync(path)) {
+      console.error(`[integration] ${name}: not written`);
+      continue;
+    }
+    const lines = readFileSync(path, "utf8").replace(/\n+$/, "").split("\n");
+    const tail = lines.slice(-80);
+    console.error(
+      `[integration] ---- ${name} (last ${tail.length} of ${lines.length} lines) ----`,
+    );
+    console.error(tail.join("\n") || "(empty)");
+  }
+}
+
 function run(command, args, options = {}) {
   return new Promise((resolve, reject) => {
     const child = spawn(command, args, {
@@ -90,7 +185,10 @@ function delay(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-async function waitForHealth(url, timeoutMs = 30000) {
+// 120 s, not 30: the gate now covers two boots — this process binds the
+// gateway immediately, but /health is proxied and only answers once the
+// supervised Python worker has imported and bound its own port.
+async function waitForHealth(url, timeoutMs = 120000) {
   const deadline = Date.now() + timeoutMs;
   let lastError = "";
   while (Date.now() < deadline) {
@@ -123,31 +221,38 @@ function stop(child) {
   });
 }
 
-const server = spawn(
-  python,
-  ["-m", "uvicorn", "server:app", "--host", host, "--port", port],
-  {
-    cwd: process.cwd(),
-    env: isolatedEnv,
-    stdio: ["ignore", "pipe", "pipe"],
-  },
-);
+async function main() {
+  console.log(`[integration] sandbox ${sandboxRoot}`);
+  console.log(`[integration] worker interpreter ${python}`);
+  const hostBinary = resolveHostBinary();
+  console.log(`[integration] starting ${hostBinary} at ${baseUrl}`);
+  // The shipped topology: lattice-host serves the product on `port` and
+  // supervises the Python worker on a free port behind it. Do not
+  // double-append /health — the wait helper already probes `${url}/health`,
+  // which the gateway proxies to the worker, so a 200 there means both
+  // processes are up.
+  const server = spawn(
+    hostBinary,
+    ["--port", port],
+    { stdio: "inherit", env: isolatedEnv, cwd: process.cwd() },
+  );
 
-server.stdout.on("data", (chunk) => process.stdout.write(chunk));
-server.stderr.on("data", (chunk) => process.stderr.write(chunk));
-
-let exitCode = 1;
-try {
-  await waitForHealth(baseUrl);
-  const result = await run(python, ["-m", "pytest", "tests/integration/", "-v"], {
-    env: { LTCAI_TEST_BASE_URL: baseUrl },
-  });
-  exitCode = result.code || 0;
-} catch (error) {
-  console.error(String(error?.message || error));
-} finally {
-  await stop(server);
-  cleanupSandbox();
+  let exitCode = 1;
+  try {
+    await waitForHealth(baseUrl);
+    console.log("[integration] front door healthy — running pytest");
+    const result = await run(python, ["-m", "pytest", "tests/integration/", "-v"], {
+      env: { LTCAI_TEST_BASE_URL: baseUrl },
+    });
+    exitCode = result.code || 0;
+  } catch (error) {
+    console.error(String(error?.message || error));
+  } finally {
+    if (exitCode !== 0) dumpWorkerLogs();
+    await stop(server);
+    cleanupSandbox();
+  }
+  process.exit(exitCode);
 }
 
-process.exit(exitCode);
+main();

package/scripts/run_sidecar_e2e.mjs

@@ -1,13 +1,18 @@
 #!/usr/bin/env node
 /**
- * Start an isolated Lattice sidecar and run Playwright first-value E2E
+ * Start an isolated Lattice product server and run Playwright first-value E2E
  * against it (Wave 3.2 residual / v9.9.5).
  *
+ * v11.6.0 "One Door": the front door is `lattice-host`, which supervises the
+ * Python AI worker (`latticeai.worker_app`) on an internal port behind it.
+ * This runner used to start `uvicorn server:app` — a root `server.py` the
+ * release deleted — so it now boots the same binary the product ships.
+ *
  * Mirrors scripts/run_integration_tests.mjs isolation rules: never touch
  * the developer's real HOME / ~/.ltcai / Brain vault.
  */
-import { spawn } from "node:child_process";
-import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
+import { spawn, spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
@@ -21,7 +26,60 @@ if (!/^\d+$/.test(port) || Number(port) < 1 || Number(port) > 65535) {
 }
 const baseUrl = `http://${host}:${port}`;
 const venvPython = join(process.cwd(), ".venv", "bin", "python");
-const python = process.env.PYTHON || (existsSync(venvPython) ? venvPython : "python3");
+const requestedPython = process.env.PYTHON || (existsSync(venvPython) ? venvPython : "python3");
+
+/**
+ * The interpreter, as an absolute path.
+ *
+ * `LTCAI_PYTHON` is handed to `lattice-host`, which spawns the worker with a
+ * `PATH` **prefixed** by `/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin`
+ * (supervisor/worker_env.rs — GUI processes inherit a minimal PATH). A bare
+ * name like `python3` is therefore looked up against a different PATH than the
+ * one this runner validated it on: on GitHub Actions the interpreter that
+ * `pip install -e "."` populated lives under /opt/hostedtoolcache, but the
+ * child finds /usr/bin/python3 first and dies with "No module named uvicorn"
+ * — the worker exit 1 that turned every proxied route into a 502. Resolving to
+ * `sys.executable` means the interpreter cannot change identity when it
+ * crosses the process boundary.
+ */
+function resolveInterpreter(candidate) {
+  const probe = spawnSync(candidate, ["-c", "import sys; print(sys.executable)"], {
+    encoding: "utf8",
+  });
+  const resolved = probe.status === 0 ? String(probe.stdout || "").trim() : "";
+  // An unresolvable candidate is passed through untouched so the supervisor
+  // reports the failure in its own words rather than this helper's.
+  return resolved && existsSync(resolved) ? resolved : candidate;
+}
+
+const python = resolveInterpreter(requestedPython);
+
+/**
+ * The `lattice-host` binary this run should exercise.
+ *
+ * An explicit pin wins (same names bin/ltcai.js accepts). Otherwise the
+ * current tree is built: a stale `rust/target/release/lattice-host` left over
+ * from an older version would otherwise be tested instead of this checkout.
+ * `cargo build` is a no-op when the tree is already built.
+ */
+function resolveHostBinary() {
+  const pinned = process.env.LATTICEAI_HOST_BIN || process.env.LTCAI_HOST;
+  if (pinned) {
+    if (!existsSync(pinned)) throw new Error(`lattice-host not found at ${pinned}`);
+    return pinned;
+  }
+  const built = join(process.cwd(), "rust", "target", "debug", "lattice-host");
+  if (process.env.LTCAI_SKIP_HOST_BUILD !== "1") {
+    console.log("[e2e] cargo build -p lattice-host");
+    const build = spawnSync("cargo", ["build", "-p", "lattice-host"], {
+      stdio: "inherit",
+      cwd: join(process.cwd(), "rust"),
+    });
+    if (build.status !== 0) throw new Error("cargo build -p lattice-host failed");
+  }
+  if (!existsSync(built)) throw new Error(`lattice-host missing: ${built}`);
+  return built;
+}
 
 const sandboxRoot = mkdtempSync(join(tmpdir(), "ltcai-e2e-"));
 const sandbox = {
@@ -53,6 +111,9 @@ const isolatedEnv = {
   LATTICEAI_BRAIN_DIR: sandbox.brain,
   LATTICEAI_AGENT_ROOT: sandbox.agent,
   LATTICEAI_OBSIDIAN_VAULT_DIR: sandbox.vault,
+  // The supervisor resolves the worker interpreter itself; naming it keeps
+  // the choice the same one this runner made (repo .venv, else python3).
+  LTCAI_PYTHON: python,
   LATTICEAI_STORAGE_ENGINE: "sqlite",
   LATTICEAI_POSTGRES_DSN: "",
   LATTICEAI_REQUIRE_AUTH: "false",
@@ -76,11 +137,38 @@ function cleanupSandbox() {
 }
 process.once("exit", cleanupSandbox);
 
+/**
+ * The supervised worker's own stdout/stderr.
+ *
+ * `lattice-host` redirects them into `$HOME/.ltcai/desktop-sidecar.log` and
+ * `.err.log` (supervisor/worker_env.rs), and this runner rewrites HOME into a
+ * sandbox it deletes on the way out — so a worker that dies takes the only
+ * explanation with it and CI shows nothing but a 502. Print the tail before
+ * cleanup: those lines are the difference between "the gateway returned 502"
+ * and "that interpreter has no uvicorn".
+ */
+function dumpWorkerLogs() {
+  for (const name of ["desktop-sidecar.err.log", "desktop-sidecar.log"]) {
+    const path = join(sandbox.home, ".ltcai", name);
+    if (!existsSync(path)) {
+      console.error(`[e2e] ${name}: not written`);
+      continue;
+    }
+    const lines = readFileSync(path, "utf8").replace(/\n+$/, "").split("\n");
+    const tail = lines.slice(-80);
+    console.error(`[e2e] ---- ${name} (last ${tail.length} of ${lines.length} lines) ----`);
+    console.error(tail.join("\n") || "(empty)");
+  }
+}
+
 function delay(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 
-async function waitForHealth(url, timeoutMs = 60000) {
+// 120 s, not 60: the gate now covers two boots — this process binds the
+// gateway immediately, but /health is proxied and only answers once the
+// supervised Python worker has imported and bound its own port.
+async function waitForHealth(url, timeoutMs = 120000) {
   const deadline = Date.now() + timeoutMs;
   let lastError = "";
   while (Date.now() < deadline) {
@@ -115,13 +203,17 @@ function stop(child) {
 
 async function main() {
   console.log(`[e2e] sandbox ${sandboxRoot}`);
-  console.log(`[e2e] starting sidecar at ${baseUrl}`);
-  // Same process model as scripts/run_integration_tests.mjs: uvicorn on
-  // server:app with an isolated env. Do not double-append /health — the
-  // wait helper already probes `${url}/health`.
+  console.log(`[e2e] worker interpreter ${python}`);
+  const hostBinary = resolveHostBinary();
+  console.log(`[e2e] starting ${hostBinary} at ${baseUrl}`);
+  // The shipped topology: lattice-host serves the product on `port` and
+  // supervises the Python worker on a free port behind it. Do not
+  // double-append /health — the wait helper already probes `${url}/health`,
+  // which the gateway proxies to the worker, so a 200 there means both
+  // processes are up.
   const server = spawn(
-    python,
-    ["-m", "uvicorn", "server:app", "--host", host, "--port", port],
+    hostBinary,
+    ["--port", port],
     { stdio: "inherit", env: isolatedEnv, cwd: process.cwd() },
   );
 
@@ -150,6 +242,7 @@ async function main() {
     console.error("[e2e] failed:", error);
     exitCode = 1;
   } finally {
+    if (exitCode !== 0) dumpWorkerLogs();
     await stop(server);
     cleanupSandbox();
   }

package/scripts/wheel_smoke.py

@@ -8,6 +8,10 @@ class of "works only with ``pip install -e .`` from the repo root" failures
 (e.g. the v3.x wheels that omitted the root ``setup.py`` wizard module while
 ``latticeai.server_app`` imported it).
 
+v11.6.0: the app it boots is the AI-Worker (``create_worker_app``), because
+that is the only application this package builds — the product server is the
+``lattice-host`` binary.
+
 Usage:
     python scripts/wheel_smoke.py                 # build + install + import + /health
     python scripts/wheel_smoke.py --wheel dist/ltcai-X.Y.Z-py3-none-any.whl
@@ -30,39 +34,45 @@ from pathlib import Path
 
 REPO_ROOT = Path(__file__).resolve().parents[1]
 
-# Every importable module the wheel ships (pyproject py-modules + packages).
+# Every importable module the wheel ships. v11.6.0 (WP-P1) reduced the package
+# to the AI-Worker: the product application, the graph store, the agent loop,
+# the telegram bridge and the setup wizard are `lattice-host`'s now, so the
+# names below are the compute surface plus the two entrypoints a bundle starts.
 WHEEL_MODULES = [
     "lattice_brain",
-    "lattice_brain.graph",
-    "lattice_brain.graph.store",
-    "lattice_brain.runtime",
-    "lattice_brain.runtime.agent_runtime",
-    "lattice_brain.storage",
-    "lattice_brain.archive",
+    "lattice_brain.embeddings",
+    "lattice_brain.graph._kg_common",
     "lattice_brain.ingestion",
-    "lattice_brain.portability",
-    "lattice_brain.workflow",
+    "lattice_brain.multimodal",
+    "lattice_brain.runtime",
     "latticeai",
-    "latticeai.server_app",
     "latticeai.app_factory",
+    "latticeai.worker_app",
     "latticeai.cli.entrypoint",
-    "latticeai.integrations",
-    "latticeai.integrations.telegram_bot",
+    "latticeai.api.worker_compute",
+    "latticeai.api.worker_seams",
     "latticeai.models.router",
-    "latticeai.core.mcp_registry",
-    "latticeai.api.knowledge_graph",
+    "latticeai.runtime.build_phases.worker_profile",
+    "latticeai.services.model_runtime",
     "latticeai.services.p_reinforce",
     "latticeai.tools",
     "latticeai.tools.knowledge",
-    "latticeai.setup.auto_setup",
-    "latticeai.setup.wizard",
-    "latticeai.services.local_knowledge",
-    "lattice_brain.graph.schema",
-    "server",
 ]
 
-# Root shims removed in 9.9.1 — the wheel must NOT ship them anymore.
+# Root shims removed in 9.9.1, plus the product modules WP-P1 deleted: the
+# wheel must NOT ship any of them. ``server`` and ``latticeai.server_app`` are
+# on this list for the same reason the shims are — a bundle that still imports
+# them is a bundle running an application that no longer exists.
 REMOVED_ROOT_MODULES = [
+    "server",
+    "latticeai.server_app",
+    "latticeai.api.chat",
+    "latticeai.core.agent",
+    "latticeai.integrations.telegram_bot",
+    "latticeai.setup.wizard",
+    "lattice_brain.graph.store",
+    "lattice_brain.storage",
+    "lattice_brain.workflow",
     "ltcai_cli",
     "auto_setup",
     "setup_wizard",
@@ -96,9 +106,9 @@ else:
 
 HEALTH_CHECK = """
 from fastapi.testclient import TestClient
-from latticeai.app_factory import create_app
+from latticeai.worker_app import create_worker_app
 
-app = create_app()
+app = create_worker_app()
 response = TestClient(app).get("/health")
 assert response.status_code == 200, f"/health returned {response.status_code}"
 payload = response.json()