@llamaventures/cli 1.15.1 → 1.17.0

package/scripts/verify-agent-routing.mjs

@@ -2,14 +2,21 @@
 
 import assert from "node:assert/strict";
 import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
 import { createServer } from "node:http";
-import { existsSync } from "node:fs";
-import { mkdtemp, rm } from "node:fs/promises";
+import { existsSync, readFileSync } from "node:fs";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
 const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const packageJson = JSON.parse(readFileSync(path.join(repoRoot, "package.json"), "utf8"));
+assert.equal(
+  packageJson.scripts?.["verify:release"],
+  "npm test && npm pack --dry-run",
+  "CLI release gate must run agent routing tests and npm pack dry-run",
+);
 assert.equal(
   existsSync(path.join(repoRoot, "docs/agent-skills.bundle.json")),
   false,
@@ -20,8 +27,30 @@ assert.equal(
   false,
   "public llama-cli must not copy the Command-side skill mirror",
 );
+const cliSource = readFileSync(path.join(repoRoot, "bin/llama.mjs"), "utf8");
+assert.match(
+  cliSource,
+  /About ONE specific deal\? \.{8} llama html publish <deal-id-or-name> --file <path> --title "\.\.\."/,
+  "top-level help must route deal-specific HTML to the agent-safe publish path",
+);
+assert.doesNotMatch(
+  cliSource,
+  /For deal-specific HTML use "llama html upload <dealId>"/,
+  "wiki help must not route deal-specific HTML to the low-level upload path",
+);
 const calls = [];
 let threadSeq = 0;
+let eventSeq = 0;
+const htmlDocs = new Map();
+
+function sha256Hex(value) {
+  return createHash("sha256").update(value).digest("hex");
+}
+
+function docsForDeal(dealId) {
+  if (!htmlDocs.has(dealId)) htmlDocs.set(dealId, new Map());
+  return htmlDocs.get(dealId);
+}
 
 async function readJson(req) {
   let raw = "";
@@ -65,11 +94,48 @@ const server = createServer(async (req, res) => {
       path: url.pathname,
       query: Object.fromEntries(url.searchParams.entries()),
       body,
+      headers: {
+        client: req.headers["x-llama-client"] ?? null,
+        clientVersion: req.headers["x-llama-client-version"] ?? null,
+        agentClient: req.headers["x-llama-agent-client"] ?? null,
+        session: req.headers["x-llama-agent-session"] ?? null,
+        command: req.headers["x-llama-command"] ?? null,
+        uploadId: req.headers["x-llama-upload-id"] ?? null,
+      },
     });
 
+    if (req.method === "POST" && url.pathname === "/api/agent/client-events") {
+      eventSeq += 1;
+      writeJson(res, {
+        ok: true,
+        eventId: eventSeq,
+        candidateId: body?.command?.endsWith(".search") ? eventSeq + 1000 : null,
+      });
+      return;
+    }
+
+    if (req.method === "POST" && url.pathname === "/api/agent/eval-feedback") {
+      writeJson(res, {
+        ok: true,
+        candidate: {
+          id: 42,
+          source_event_id: body?.eventId ?? null,
+          feedback: body?.action ?? null,
+        },
+      });
+      return;
+    }
+
     if (req.method === "GET" && url.pathname === "/api/agent/manifest") {
       writeJson(res, {
         ok: true,
+        contract: {
+          contract_version: "agent-contract.v1",
+          cli: {
+            client_version: url.searchParams.get("clientVersion"),
+            status: "ok",
+          },
+        },
         briefing: "runtime briefing: use skills_search, skills_read, and object_inspect",
         llama_os: {
           visible_skill_count: 49,
@@ -85,6 +151,21 @@ const server = createServer(async (req, res) => {
       return;
     }
 
+    if (req.method === "GET" && url.pathname === "/api/agent/briefing") {
+      writeJson(res, {
+        ok: true,
+        contract: {
+          contract_version: "agent-contract.v1",
+          cli: {
+            client_version: url.searchParams.get("clientVersion"),
+            status: "ok",
+          },
+        },
+        briefing: "server-owned briefing: check CLI, use Pipeline First, prefer CLI/MCP",
+      });
+      return;
+    }
+
     if (req.method === "GET" && url.pathname === "/api/agent/skills") {
       writeJson(res, {
         ok: true,
@@ -136,6 +217,114 @@ const server = createServer(async (req, res) => {
       return;
     }
 
+    if (req.method === "GET" && url.pathname === "/api/wiki/search") {
+      writeJson(res, [
+        {
+          slug: "llama-weekly-2026-06-16",
+          title: "Llama Weekly 2026-06-16",
+        },
+      ]);
+      return;
+    }
+
+    if (req.method === "GET" && url.pathname === "/api/deals") {
+      writeJson(res, {
+        deals: [
+          {
+            uuid: "deal-html",
+            companyName: "Acme AI",
+            founders: "Ada Founder",
+            description: "mock deal for HTML upload tests",
+          },
+        ],
+        total: 1,
+        limit: Number(url.searchParams.get("limit") || 200),
+        offset: 0,
+      });
+      return;
+    }
+
+    const docsMatch = url.pathname.match(/^\/api\/deals\/([^/]+)\/documents$/);
+    if (docsMatch) {
+      const dealId = decodeURIComponent(docsMatch[1]);
+      if (dealId !== "deal-html") {
+        res.writeHead(404, { "Content-Type": "application/json" });
+        res.end(JSON.stringify({ error: "deal not found" }));
+        return;
+      }
+      if (req.method === "GET") {
+        const docs = Array.from(docsForDeal(dealId).entries()).map(([slug, doc]) => ({
+          slug,
+          title: doc.title,
+          latest_version: doc.version ?? null,
+          latest_updated_at: doc.version ? "2026-06-23T04:00:00Z" : null,
+        }));
+        writeJson(res, { documents: docs });
+        return;
+      }
+      if (req.method === "POST") {
+        const docs = docsForDeal(dealId);
+        const slug = body?.slug;
+        docs.set(slug, {
+          ...(docs.get(slug) || {}),
+          title: body?.title || slug,
+        });
+        writeJson(res, { ok: true, slug, title: body?.title || slug });
+        return;
+      }
+    }
+
+    const htmlMatch = url.pathname.match(/^\/api\/deals\/([^/]+)\/documents\/([^/]+)\/html$/);
+    if (htmlMatch) {
+      const dealId = decodeURIComponent(htmlMatch[1]);
+      const slug = decodeURIComponent(htmlMatch[2]);
+      const docs = docsForDeal(dealId);
+      if (req.method === "PUT") {
+        const html = typeof body?.html === "string" ? body.html : "";
+        const previous = docs.get(slug) || { title: slug, version: 0 };
+        const version = Number(previous.version || 0) + 1;
+        const bytes = Buffer.byteLength(html, "utf8");
+        const sha256 = sha256Hex(html);
+        docs.set(slug, {
+          ...previous,
+          html,
+          version,
+          bytes,
+          sha256,
+          source: body?.source || "cli",
+          client_upload_id: body?.client_upload_id || null,
+        });
+        writeJson(res, {
+          ok: true,
+          document_slug: slug,
+          version,
+          bytes,
+          sha256,
+          client_upload_id: body?.client_upload_id || null,
+          idempotent_replay: false,
+        });
+        return;
+      }
+      if (req.method === "GET") {
+        const doc = docs.get(slug);
+        if (!doc?.html) {
+          writeJson(res, { empty: true });
+          return;
+        }
+        writeJson(res, {
+          empty: false,
+          document_slug: slug,
+          version: doc.version,
+          bytes: doc.bytes,
+          sha256: doc.sha256,
+          source: doc.source,
+          created_at: "2026-06-23T04:00:00Z",
+          html: doc.html,
+        });
+        return;
+      }
+    }
+
     if (req.method === "POST" && /^\/api\/deals\/[^/]+\/threads$/.test(url.pathname)) {
       threadSeq += 1;
       writeJson(res, { id: `thread-${threadSeq}` });
@@ -191,25 +380,34 @@ function resetCalls() {
   threadSeq = 0;
 }
 
+function businessCalls() {
+  return calls.filter((call) => call.path !== "/api/agent/client-events");
+}
+
+function telemetryCalls() {
+  return calls.filter((call) => call.path === "/api/agent/client-events");
+}
+
 function paths() {
-  return calls.map((call) => `${call.method} ${call.path}`);
+  return businessCalls().map((call) => `${call.method} ${call.path}`);
 }
 
 function assertNoEnrichCall() {
   assert.equal(
-    calls.some((call) => call.path.endsWith("/enrich")),
+    businessCalls().some((call) => call.path.endsWith("/enrich")),
     false,
     `expected no /enrich call, got ${paths().join(", ")}`,
   );
 }
 
 function assertThreadRun({ title, messageIncludes }) {
-  assert.equal(calls.length, 2, `expected thread create + SSE run, got ${paths().join(", ")}`);
-  assert.match(calls[0].path, /^\/api\/deals\/[^/]+\/threads$/);
-  assert.equal(calls[0].body?.title, title);
-  assert.match(calls[1].path, /^\/api\/deals\/[^/]+\/threads\/thread-1$/);
+  const relevant = businessCalls();
+  assert.equal(relevant.length, 2, `expected thread create + SSE run, got ${paths().join(", ")}`);
+  assert.match(relevant[0].path, /^\/api\/deals\/[^/]+\/threads$/);
+  assert.equal(relevant[0].body?.title, title);
+  assert.match(relevant[1].path, /^\/api\/deals\/[^/]+\/threads\/thread-1$/);
   for (const needle of messageIncludes) {
-    assert.match(calls[1].body?.message ?? "", new RegExp(escapeRegExp(needle)));
+    assert.match(relevant[1].body?.message ?? "", new RegExp(escapeRegExp(needle)));
   }
 }
 
@@ -313,18 +511,29 @@ const baseUrl = `http://${address.address}:${address.port}`;
 const homeDir = await mkdtemp(path.join(os.tmpdir(), "llama-cli-routing-"));
 
 try {
+  resetCalls();
+  const onboardRun = await runCli(["agent-onboard"], baseUrl, homeDir);
+  assert.match(onboardRun.stdout, /server-owned briefing/);
+  assert.deepEqual(paths(), ["GET /api/agent/briefing"]);
+  assert.ok(businessCalls()[0].query.clientVersion, "agent-onboard passes clientVersion");
+  assert.equal(telemetryCalls()[0].body?.command, "agent.briefing");
+  assert.equal(telemetryCalls()[0].body?.client, "cli");
+  assert.ok(telemetryCalls()[0].body?.sessionId, "telemetry includes an agent session id");
+  assert.equal(businessCalls()[0].headers.command, "agent.briefing");
+
   resetCalls();
   const bootstrapRun = await runCli(["agent", "bootstrap", "--limit", "3"], baseUrl, homeDir);
   assert.match(bootstrapRun.stdout, /runtime briefing/);
   assert.deepEqual(paths(), ["GET /api/agent/manifest"]);
-  assert.equal(calls[0].query.limit, "3");
+  assert.equal(businessCalls()[0].query.limit, "3");
+  assert.ok(businessCalls()[0].query.clientVersion, "agent bootstrap passes clientVersion");
 
   resetCalls();
   const skillSearchRun = await runCli(["skills", "search", "pipeline", "--limit", "5"], baseUrl, homeDir);
   assert.match(skillSearchRun.stdout, /llama-command/);
   assert.deepEqual(paths(), ["GET /api/agent/skills"]);
-  assert.equal(calls[0].query.q, "pipeline");
-  assert.equal(calls[0].query.limit, "5");
+  assert.equal(businessCalls()[0].query.q, "pipeline");
+  assert.equal(businessCalls()[0].query.limit, "5");
 
   resetCalls();
   const skillShowRun = await runCli(["skills", "show", "llama-command"], baseUrl, homeDir);
@@ -336,7 +545,34 @@ try {
   assert.match(explainRun.stdout, /Status: deleted/);
   assert.match(explainRun.stdout, /Deleted by Kevin Yu/);
   assert.deepEqual(paths(), ["GET /api/agent/explain"]);
-  assert.equal(calls[0].query.q, "https://command.llamaventures.vc/wiki/missing-page");
+  assert.equal(businessCalls()[0].query.q, "https://command.llamaventures.vc/wiki/missing-page");
+
+  resetCalls();
+  const wikiRun = await runCli(["wiki", "search", "llama weekly"], baseUrl, homeDir);
+  assert.match(wikiRun.stdout, /llama-weekly-2026-06-16/);
+  assert.deepEqual(paths(), ["GET /api/wiki/search"]);
+  assert.equal(telemetryCalls()[0].body?.command, "wiki.search");
+  assert.equal(telemetryCalls()[0].body?.query, "llama weekly");
+
+  resetCalls();
+  const evalRun = await runCli(
+    [
+      "eval",
+      "bad",
+      "--last",
+      "--reason",
+      "missed dev weekly",
+      "--expect",
+      "wiki:llamaos-weekly-2026-06-17",
+    ],
+    baseUrl,
+    homeDir,
+  );
+  assert.match(evalRun.stdout, /"feedback": "bad"/);
+  assert.deepEqual(paths(), ["POST /api/agent/eval-feedback"]);
+  assert.equal(businessCalls()[0].body?.action, "bad");
+  assert.equal(businessCalls()[0].body?.eventId, 6);
+  assert.equal(businessCalls()[0].body?.expected?.wikiSlugs?.[0], "llamaos-weekly-2026-06-17");
 
   resetCalls();
   const enrichRun = await runCli(
@@ -369,9 +605,9 @@ try {
     homeDir,
   );
   assert.deepEqual(paths(), ["POST /api/deals/deal-cli/enrich"]);
-  assert.equal(calls[0].body?.apply, true);
-  assert.equal(calls[0].body?.dryRun, false);
-  assert.equal(calls[0].body?.executor, "server_agent");
+  assert.equal(businessCalls()[0].body?.apply, true);
+  assert.equal(businessCalls()[0].body?.dryRun, false);
+  assert.equal(businessCalls()[0].body?.executor, "server_agent");
 
   resetCalls();
   const agentRun = await runCli(
@@ -386,6 +622,68 @@ try {
     messageIncludes: ["custom server task"],
   });
 
+  const largeHtmlPath = path.join(homeDir, "full-memo.html");
+  const largeHtml =
+    "<!doctype html><html><head><title>Full Memo</title></head><body>" +
+    `<p>${"agent-safe upload ".repeat(18000)}</p>` +
+    "</body></html>";
+  assert.ok(
+    Buffer.byteLength(largeHtml, "utf8") > 252 * 1024,
+    "routing test HTML must exceed the incident-sized 252KB memo",
+  );
+  await writeFile(largeHtmlPath, largeHtml);
+
+  resetCalls();
+  const publishRun = await runCli(
+    [
+      "html",
+      "publish",
+      "Acme AI",
+      "--file",
+      largeHtmlPath,
+      "--title",
+      "Full Memo",
+      "--doc",
+      "full-memo",
+    ],
+    baseUrl,
+    homeDir,
+  );
+  const publishPayload = JSON.parse(publishRun.stdout);
+  assert.equal(publishPayload.ok, true);
+  assert.equal(publishPayload.deal_uuid, "deal-html");
+  assert.equal(publishPayload.document_slug, "full-memo");
+  assert.equal(publishPayload.verified?.ok, true);
+  assert.match(publishPayload.client_upload_id, /^cli-[0-9a-f-]{36}$/);
+  assert.match(publishPayload.sha256, /^[a-f0-9]{64}$/);
+  assert.equal(publishPayload.verified?.sha256, publishPayload.sha256);
+  assert.deepEqual(paths(), [
+    "GET /api/deals/Acme%20AI/documents",
+    "GET /api/deals",
+    "GET /api/deals/deal-html/documents",
+    "POST /api/deals/deal-html/documents",
+    "PUT /api/deals/deal-html/documents/full-memo/html",
+    "GET /api/deals/deal-html/documents/full-memo/html",
+  ]);
+  assert.equal(businessCalls()[4].body?.html, largeHtml);
+  assert.equal(businessCalls()[4].body?.source, "cli");
+  assert.equal(businessCalls()[4].body?.client_upload_id, publishPayload.client_upload_id);
+  assert.equal(businessCalls()[4].headers.uploadId, publishPayload.client_upload_id);
+  const publishTelemetry = telemetryCalls().find(
+    (call) =>
+      call.body?.method === "PUT" &&
+      call.body?.endpoint === "/api/deals/deal-html/documents/full-memo/html",
+  );
+  assert.ok(publishTelemetry, "publish upload request must record telemetry");
+  assert.equal(publishTelemetry.body?.args?.html?.redacted, true);
+  assert.equal(publishTelemetry.body?.args?.html?.bytes, Buffer.byteLength(largeHtml, "utf8"));
+  assert.equal(publishTelemetry.body?.args?.html?.sha256, sha256Hex(largeHtml));
+  assert.doesNotMatch(
+    JSON.stringify(telemetryCalls()),
+    /agent-safe upload agent-safe upload agent-safe upload/,
+    "telemetry must not contain raw memo HTML text",
+  );
+
   resetCalls();
   const mcpResult = await callMcpTool(
     "deal_enrich",
@@ -409,19 +707,90 @@ try {
   assert.equal(payload.threadId, "thread-1");
   assert.equal(payload.text, "agent done");
 
+  resetCalls();
+  const inlineGuard = await callMcpTool(
+    "html_upload",
+    {
+      dealId: "deal-html",
+      documentSlug: "inline-too-large",
+      html: largeHtml,
+    },
+    baseUrl,
+    homeDir,
+  );
+  assert.equal(inlineGuard.isError, true);
+  assert.match(inlineGuard.content?.[0]?.text ?? "", /Use html_upload_file/);
+  assert.deepEqual(paths(), []);
+
+  resetCalls();
+  const inlineBundleGuard = await callMcpTool(
+    "html_upload_bundle",
+    {
+      dealId: "deal-html",
+      documentSlug: "bundle-too-large",
+      html: largeHtml,
+      assets: [
+        {
+          path: "full-memo_files/cover.txt",
+          contentType: "text/plain",
+          base64: Buffer.from("asset").toString("base64"),
+        },
+      ],
+    },
+    baseUrl,
+    homeDir,
+  );
+  assert.equal(inlineBundleGuard.isError, true);
+  assert.match(inlineBundleGuard.content?.[0]?.text ?? "", /Use html_upload_file/);
+  assert.deepEqual(paths(), []);
+
+  resetCalls();
+  const mcpFile = await callMcpTool(
+    "html_upload_file",
+    {
+      dealId: "deal-html",
+      documentSlug: "mcp-file",
+      filePath: largeHtmlPath,
+    },
+    baseUrl,
+    homeDir,
+  );
+  const mcpFilePayload = JSON.parse(mcpFile.content?.[0]?.text ?? "{}");
+  assert.equal(mcpFilePayload.ok, true);
+  assert.equal(mcpFilePayload.verified?.ok, true);
+  assert.match(mcpFilePayload.client_upload_id, /^mcp-[0-9a-f-]{36}$/);
+  assert.match(mcpFilePayload.sha256, /^[a-f0-9]{64}$/);
+  assert.equal(mcpFilePayload.verified?.sha256, mcpFilePayload.sha256);
+  assert.deepEqual(paths(), [
+    "PUT /api/deals/deal-html/documents/mcp-file/html",
+    "GET /api/deals/deal-html/documents/mcp-file/html",
+  ]);
+  assert.equal(businessCalls()[0].body?.client_upload_id, mcpFilePayload.client_upload_id);
+  assert.equal(businessCalls()[0].headers.uploadId, mcpFilePayload.client_upload_id);
+  const mcpFileTelemetry = telemetryCalls().find(
+    (call) =>
+      call.body?.method === "PUT" &&
+      call.body?.endpoint === "/api/deals/deal-html/documents/mcp-file/html",
+  );
+  assert.ok(mcpFileTelemetry, "MCP file upload request must record telemetry");
+  assert.equal(mcpFileTelemetry.body?.args?.html?.redacted, true);
+  assert.equal(mcpFileTelemetry.body?.args?.html?.sha256, sha256Hex(largeHtml));
+
   resetCalls();
   const mcpBootstrap = await callMcpTool("agent_bootstrap", { limit: 2 }, baseUrl, homeDir);
   const bootstrapPayload = JSON.parse(mcpBootstrap.content?.[0]?.text ?? "{}");
   assert.equal(bootstrapPayload.ok, true);
   assert.deepEqual(paths(), ["GET /api/agent/manifest"]);
-  assert.equal(calls[0].query.limit, "2");
+  assert.equal(businessCalls()[0].query.limit, "2");
+  assert.ok(businessCalls()[0].query.clientVersion, "mcp agent_bootstrap passes clientVersion");
+  assert.equal(telemetryCalls()[0].body?.client, "mcp");
 
   resetCalls();
   const mcpSkills = await callMcpTool("skills_search", { q: "command", limit: 4 }, baseUrl, homeDir);
   const skillsPayload = JSON.parse(mcpSkills.content?.[0]?.text ?? "{}");
   assert.equal(skillsPayload.skills?.[0]?.slug, "llama-command");
   assert.deepEqual(paths(), ["GET /api/agent/skills"]);
-  assert.equal(calls[0].query.q, "command");
+  assert.equal(businessCalls()[0].query.q, "command");
 
   resetCalls();
   const mcpSkillRead = await callMcpTool("skills_read", { slug: "llama-command" }, baseUrl, homeDir);