@cohortapp/agent-sdk 2.3.0

package/scripts/daemon/classifier.test.mjs

@@ -0,0 +1,238 @@
+/**
+ * classifier.test.mjs — node:test coverage for the classifier's Tier-2 seam.
+ *
+ * The classifier's Tier-2 fallback used to depend on a TOP-LEVEL
+ * `import OpenAI from "openai"`, but `openai` is NOT a dependency of this
+ * package — so in a clean install / CI the module failed to import and the
+ * whole Tier-2 path was dead. It now routes Tier-2 through
+ * lib/model-router/llm-task.mjs `llmTask()` (cheapest direct catalog model,
+ * no SDK), with an injectable `fetchImpl` so tests need no network.
+ *
+ * These tests assert:
+ *   1. the module imports CLEANLY with no `openai` package present;
+ *   2. when Tier-1 (claude --print) fails, Tier-2 `llmTask` produces the
+ *      classification (injected fetchImpl returns a JSON object), and the
+ *      output keeps the documented schema;
+ *   3. when both Tier-1 AND Tier-2 fail, classification falls through to the
+ *      Tier-3 rule-based path (unchanged behaviour);
+ *   4. wrapExternalContent fencing still wraps untrusted content in the
+ *      prompt handed to Tier-2.
+ *
+ * Hermetic: we inject a catalog + config + env + fetchImpl into classifyItem
+ * via its opts arg, force Tier-1 to fail by pointing CLAUDE_BIN at a missing
+ * binary, and never touch the network.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { createRequire } from "node:module";
+
+import * as classifier from "./classifier.mjs";
+
+// ── 1. Imports cleanly with no openai package present ───────────────────────
+
+test("classifier.mjs imports cleanly with no openai package installed", () => {
+  // The module-level import above already succeeded (this file loaded), which
+  // is the load-bearing assertion. Double-check openai is genuinely absent so
+  // this test fails loudly if someone reintroduces the hard dependency.
+  const require = createRequire(import.meta.url);
+  assert.throws(
+    () => require.resolve("openai"),
+    /Cannot find module 'openai'|Cannot find package 'openai'/,
+    "openai must NOT be resolvable — the classifier must not depend on it"
+  );
+  assert.equal(typeof classifier.classifyItem, "function", "classifyItem is exported");
+});
+
+// ── Hermetic fixtures for the Tier-2 llmTask seam ───────────────────────────
+
+// A minimal direct-harness catalog with one cheap anthropic row so llmTask can
+// pick a model for "sensitive" inbox data (deny-default permits anthropic).
+const ANTH_YAML = `
+provider: anthropic
+auth_env: ANTHROPIC_API_KEY
+endpoints:
+  anthropic: https://api.anthropic.com
+models:
+  - id: claude-haiku-test
+    status: available
+    context_tokens: 200000
+    cost: { input: 1.0, output: 5.0 }
+    cost_provenance: { source: official, fetched: "2026-06-10", volatile: false }
+    compat: { cache_control: anthropic }
+    tool_reliability: A
+    harness: { session: true, direct: true, batch: true }
+`;
+
+async function hermeticCatalog() {
+  const { loadCatalog } = await import("../../lib/model-router/catalog.mjs");
+  return loadCatalog("/tmp/nope", { catalogText: { anthropic: ANTH_YAML } });
+}
+
+function publicConfig() {
+  return {
+    schema_version: 2,
+    backends: {
+      anthropic: { allowed_data_classes: ["public", "internal", "sensitive"] },
+    },
+    defaults: { data_class: "sensitive" },
+  };
+}
+
+/** An anthropic-messages response carrying a JSON classification as text. */
+function anthropicJsonResponse(obj) {
+  return {
+    ok: true,
+    status: 200,
+    async json() {
+      return {
+        content: [{ type: "text", text: JSON.stringify(obj) }],
+        usage: { input_tokens: 50, output_tokens: 12 },
+      };
+    },
+  };
+}
+
+/**
+ * Build the llmTask deps that classifyItem threads through to its Tier-2 path.
+ * `fetchImpl` is REQUIRED so no network is touched; ledger is a no-op.
+ */
+async function tier2Deps(over = {}) {
+  return {
+    // Tier-1 always fails in these tests → Tier-2 (or Tier-3) engages.
+    runClaudeCLI: failingTier1,
+    llmTaskOpts: {
+      catalog: await hermeticCatalog(),
+      config: publicConfig(),
+      env: { ANTHROPIC_API_KEY: "test-key" },
+      now: () => 1_700_000_000_000,
+      isOpen: () => ({ open: false, until: 0, reason: null, strikes: 0 }),
+      writeLedger: () => {},
+      ...over,
+    },
+  };
+}
+
+// Force Tier-1 (claude --print) to fail deterministically via the injectable
+// runner seam — no dependency on a `claude` binary being installed. The
+// classifier turns a thrown Tier-1 runner into a fall-through to Tier-2.
+const failingTier1 = async () => {
+  throw new Error("claude CLI unavailable (test)");
+};
+
+const baseItem = {
+  sender: "casey",
+  sender_privilege: "external",
+  channel: "dm/casey",
+  service: "slack",
+  content: "Following up on the Singapore entity paperwork.",
+  is_dm: true,
+};
+
+// ── 2. Tier-1 fails → Tier-2 llmTask classifies ─────────────────────────────
+
+test("Tier-2 llmTask produces the classification when Tier-1 fails", async () => {
+  let sawFetch = false;
+  let sawPrompt = "";
+  const deps = await tier2Deps({
+    fetchImpl: async (url, init) => {
+      sawFetch = true;
+      const body = JSON.parse(init.body);
+      // tool-less invariant inherited from llmTask
+      assert.equal(body.tools, undefined, "Tier-2 request is tool-less");
+      // Capture the user prompt to assert fencing later.
+      const userMsg = (body.messages || []).find((m) => m.role === "user");
+      sawPrompt = userMsg ? userMsg.content : "";
+      return anthropicJsonResponse({
+        priority: "high",
+        action: "respond",
+        model: "sonnet",
+        summary: "External follow-up on Singapore entity",
+        category: "action_required",
+        directed_at_agent: true,
+      });
+    },
+  });
+
+  const out = await classifier.classifyItem(baseItem, deps);
+
+  assert.equal(sawFetch, true, "Tier-2 actually called the injected fetchImpl");
+  assert.equal(out.priority, "high");
+  assert.equal(out.action, "respond");
+  assert.equal(out.model, "sonnet");
+  assert.equal(out.category, "action_required");
+  assert.equal(out.directed_at_agent, true);
+  assert.equal(typeof out.summary, "string");
+
+  // 4. wrapExternalContent fencing: untrusted content must be fenced in the
+  // prompt handed to Tier-2 (the boundary marker is randomised per call, but
+  // the raw content must never appear unfenced as a bare line).
+  assert.match(sawPrompt, /BEGIN UNTRUSTED|UNTRUSTED|⟦|external-content/i,
+    "untrusted content is wrapped by wrapExternalContent before Tier-2");
+  assert.match(sawPrompt, /Singapore entity/, "the content still reaches the model (fenced)");
+});
+
+// ── 3. Tier-1 AND Tier-2 fail → Tier-3 rules ────────────────────────────────
+
+test("falls through to Tier-3 rules when llmTask fails", async () => {
+  // fetchImpl throws → llmTask classifies it as a transport error and returns
+  // { ok:false } (fail-open, never throws) → classifier drops to rules.
+  const deps = await tier2Deps({
+    fetchImpl: async () => {
+      throw new Error("network down");
+    },
+  });
+
+  const item = {
+    ...baseItem,
+    sender: "Avery (CEO)",
+    sender_privilege: "ceo",
+  };
+
+  const out = await classifier.classifyItem(item, deps);
+
+  // Rule-based path: CEO DM → critical / opus / respond / directed.
+  assert.equal(out.priority, "critical");
+  assert.equal(out.model, "opus");
+  assert.equal(out.action, "respond");
+  assert.equal(out.directed_at_agent, true);
+  assert.equal(out.category, "action_required");
+});
+
+test("falls through to Tier-3 rules when llmTask finds no candidate model", async () => {
+  // No catalog row qualifies for "sensitive" data without an anthropic
+  // credential → llmTask returns { ok:false, reason:"missing_credential" }.
+  const deps = await tier2Deps({
+    env: {}, // no ANTHROPIC_API_KEY
+    fetchImpl: async () => {
+      throw new Error("fetch should not be reached when no model qualifies");
+    },
+  });
+
+  const out = await classifier.classifyItem(
+    { ...baseItem, content: "urgent: blocker on the deploy" },
+    deps
+  );
+
+  // Urgent keyword rule → high priority via Tier-3.
+  assert.equal(out.priority, "high");
+  // DM → directed → respond.
+  assert.equal(out.action, "respond");
+  assert.equal(out.directed_at_agent, true);
+});
+
+// ── classify-by-kind short-circuit is preserved (no LLM) ────────────────────
+
+test("reaction kind short-circuits before any LLM tier (no fetch)", async () => {
+  const deps = await tier2Deps({
+    fetchImpl: async () => {
+      throw new Error("fetch must not be called for reaction kind");
+    },
+  });
+  const out = await classifier.classifyItem(
+    { kind: "reaction", sender: "casey", content: "added :eyes:" },
+    deps
+  );
+  assert.equal(out.action, "archive");
+  assert.equal(out.directed_at_agent, false);
+});

package/scripts/daemon/classify-kind.mjs

@@ -0,0 +1,54 @@
+/**
+ * classify-kind.mjs — deterministic classification for non-message event kinds.
+ *
+ * WS2 adds reaction / channel_cc inbox items. These don't warrant an LLM call:
+ *   - reaction   an emoji acknowledgement — FYI, never a reply.
+ *   - channel_cc a non-directed message in a watched channel — observe, don't
+ *                reply (directed_at_agent:false → the daemon's directed-gate
+ *                archives it).
+ *
+ * Extracted into its own dependency-free module so it can be unit-tested
+ * without importing classifier.mjs (which pulls in the agent-local `openai`
+ * dependency). classifier.mjs imports `classifyByKind` from here.
+ *
+ * @module scripts/daemon/classify-kind
+ */
+
+"use strict";
+
+/**
+ * @param {object} item  classify input; only `kind`, `sender`, `content`,
+ *                        `subject` are read.
+ * @returns {object|null} a classification result, or null for ordinary messages
+ *                        (so the normal LLM classifier path runs).
+ */
+export function classifyByKind(item) {
+  const kind = item?.kind || "message";
+
+  if (kind === "reaction") {
+    return {
+      priority: "normal",
+      action: "archive",
+      model: "sonnet",
+      summary: `Reaction from ${item?.sender || "someone"}`,
+      category: "fyi",
+      directed_at_agent: false,
+    };
+  }
+
+  if (kind === "channel_cc") {
+    const snippet = String(item?.content || item?.subject || "").slice(0, 80);
+    return {
+      priority: "normal",
+      action: "archive",
+      model: "sonnet",
+      summary: `Channel CC: ${snippet}`,
+      category: "fyi",
+      directed_at_agent: false,
+    };
+  }
+
+  return null;
+}
+
+export default { classifyByKind };

package/scripts/daemon/classify-kind.test.mjs

@@ -0,0 +1,40 @@
+/**
+ * classify-kind.test.mjs — node:test coverage for classify-kind.mjs
+ *
+ * Pure + hermetic (no openai/dotenv import). Verifies reaction/channel_cc
+ * short-circuit deterministically to observe-not-reply, and ordinary messages
+ * fall through (null) to the LLM path.
+ */
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+
+import { classifyByKind } from "./classify-kind.mjs";
+
+test("reaction kind → archive / fyi / not directed", () => {
+  const r = classifyByKind({ kind: "reaction", sender: "casey", content: "added :eyes:" });
+  assert.equal(r.action, "archive");
+  assert.equal(r.category, "fyi");
+  assert.equal(r.directed_at_agent, false);
+  assert.match(r.summary, /Reaction from casey/);
+});
+
+test("channel_cc kind → archive / fyi / not directed (observe-not-reply)", () => {
+  const r = classifyByKind({ kind: "channel_cc", content: "fyi the deploy finished" });
+  assert.equal(r.action, "archive");
+  assert.equal(r.directed_at_agent, false);
+  assert.match(r.summary, /Channel CC/);
+});
+
+test("ordinary message kind → null (falls through to LLM)", () => {
+  assert.equal(classifyByKind({ kind: "message", content: "hi" }), null);
+  assert.equal(classifyByKind({ content: "no kind field" }), null);
+  assert.equal(classifyByKind({}), null);
+});
+
+test("channel_cc summary truncates long content", () => {
+  const long = "x".repeat(200);
+  const r = classifyByKind({ kind: "channel_cc", content: long });
+  // "Channel CC: " + 80 chars
+  assert.ok(r.summary.length <= "Channel CC: ".length + 80);
+});