litmus-cli 1.3.9 → 1.3.11

package/dist/lib/hook-logger.cjs

@@ -1,20 +1,30 @@
 #!/usr/bin/env node
 /**
  * Litmus AI prompt logger.
- * Universal hook script invoked by Claude Code, GitHub Copilot CLI, and OpenAI Codex CLI.
+ * Universal hook script invoked by Claude Code, GitHub Copilot CLI, OpenAI Codex
+ * CLI, and Cursor.
  *
  * Usage:
  *   node hook-logger.cjs <tool>
  *
  * Each tool pipes JSON on stdin when a user submits a prompt:
  *   - Claude Code (UserPromptSubmit): stdin JSON with session_id, prompt content
- *   - Copilot CLI (userPromptSubmitted): { timestamp, cwd, prompt }
+ *   - GitHub Copilot (UserPromptSubmit): CLI sends { sessionId, timestamp, cwd,
+ *     prompt }; VS Code Copilot Chat sends { hook_event_name, session_id,
+ *     transcript_path, timestamp, prompt } and notably NO cwd
  *   - Codex CLI: experimental hooks support (format TBD)
+ *   - Cursor (beforeSubmitPrompt): { conversation_id, generation_id, prompt, ... }
+ *
+ * Cursor is the only one that expects something back on stdout — see
+ * `emitCursorVerdictIfNeeded`. For every other tool stdout MUST stay empty:
+ * Claude Code injects it into the candidate's prompt as context.
  *
  * Normalizes to: { ts, type: "ai_prompt", tool, prompt, sessionId? }
  * Appends to .litmus/activity.jsonl — the same file the watcher uses,
  * so prompts flow through the existing analysis pipeline.
- * Also uploads the event to the server (fire-and-forget) for real-time backup.
+ * Also uploads the event to /cli/activity so it reaches candidate_activity_logs.
+ * That upload is AWAITED (bounded) — see uploadEvent for why an unawaited one
+ * silently stopped working.
  */
 
 const fs = require("fs")
@@ -35,6 +45,21 @@ const STDIN_TIMEOUT_MS = 5000
 
 const tool = process.argv[2] || "unknown"
 
+// Optional explicit assessment directory (argv[3]).
+//
+// Repo-scope hooks are written INTO an assessment, so they know its absolute
+// path at install time and should never have to guess. That matters most for
+// Copilot: VS Code Copilot Chat's payload carries no `cwd` and no workspace
+// roots, so without this the only resolution left is the cwd walk-up (wrong
+// whenever the editor's process starts elsewhere) or the active-assessments
+// registry — and the registry is not always populated. `litmus init` removes
+// the entry in project-scope mode, which silently disabled Copilot Chat capture
+// for exactly the candidates whose HOME could not take the user-scope hook.
+//
+// Passing it explicitly also sidesteps the registry's first-match behaviour,
+// which misattributes prompts when a candidate has more than one assessment.
+const explicitAssessmentDir = process.argv[3] || null
+
 // A directory only counts as an assessment if it has a config.json. Without
 // this guard, the walk-up would resolve to `~/.litmus/` (created by the
 // user-scope install) for any Claude Code session whose cwd is in HOME but
@@ -49,15 +74,9 @@ function isAssessmentLitmusDir(litmusDir) {
   }
 }
 
-// Find the .litmus directory — walk up from cwd, then fall back to the
-// active-assessments registry that `litmus init` writes to ~/.litmus/.
-// The cwd walk-up handles the common case (candidate opens VS Code at the
-// assessment dir). The registry fallback handles the case where Claude Code,
-// Codex, or Copilot CLI fires from a cwd outside the assessment dir — e.g.
-// VS Code workspace opened at a parent dir, or the candidate ran `claude`
-// from their home dir.
-function findLitmusDir() {
-  let dir = process.cwd()
+// Walk up from `startDir` looking for an assessment .litmus dir.
+function walkUpForLitmusDir(startDir) {
+  let dir = startDir
   for (let i = 0; i < 10; i++) {
     const candidate = path.join(dir, ".litmus")
     if (isAssessmentLitmusDir(candidate)) return candidate
@@ -65,6 +84,27 @@ function findLitmusDir() {
     if (parent === dir) break
     dir = parent
   }
+  return null
+}
+
+// Find the .litmus directory — walk up from any workspace roots the tool told
+// us about, then from cwd, then fall back to the active-assessments registry
+// that `litmus init` writes to ~/.litmus/.
+//
+// `hintDirs` matters because not every tool runs the hook from the workspace.
+// Verified against a real Cursor 3.13.25 session: Cursor invokes the hook with
+// cwd = ~/.cursor and passes the actual workspace in `workspace_roots`. Without
+// the hint the cwd walk-up can never match, and we'd silently fall through to
+// the registry — which returns the FIRST active assessment, misattributing
+// prompts whenever a candidate has more than one on the go.
+function findLitmusDir(hintDirs = []) {
+  for (const hint of hintDirs) {
+    if (typeof hint !== "string" || !hint) continue
+    const found = walkUpForLitmusDir(hint)
+    if (found) return found
+  }
+  const fromCwd = walkUpForLitmusDir(process.cwd())
+  if (fromCwd) return fromCwd
   try {
     const home = os.homedir()
     if (!home) return null
@@ -90,46 +130,133 @@ function readConfig(litmusDir) {
   }
 }
 
-// Fire-and-forget upload to /cli/activity.
-// Uses socket.unref() so Node exits immediately without waiting for the response —
-// the OS TCP stack still flushes the outbound data after our process exits.
+// Upload budget. The measured round trip to the backend is ~240ms (dns 15ms,
+// tcp 39ms, TLS 154ms, complete 242ms against Railway), so 2s is generous while
+// staying far inside every hook runner's allowance: Claude Code gives its
+// UserPromptSubmit hook 5s, Cursor 10s, Codex 30s. Exceeding it is not worth
+// delaying a candidate's prompt over — activity.jsonl already has the event and
+// the submission zip carries it to the grader.
+const UPLOAD_TIMEOUT_MS = 2000
+
+/**
+ * POST the event to /cli/activity and WAIT for it to complete.
+ *
+ * This used to be fire-and-forget: it called `socket.unref()` so Node wouldn't
+ * hold the process open for the response. That only ever worked by accident —
+ * `readStdin`'s timeout timer was never cleared, which kept the event loop alive
+ * ~5s, comfortably longer than the upload. Clearing that timer (correctly, it was
+ * spending the entire hook budget doing nothing) dropped the process lifetime to
+ * ~26ms, which is BEFORE tcp connect at 39ms, let alone TLS at 154ms. Every
+ * hook-captured prompt silently stopped reaching candidate_activity_logs.
+ *
+ * That was invisible because grading still worked: the grader merges the in-zip
+ * activity.jsonl as well as the DB rows. But anything reading the DB alone went
+ * blank — including the report's own AI-prompts drawer, which queries
+ * candidateActivityLog and renders nothing when empty. Confirmed in the caroline
+ * DB: zero ai_prompt rows for a submission whose zip had three, while the
+ * long-lived watcher's uploads (heartbeat, env_detected) all landed.
+ *
+ * So we await it now, bounded. Resolves rather than rejects on every failure
+ * path: a telemetry upload must never fail a candidate's prompt. Both timers are
+ * cleared on settle — the bug above is exactly what an uncleared timer costs.
+ */
 function uploadEvent(config, event) {
-  if (!config || !config.token || !config.backendUrl) return
+  if (!config || !config.token || !config.backendUrl) return Promise.resolve()
+
+  let url
+  try {
+    url = new URL(`${config.backendUrl}/cli/activity`)
+  } catch {
+    return Promise.resolve() // malformed backendUrl — nothing to do
+  }
 
   const body = JSON.stringify({ events: [event] })
-  const url = new URL(`${config.backendUrl}/cli/activity`)
   const reqFn = url.protocol === "https:" ? https.request : http.request
 
-  const req = reqFn(url, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      "Content-Length": Buffer.byteLength(body),
-      "Authorization": `Bearer ${config.token}`,
-    },
-    timeout: 5000,
+  return new Promise((resolve) => {
+    let settled = false
+    let guard = null
+    const done = () => {
+      if (settled) return
+      settled = true
+      if (guard) clearTimeout(guard)
+      resolve()
+    }
+
+    const req = reqFn(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "Content-Length": Buffer.byteLength(body),
+        "Authorization": `Bearer ${config.token}`,
+      },
+      timeout: UPLOAD_TIMEOUT_MS,
+    })
+    // Drain the response so the socket can close and the event loop empty.
+    req.on("response", (res) => { res.resume(); res.on("end", done); res.on("error", done) })
+    req.on("error", done)        // network down, DNS, refused, destroyed
+    req.on("timeout", () => { req.destroy(); done() })
+    // Backstop in case none of the above fires (a socket wedged before connect).
+    guard = setTimeout(() => { try { req.destroy() } catch { /* already gone */ } done() }, UPLOAD_TIMEOUT_MS)
+    req.end(body)
   })
-  req.on("error", () => {}) // Swallow — fire-and-forget
-  req.on("timeout", () => { req.destroy() })
-  req.on("socket", (socket) => { socket.unref() }) // Don't keep process alive for response
-  req.end(body)
 }
 
 function readStdin() {
   return new Promise((resolve) => {
     let resolved = false
+    let timer = null
     const chunks = []
+    // clearTimeout is load-bearing, not tidiness. A pending timer keeps the
+    // Node event loop alive after main() has finished, so without this the
+    // process lingers for the FULL STDIN_TIMEOUT_MS on every single
+    // invocation even though stdin closed in milliseconds. That silently ate
+    // the entire 5s budget Claude Code allows its UserPromptSubmit hook.
+    const done = () => {
+      if (resolved) return
+      resolved = true
+      if (timer) clearTimeout(timer)
+      resolve(chunks.join(""))
+    }
     process.stdin.setEncoding("utf8")
-    process.stdin.on("data", (chunk) => chunks.push(chunk))
-    process.stdin.on("end", () => {
-      if (!resolved) { resolved = true; resolve(chunks.join("")) }
+    process.stdin.on("data", (chunk) => {
+      chunks.push(chunk)
+      // Resolve as soon as the payload is a COMPLETE JSON document instead of
+      // waiting for EOF.
+      //
+      // Not every hook runner closes stdin after writing. Cursor demonstrably
+      // does not: every Cursor prompt burned the full STDIN_TIMEOUT_MS, which
+      // killed the 5s-budget Claude entry outright (exit 1). Waiting for EOF
+      // therefore spends the whole hook budget doing nothing, and the budget is
+      // what the awaited /cli/activity upload has to fit inside — so on any
+      // such runner the process is killed before the upload lands and prompts
+      // silently stop reaching candidate_activity_logs. That is precisely the
+      // regression 2c1aeeb1b fixed for Cursor; resolving on a complete document
+      // removes the whole failure mode rather than re-tuning timeouts per tool.
+      //
+      // Safe against partial reads: every payload is a JSON object, and a
+      // truncated object never parses (`{"prompt":` throws). Worst case this
+      // never fires and we fall back to EOF or the timeout, i.e. today's
+      // behaviour.
+      if (isCompleteJson(chunks.join(""))) done()
     })
-    setTimeout(() => {
-      if (!resolved) { resolved = true; resolve(chunks.join("")) }
-    }, STDIN_TIMEOUT_MS)
+    process.stdin.on("end", done)
+    process.stdin.on("error", done)   // closed/absent stdin must not hang either
+    timer = setTimeout(done, STDIN_TIMEOUT_MS)
   })
 }
 
+function isCompleteJson(buf) {
+  const trimmed = buf.trim()
+  if (!trimmed) return false
+  try {
+    JSON.parse(trimmed)
+    return true
+  } catch {
+    return false
+  }
+}
+
 function extractPrompt(input, toolName) {
   try {
     const data = JSON.parse(input)
@@ -145,10 +272,24 @@ function extractPrompt(input, toolName) {
         }
       }
       case "copilot": {
-        // Copilot CLI userPromptSubmitted: { timestamp, cwd, prompt }
+        // GitHub Copilot, both surfaces. We register the hook under the
+        // PascalCase `UserPromptSubmit` key, which Copilot CLI maps onto its
+        // internal `userPromptSubmitted` (alias map verified in the first-party
+        // @github/copilot SDK) and which VS Code Copilot Chat uses natively —
+        // so one config serves the terminal binary, the cloud agent, VS Code
+        // and JetBrains.
+        //
+        // Payloads differ slightly by surface, so read both spellings:
+        //   Copilot CLI   { sessionId | session_id, timestamp, cwd, prompt }
+        //   VS Code Chat  { hook_event_name, session_id?, transcript_path?,
+        //                   timestamp, prompt }   <- note: no cwd
+        // The missing cwd is why this arm must never rely on the hook's working
+        // directory to locate the assessment; resolution comes from the
+        // registry fallback in findLitmusDir.
+        const prompt = data.prompt || null
         return {
-          prompt: data.prompt || null,
-          sessionId: null,
+          prompt: typeof prompt === "string" ? prompt : JSON.stringify(prompt),
+          sessionId: data.session_id || data.sessionId || null,
         }
       }
       case "codex": {
@@ -161,6 +302,25 @@ function extractPrompt(input, toolName) {
           sessionId: data.session_id || null,
         }
       }
+      case "cursor": {
+        // Cursor beforeSubmitPrompt. Payload captured verbatim from a real
+        // Cursor 3.13.25 agent session:
+        //   { conversation_id, generation_id, model, model_id, model_params,
+        //     composer_mode, prompt, attachments, session_id, hook_event_name,
+        //     cursor_version, workspace_roots, user_email, transcript_path }
+        // session_id and conversation_id carry the same value; prefer session_id
+        // to match every other tool here. `workspace_roots` is load-bearing —
+        // Cursor runs the hook with cwd = ~/.cursor, so it is the only reliable
+        // pointer back to the assessment directory.
+        // Deliberately NOT captured: user_email (PII we have no use for) and
+        // transcript_path (points at Cursor's own on-disk agent transcript).
+        const prompt = data.prompt || null
+        return {
+          prompt: typeof prompt === "string" ? prompt : JSON.stringify(prompt),
+          sessionId: data.session_id || data.conversation_id || null,
+          workspaceRoots: Array.isArray(data.workspace_roots) ? data.workspace_roots : [],
+        }
+      }
       default: {
         // Best-effort: look for common field names
         const prompt = data.prompt || data.user_message || data.content || null
@@ -171,25 +331,156 @@ function extractPrompt(input, toolName) {
       }
     }
   } catch {
-    return { prompt: null, sessionId: null }
+    return { prompt: null, sessionId: null, workspaceRoots: [] }
+  }
+}
+
+// Cursor requires a JSON verdict on stdout; non-JSON output trips a parse error
+// in its hook runtime and the integration silently breaks. We are pure
+// observation and never block a prompt, so the verdict is a constant.
+//
+// Emitted FIRST, before any capture work, so a slow filesystem, a missing
+// assessment, or an outright crash can never delay or swallow it — the
+// candidate's agent proceeds regardless of whether we captured anything.
+//
+// Gated on the RAW argv tool, never on `effectiveTool`: Claude Code injects
+// hook stdout into the candidate's prompt as added context, and `effectiveTool`
+// relabels claude -> cursor when Claude Code is driven from inside Cursor.
+// Keying off the relabel would leak `{"continue":true}` into those prompts.
+function emitCursorVerdictIfNeeded() {
+  if (tool !== "cursor") return
+  try {
+    process.stdout.write(JSON.stringify({ continue: true }))
+  } catch {
+    // Closed/broken stdout — Cursor treats absent output as "proceed" anyway.
+  }
+}
+
+// Cursor merges EVERY config source it finds and runs all matching hooks, and it
+// translates Claude Code's `UserPromptSubmit` onto its own `beforeSubmitPrompt`.
+// Since `litmus init` installs into both ~/.cursor/hooks.json and
+// ~/.claude/settings.json, one Cursor prompt fires us TWICE. Confirmed in
+// Cursor's own hooks log:
+//
+//   Found 2 hook(s) to execute for step: beforeSubmitPrompt
+//   Executing hook 1/2 from user config          (~/.cursor/hooks.json)
+//   Executing hook 2/2 from claude-user config   (~/.claude/settings.json)
+//
+// Both invocations got the identical Cursor payload, so a single prompt produced
+// two activity rows with the same text and timestamp, differing only in `tool`
+// (`cursor` and `claude`). That double-counts prompts and misattributes half of
+// them to Claude Code, which was never involved.
+//
+// Dedupe by identity rather than by inspecting the log: the two hooks run
+// concurrently (observed writing at the same millisecond), so any
+// read-then-write check would race. Instead the claude-registered invocation
+// stands down when the payload is visibly Cursor's — `cursor_version` is present
+// on every beforeSubmitPrompt payload and never on a Claude Code one — leaving
+// the dedicated cursor hook as the single writer.
+//
+// Gated on our cursor hook actually being installed. If that install failed
+// (unwritable ~/.cursor, refused config shape) nothing else would capture the
+// prompt, and dropping it would be worse than a duplicate.
+function isCursorPayload(data) {
+  if (!data || typeof data !== "object") return false
+  return typeof data.cursor_version === "string"
+    || data.hook_event_name === "beforeSubmitPrompt"
+}
+
+// Dedupe by CLAIMING the prompt, not by predicting whether the sibling hook will
+// succeed. An earlier version inspected ~/.cursor/hooks.json and stood down if a
+// Cursor hook looked installed, but that is guesswork about another process, and
+// the failure is asymmetric: guess wrong and the prompt is lost entirely, where
+// the worst case in the other direction is a duplicate row. A hook can be
+// configured and still not run (logger missing, truncated, unreadable, or broken
+// in a way no amount of statting reveals), so there is no check that makes the
+// prediction sound.
+//
+// Instead, both invocations race to create a marker for this specific prompt.
+// `openSync(..., "wx")` is O_CREAT|O_EXCL, so exactly one wins — no read-then-write
+// window, which matters because the two hooks run concurrently (observed writing
+// in the same millisecond). The winner writes the event; the loser exits. If the
+// Cursor hook never runs at all, the Claude invocation simply wins uncontested.
+//
+// Keyed on `generation_id`, which is per prompt submission. NOT conversation_id,
+// which is stable for a whole thread and would collapse every prompt in a
+// conversation into one. If it is absent we do not dedupe at all — a duplicate is
+// the acceptable failure, a dropped prompt is not.
+const CLAIM_DIR = path.join(os.homedir() || os.tmpdir(), ".litmus", "claims")
+const CLAIM_TTL_MS = 60 * 60 * 1000
+
+// Keep the markers from accumulating in the candidate's HOME forever. Cheap: this
+// directory only ever holds an hour of prompts.
+function pruneClaims() {
+  try {
+    const cutoff = Date.now() - CLAIM_TTL_MS
+    for (const name of fs.readdirSync(CLAIM_DIR)) {
+      const p = path.join(CLAIM_DIR, name)
+      try {
+        if (fs.statSync(p).mtimeMs < cutoff) fs.unlinkSync(p)
+      } catch { /* raced with another invocation — fine */ }
+    }
+  } catch { /* no dir yet, or unreadable */ }
+}
+
+/** True when THIS invocation should write the event. */
+function claimPrompt(generationId) {
+  if (typeof generationId !== "string" || !generationId) return true // can't dedupe → capture
+  const safe = generationId.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 128)
+  try {
+    fs.mkdirSync(CLAIM_DIR, { recursive: true })
+    pruneClaims()
+    fs.closeSync(fs.openSync(path.join(CLAIM_DIR, safe), "wx"))
+    return true
+  } catch (e) {
+    if (e && e.code === "EEXIST") return false // sibling already has it
+    return true // unwritable claim dir → fail toward capture
   }
 }
 
 async function main() {
-  const litmusDir = findLitmusDir()
-  if (!litmusDir) return // Not in a Litmus assessment — silently exit
+  emitCursorVerdictIfNeeded()
 
+  // stdin is read BEFORE resolving the assessment dir: the payload is what
+  // tells us where the workspace actually is (see findLitmusDir's hintDirs).
   const raw = await readStdin()
   if (!raw.trim()) return
 
-  // Detect Cursor firing Claude Code hooks — Cursor sets CURSOR_TRACE_ID
-  const effectiveTool = (tool === "claude" && process.env.CURSOR_TRACE_ID)
+  // Detect Cursor firing Claude Code hooks. CURSOR_TRACE_ID is checked for the
+  // real Claude-Code-inside-Cursor case (a Claude-shaped payload from Cursor's
+  // integrated terminal), but it is NOT set when Cursor's own hooks service
+  // spawns us — verified against a real session, where the compat-path event
+  // stayed labelled `claude`. The payload shape is the reliable signal.
+  let parsed = null
+  try { parsed = JSON.parse(raw) } catch { /* extractPrompt reports the failure */ }
+  const cursorShaped = isCursorPayload(parsed)
+
+  // The relabel is what makes the claim below label-safe: whichever invocation
+  // wins the race, a Cursor-shaped payload is recorded as tool="cursor", so the
+  // event never blames Claude Code for a prompt it had no part in.
+  const effectiveTool = (tool === "claude" && (cursorShaped || process.env.CURSOR_TRACE_ID))
     ? "cursor"
     : tool
 
-  const { prompt, sessionId } = extractPrompt(raw, effectiveTool)
+  const { prompt, sessionId, workspaceRoots } = extractPrompt(raw, effectiveTool)
   if (!prompt) return
 
+  // Explicit hint first: it is the only source that is correct by construction.
+  // Then the tool-supplied workspace roots, then cwd, then the registry.
+  const hintDirs = explicitAssessmentDir
+    ? [explicitAssessmentDir, ...(workspaceRoots || [])]
+    : (workspaceRoots || [])
+  const litmusDir = findLitmusDir(hintDirs)
+  if (!litmusDir) return // Not in a Litmus assessment — silently exit
+
+  // Claim as LATE as possible — only once we know we have a prompt and somewhere
+  // to put it. Claiming earlier would let an invocation that then bails (no
+  // prompt, no assessment in scope) consume the claim and silence the sibling
+  // that would have written successfully.
+  if (cursorShaped && !claimPrompt(parsed && parsed.generation_id)) {
+    return // the sibling hook is writing this same prompt
+  }
+
   const truncated = prompt.length > MAX_PROMPT_LENGTH
     ? prompt.slice(0, MAX_PROMPT_LENGTH) + "...[truncated]"
     : prompt
@@ -210,12 +501,14 @@ async function main() {
     // Non-critical
   }
 
-  // 2. Upload to server (fire-and-forget, for real-time backup)
+  // 2. Upload to the server so the event reaches candidate_activity_logs. Awaited
+  // (bounded by UPLOAD_TIMEOUT_MS) because the process now exits far too fast for
+  // an unawaited request to survive — see uploadEvent.
   const config = readConfig(litmusDir)
-  uploadEvent(config, event)
+  await uploadEvent(config, event)
 
-  // No explicit process.exit() — the event loop drains naturally.
-  // The HTTP socket is unref'd so it won't keep the process alive.
+  // No explicit process.exit() — the event loop drains naturally once the upload
+  // has settled and its timers are cleared.
 }
 
 main().catch(() => process.exit(0))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "litmus-cli",
-  "version": "1.3.9",
+  "version": "1.3.11",
   "description": "CLI tool for Litmus engineering assessments",
   "license": "MIT",
   "author": "elenazhao",