@indigoai-us/hq-cloud 6.3.1 → 6.3.3

package/src/skill-telemetry.test.ts

@@ -5,6 +5,8 @@ import * as path from "node:path";
 import {
   extractSkillEvents,
   extractCodexSkillEvents,
+  extractCodexSkillToolEvents,
+  codexRowTurnId,
   parseCodexSessionMeta,
   collectAndSendSkillTelemetry,
 } from "./skill-telemetry.js";
@@ -173,6 +175,244 @@ describe("extractCodexSkillEvents (Codex CLI rollouts)", () => {
   });
 });
 
+describe("extractCodexSkillToolEvents (Codex model-driven SKILL.md reads)", () => {
+  const ctx = { sessionId: "sess-x", cwd: "/home/ec2-user/hq" };
+
+  // Codex surfaces a completed exec as `exec_command_end` with `command` as
+  // `["/bin/zsh","-lc","<cmd>"]`, a `turn_id`, the run `cwd`, and a `parsed_cmd`
+  // it tags `type: "read"` for inspection commands.
+  function execEnd(cmd: string, over: Record<string, unknown> = {}) {
+    return {
+      timestamp: "2026-04-27T07:51:30.395Z",
+      type: "event_msg",
+      payload: {
+        type: "exec_command_end",
+        call_id: "call_1",
+        turn_id: "turn-A",
+        command: ["/bin/zsh", "-lc", cmd],
+        cwd: "/home/ec2-user/hq",
+        parsed_cmd: [{ type: "read", cmd }],
+        ...over,
+      },
+    };
+  }
+
+  it("extracts a skill load from a sed read of SKILL.md, tagged source:model", () => {
+    const ev = extractCodexSkillToolEvents(
+      execEnd("sed -n '1,240p' .agents/skills/learn/SKILL.md"),
+      ctx,
+    );
+    expect(ev).toEqual([
+      {
+        skill: "learn",
+        source: "model",
+        sessionId: "sess-x",
+        timestamp: "2026-04-27T07:51:30.395Z",
+        cwd: "/home/ec2-user/hq",
+        uuid: "codex:skill:sess-x:turn-A:learn",
+        hasArgs: false,
+      },
+    ]);
+  });
+
+  it("captures the directory above SKILL.md as the skill name, even when nested (bridge layout)", () => {
+    const ev = extractCodexSkillToolEvents(
+      execEnd("rg -n description /Users/me/.codex/skills/hq/indigo:hello-world/SKILL.md"),
+      ctx,
+    )[0];
+    // skills/hq/<name>/SKILL.md → <name>, not "hq".
+    expect(ev.skill).toBe("indigo:hello-world");
+    expect(ev.source).toBe("model");
+  });
+
+  it("absolute and relative SKILL.md paths both resolve", () => {
+    expect(
+      extractCodexSkillToolEvents(execEnd("cat /abs/path/.agents/skills/quiz/SKILL.md"), ctx)[0].skill,
+    ).toBe("quiz");
+    expect(
+      extractCodexSkillToolEvents(execEnd("nl -ba .agents/skills/summary/SKILL.md"), ctx)[0].skill,
+    ).toBe("summary");
+  });
+
+  it("dedups a turn's repeated reads of the same skill via the shared seen set", () => {
+    const seen = new Set<string>();
+    const first = extractCodexSkillToolEvents(
+      execEnd("sed -n '1,240p' .agents/skills/learn/SKILL.md"),
+      ctx,
+      seen,
+    );
+    const second = extractCodexSkillToolEvents(
+      execEnd("rg -n 'frequency' .agents/skills/learn/SKILL.md"),
+      ctx,
+      seen,
+    );
+    expect(first).toHaveLength(1);
+    expect(second).toEqual([]); // same (session, turn, skill) → collapsed
+  });
+
+  it("does NOT dedup the same skill across different turns", () => {
+    const seen = new Set<string>();
+    const a = extractCodexSkillToolEvents(
+      execEnd("sed -n '1,240p' .agents/skills/learn/SKILL.md", { turn_id: "turn-A" }),
+      ctx,
+      seen,
+    );
+    const b = extractCodexSkillToolEvents(
+      execEnd("sed -n '1,240p' .agents/skills/learn/SKILL.md", { turn_id: "turn-B" }),
+      ctx,
+      seen,
+    );
+    expect(a).toHaveLength(1);
+    expect(b).toHaveLength(1);
+    expect(b[0].uuid).toBe("codex:skill:sess-x:turn-B:learn");
+  });
+
+  it("excludes edits: parsed_cmd not a read, or a write redirect into SKILL.md", () => {
+    // Codex classifies the exec as something other than a read.
+    expect(
+      extractCodexSkillToolEvents(
+        execEnd("apply_patch .agents/skills/learn/SKILL.md", { parsed_cmd: [{ type: "write" }] }),
+        ctx,
+      ),
+    ).toEqual([]);
+    // A shell write into the skill file, even with a read verb elsewhere.
+    expect(
+      extractCodexSkillToolEvents(
+        execEnd("cat tmp > .agents/skills/learn/SKILL.md", { parsed_cmd: undefined }),
+        ctx,
+      ),
+    ).toEqual([]);
+  });
+
+  it("prefers the row cwd, falling back to session ctx cwd", () => {
+    expect(
+      extractCodexSkillToolEvents(
+        execEnd("sed -n '1,5p' .agents/skills/learn/SKILL.md", { cwd: "/home/ec2-user/hq/sub" }),
+        ctx,
+      )[0].cwd,
+    ).toBe("/home/ec2-user/hq/sub");
+    expect(
+      extractCodexSkillToolEvents(
+        execEnd("sed -n '1,5p' .agents/skills/learn/SKILL.md", { cwd: undefined }),
+        ctx,
+      )[0].cwd,
+    ).toBe(ctx.cwd);
+  });
+
+  it("uuid falls back to timestamp without a turn_id, and is omitted without a session", () => {
+    const noTurn = extractCodexSkillToolEvents(
+      execEnd("sed -n '1,5p' .agents/skills/learn/SKILL.md", { turn_id: undefined }),
+      ctx,
+    )[0];
+    expect(noTurn.uuid).toBe("codex:skill:sess-x:2026-04-27T07:51:30.395Z:learn");
+    const noSession = extractCodexSkillToolEvents(
+      execEnd("sed -n '1,5p' .agents/skills/learn/SKILL.md"),
+      {},
+    )[0];
+    expect(noSession.uuid).toBeUndefined();
+  });
+
+  it("ignores non-exec rows and execs that touch no SKILL.md", () => {
+    expect(extractCodexSkillToolEvents(execEnd("sed -n '1,5p' src/index.ts"), ctx)).toEqual([]);
+    expect(
+      extractCodexSkillToolEvents({ type: "event_msg", payload: { type: "task_started" } }, ctx),
+    ).toEqual([]);
+    expect(
+      extractCodexSkillToolEvents({ type: "event_msg", payload: { type: "user_message", message: "/deploy" } }, ctx),
+    ).toEqual([]);
+    expect(extractCodexSkillToolEvents(null, ctx)).toEqual([]);
+  });
+
+  // The OTHER exec shape some Codex CLI versions emit (observed on 0.128): a
+  // `response_item` `function_call` named `exec_command`, with the command +
+  // workdir in `arguments` (a JSON string) and NO turn_id/parsed_cmd on the row.
+  function fnCall(cmd: string, args: Record<string, unknown> = {}, over: Record<string, unknown> = {}) {
+    return {
+      timestamp: "2026-06-10T16:39:41.260Z",
+      type: "response_item",
+      payload: {
+        type: "function_call",
+        name: "exec_command",
+        call_id: "call_abc",
+        arguments: JSON.stringify({ cmd, workdir: "/home/ec2-user/hq", ...args }),
+        ...over,
+      },
+    };
+  }
+
+  it("extracts a skill load from the function_call exec shape, turn from ctx", () => {
+    const ev = extractCodexSkillToolEvents(
+      fnCall("sed -n '1,80p' .claude/skills/indigo:hello-world/SKILL.md"),
+      { ...ctx, turnId: "turn-Z" },
+    );
+    expect(ev).toEqual([
+      {
+        skill: "indigo:hello-world",
+        source: "model",
+        sessionId: "sess-x",
+        timestamp: "2026-06-10T16:39:41.260Z",
+        cwd: "/home/ec2-user/hq", // from arguments.workdir
+        uuid: "codex:skill:sess-x:turn-Z:indigo:hello-world",
+        hasArgs: false,
+      },
+    ]);
+  });
+
+  it("function_call: read-intent decided by command text (no parsed_cmd present)", () => {
+    // A read verb → counted.
+    expect(
+      extractCodexSkillToolEvents(fnCall("wc -l .claude/skills/quiz/SKILL.md"), ctx)[0].skill,
+    ).toBe("quiz");
+    // A write redirect into the skill file → excluded.
+    expect(
+      extractCodexSkillToolEvents(fnCall("printf x > .claude/skills/quiz/SKILL.md"), ctx),
+    ).toEqual([]);
+  });
+
+  it("function_call: ignores non-exec tool names (e.g. apply_patch edits)", () => {
+    expect(
+      extractCodexSkillToolEvents(
+        fnCall("anything", {}, { name: "apply_patch" }),
+        ctx,
+      ),
+    ).toEqual([]);
+  });
+
+  it("function_call: falls back to ctx.cwd when arguments has no workdir", () => {
+    expect(
+      extractCodexSkillToolEvents(
+        fnCall("sed -n '1,5p' .agents/skills/learn/SKILL.md", { workdir: undefined }),
+        ctx,
+      )[0].cwd,
+    ).toBe(ctx.cwd);
+  });
+
+  it("dedups across the two exec shapes within one turn (no double count)", () => {
+    const seen = new Set<string>();
+    // exec_command_end form for skill `learn` in turn-A …
+    const a = extractCodexSkillToolEvents(
+      execEnd("sed -n '1,240p' .agents/skills/learn/SKILL.md", { turn_id: "turn-A" }),
+      ctx,
+      seen,
+    );
+    // … then the function_call form for the same skill in the same turn.
+    const b = extractCodexSkillToolEvents(
+      fnCall("rg -n x .agents/skills/learn/SKILL.md"),
+      { ...ctx, turnId: "turn-A" },
+      seen,
+    );
+    expect(a).toHaveLength(1);
+    expect(b).toEqual([]); // same (session, turn, skill) → collapsed
+  });
+
+  it("codexRowTurnId reads turn_id from a row's payload, else undefined", () => {
+    expect(codexRowTurnId({ type: "turn_context", payload: { turn_id: "turn-9" } })).toBe("turn-9");
+    expect(codexRowTurnId({ type: "event_msg", payload: { type: "exec_command_end", turn_id: "turn-7" } })).toBe("turn-7");
+    expect(codexRowTurnId({ type: "response_item", payload: { type: "function_call" } })).toBeUndefined();
+    expect(codexRowTurnId(null)).toBeUndefined();
+  });
+});
+
 describe("collectAndSendSkillTelemetry — hqRoot scoping", () => {
   function stubClient(captured: SkillInvocationBatch[]) {
     return {
@@ -489,4 +729,197 @@ describe("collectAndSendSkillTelemetry — hqRoot scoping", () => {
 
     await fs.rm(tmp, { recursive: true, force: true });
   });
+
+  // A Codex rollout that drives a skill the way the model actually does: by
+  // reading its SKILL.md via exec_command. `reads` is a list of
+  // [skill, turnId, cmdTail] tuples turned into exec_command_end rows.
+  function codexModelRollout(
+    sessionId: string,
+    cwd: string,
+    reads: Array<[skill: string, turnId: string, extra?: Record<string, unknown>]>,
+  ): string {
+    const lines = [
+      row({ timestamp: "2026-06-08T10:00:00.000Z", type: "session_meta", payload: { id: sessionId, cwd, originator: "codex_exec" } }),
+      ...reads.map(([skill, turnId, extra], i) =>
+        row({
+          timestamp: `2026-06-08T10:0${i}:00.000Z`,
+          type: "event_msg",
+          payload: {
+            type: "exec_command_end",
+            call_id: `call_${i}`,
+            turn_id: turnId,
+            command: ["/bin/zsh", "-lc", `sed -n '1,240p' ${cwd}/.agents/skills/${skill}/SKILL.md`],
+            cwd,
+            parsed_cmd: [{ type: "read" }],
+            ...(extra ?? {}),
+          },
+        }),
+      ),
+    ];
+    return lines.join("\n") + "\n";
+  }
+
+  it("captures Codex model-driven skill loads, deduped per turn", async () => {
+    const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-codexmodel-"));
+    const codex = path.join(tmp, "sessions", "2026", "06", "08");
+    await fs.mkdir(codex, { recursive: true });
+    const hqRoot = "/home/ec2-user/hq";
+
+    await fs.writeFile(
+      path.join(codex, "rollout-model.jsonl"),
+      codexModelRollout("sess-model", hqRoot, [
+        ["learn", "turn-A"], // turn-A read #1
+        ["learn", "turn-A"], // turn-A read #2 — same turn+skill, must collapse
+        ["learn", "turn-B"], // turn-B — distinct turn, counts again
+        ["quiz", "turn-B"], // different skill, same turn — counts
+      ]),
+      "utf-8",
+    );
+
+    const captured: SkillInvocationBatch[] = [];
+    const result = await collectAndSendSkillTelemetry({
+      client: stubClient(captured),
+      machineId: "m",
+      installerVersion: "t",
+      hqRoot,
+      claudeProjectsRoot: path.join(tmp, "no-claude"),
+      codexSessionsRoot: path.join(tmp, "sessions"),
+      cursorPath: path.join(tmp, "cursor.json"),
+    });
+
+    const events = captured.flatMap((b) => b.events);
+    // 4 reads → 3 events: the two turn-A `learn` reads collapse to one.
+    expect(result.eventsSent).toBe(3);
+    expect(events.every((e) => e.source === "model")).toBe(true);
+    expect(events.map((e) => `${e.skill}@${e.uuid}`).sort()).toEqual([
+      "learn@codex:skill:sess-model:turn-A:learn",
+      "learn@codex:skill:sess-model:turn-B:learn",
+      "quiz@codex:skill:sess-model:turn-B:quiz",
+    ]);
+
+    await fs.rm(tmp, { recursive: true, force: true });
+  });
+
+  it("captures the function_call exec shape, attributing the turn from turn_context", async () => {
+    // Mirrors a real Codex 0.128 session: SKILL.md reads arrive as
+    // `response_item` `function_call` execs (no per-row turn_id), preceded by a
+    // `turn_context` that names the turn. The collector must thread that turn id.
+    const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-fncall-"));
+    const codex = path.join(tmp, "sessions", "2026", "06", "10");
+    await fs.mkdir(codex, { recursive: true });
+    const hqRoot = "/home/ec2-user/hq";
+
+    const lines = [
+      row({ timestamp: "2026-06-10T16:38:32.000Z", type: "session_meta", payload: { id: "sess-fn", cwd: hqRoot, originator: "codex_exec" } }),
+      row({ timestamp: "2026-06-10T16:38:44.000Z", type: "turn_context", payload: { turn_id: "turn-1", cwd: hqRoot } }),
+      // Two reads of the same skill in turn-1 — the wc then the sed, as Codex does.
+      row({ timestamp: "2026-06-10T16:39:26.000Z", type: "response_item", payload: { type: "function_call", name: "exec_command", call_id: "c1", arguments: JSON.stringify({ cmd: "wc -l .claude/skills/indigo:hello-world/SKILL.md", workdir: hqRoot }) } }),
+      row({ timestamp: "2026-06-10T16:39:41.000Z", type: "response_item", payload: { type: "function_call", name: "exec_command", call_id: "c2", arguments: JSON.stringify({ cmd: "sed -n '1,80p' .claude/skills/indigo:hello-world/SKILL.md", workdir: hqRoot }) } }),
+    ].join("\n") + "\n";
+    await fs.writeFile(path.join(codex, "rollout-fn.jsonl"), lines, "utf-8");
+
+    const captured: SkillInvocationBatch[] = [];
+    const result = await collectAndSendSkillTelemetry({
+      client: stubClient(captured),
+      machineId: "m",
+      installerVersion: "t",
+      hqRoot,
+      claudeProjectsRoot: path.join(tmp, "no-claude"),
+      codexSessionsRoot: path.join(tmp, "sessions"),
+      cursorPath: path.join(tmp, "cursor.json"),
+    });
+
+    const events = captured.flatMap((b) => b.events);
+    // Two reads, one turn, one skill → exactly one event, attributed to turn-1.
+    expect(result.eventsSent).toBe(1);
+    expect(events[0]).toMatchObject({
+      skill: "indigo:hello-world",
+      source: "model",
+      cwd: hqRoot,
+      uuid: "codex:skill:sess-fn:turn-1:indigo:hello-world",
+    });
+
+    await fs.rm(tmp, { recursive: true, force: true });
+  });
+
+  it("does not count SKILL.md edits (apply_patch / write) as skill usage", async () => {
+    const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-edit-"));
+    const codex = path.join(tmp, "sessions", "2026", "06", "08");
+    await fs.mkdir(codex, { recursive: true });
+    const hqRoot = "/home/ec2-user/hq";
+
+    await fs.writeFile(
+      path.join(codex, "rollout-edit.jsonl"),
+      codexModelRollout("sess-edit", hqRoot, [
+        ["learn", "turn-A", { parsed_cmd: [{ type: "write" }] }], // an edit — excluded
+      ]),
+      "utf-8",
+    );
+
+    const captured: SkillInvocationBatch[] = [];
+    const result = await collectAndSendSkillTelemetry({
+      client: stubClient(captured),
+      machineId: "m",
+      installerVersion: "t",
+      hqRoot,
+      claudeProjectsRoot: path.join(tmp, "no-claude"),
+      codexSessionsRoot: path.join(tmp, "sessions"),
+      cursorPath: path.join(tmp, "cursor.json"),
+    });
+
+    expect(result.eventsSent).toBe(0);
+    await fs.rm(tmp, { recursive: true, force: true });
+  });
+
+  it("scope now counts skills run from HQ subfolders (worktrees), still excludes siblings", async () => {
+    const tmp = await fs.mkdtemp(path.join(os.tmpdir(), "skill-tel-scope-"));
+    const projects = path.join(tmp, "projects");
+    const codex = path.join(tmp, "sessions", "2026", "06", "08");
+    await fs.mkdir(projects, { recursive: true });
+    await fs.mkdir(codex, { recursive: true });
+    const hqRoot = "/home/ec2-user/hq";
+    const worktree = `${hqRoot}/.claude/worktrees/wt1`;
+    const sibling = `${hqRoot}-other`; // shares the prefix but is a different repo
+
+    // Claude skill run from a worktree under the HQ root — newly counted.
+    const dirWt = path.join(projects, "wt");
+    await fs.mkdir(dirWt, { recursive: true });
+    await fs.writeFile(
+      path.join(dirWt, "wt.jsonl"),
+      row({ type: "assistant", sessionId: "s-wt", timestamp: "2026-06-08T10:00:00Z", cwd: worktree, message: { role: "assistant", content: [{ type: "tool_use", id: "t1", name: "Skill", input: { skill: "handoff" } }] } }) + "\n",
+      "utf-8",
+    );
+    // Claude skill run from a sibling repo sharing the prefix — must stay excluded.
+    const dirSib = path.join(projects, "sib");
+    await fs.mkdir(dirSib, { recursive: true });
+    await fs.writeFile(
+      path.join(dirSib, "sib.jsonl"),
+      row({ type: "assistant", sessionId: "s-sib", timestamp: "2026-06-08T10:01:00Z", cwd: sibling, message: { role: "assistant", content: [{ type: "tool_use", id: "t2", name: "Skill", input: { skill: "sibling-skill" } }] } }) + "\n",
+      "utf-8",
+    );
+    // Codex model load from a worktree under the HQ root — newly counted.
+    await fs.writeFile(
+      path.join(codex, "rollout-wt.jsonl"),
+      codexModelRollout("sess-wt", worktree, [["quiz", "turn-A"]]),
+      "utf-8",
+    );
+
+    const captured: SkillInvocationBatch[] = [];
+    const result = await collectAndSendSkillTelemetry({
+      client: stubClient(captured),
+      machineId: "m",
+      installerVersion: "t",
+      hqRoot,
+      claudeProjectsRoot: projects,
+      codexSessionsRoot: path.join(tmp, "sessions"),
+      cursorPath: path.join(tmp, "cursor.json"),
+    });
+
+    const skills = captured.flatMap((b) => b.events.map((e) => e.skill)).sort();
+    expect(result.eventsSent).toBe(2);
+    expect(skills).toEqual(["handoff", "quiz"]);
+    expect(skills).not.toContain("sibling-skill");
+
+    await fs.rm(tmp, { recursive: true, force: true });
+  });
 });