@app-vitals/agent-runtime 0.0.0-stage → 0.1.0

package/fixtures/stream-json/skill-attribution.ts

@@ -0,0 +1,251 @@
+/**
+ * Hand-authored synthetic `claude -p --output-format stream-json --verbose`
+ * transcript exercising `agent/src/runtime/run-telemetry.ts` end-to-end through
+ * `_consumeStream`: first-turn baseline capture, turn/tool-call counting, and
+ * per-skill attribution with a sub-agent turn in the middle.
+ *
+ * NOTE: hand-authored per the documented public Claude Code CLI stream-json
+ * schema (the same shapes the fixtures beside this one use). Numbers are
+ * chosen so each attribution bucket's totals are distinct and checkable.
+ *
+ * Shape of the session:
+ *  - one `system`/`init` line
+ *  - `msg_1` (root): text only, usage = the first-turn baseline
+ *  - `msg_2` (root): a `Skill` tool_use invoking "shipwright:task-store"
+ *    — emitted twice with the same message id (dedupe), the second line
+ *    carrying the usage
+ *  - a `user` tool_result line (no usage)
+ *  - `msg_3` (skill): first usage-bearing turn after the invoke — its
+ *    input + cache_creation is the skill's invokeContextDelta; it also fires
+ *    an `Agent` tool_use (subagent_type "Explore")
+ *  - `sub_1` (agent): a sub-agent turn carrying `parent_tool_use_id`
+ *  - `msg_4` (skill): a plain Read tool_use + usage, still charged to the skill
+ *  - one terminal `result`/success line
+ */
+
+import type { RunTelemetry } from "@app-vitals/agent-runtime/run-telemetry";
+
+const SONNET = "claude-sonnet-4-6";
+const SESSION_ID = "sess-skill-attribution";
+
+export const lines: string[] = [
+  JSON.stringify({
+    type: "system",
+    subtype: "init",
+    session_id: SESSION_ID,
+    model: SONNET,
+    tools: ["Read", "Skill", "Agent"],
+  }),
+  JSON.stringify({
+    type: "assistant",
+    message: {
+      id: "msg_1",
+      role: "assistant",
+      model: SONNET,
+      content: [{ type: "text", text: "Starting." }],
+      usage: {
+        input_tokens: 2,
+        output_tokens: 10,
+        cache_read_input_tokens: 30_000,
+        cache_creation_input_tokens: 40_000,
+      },
+    },
+    session_id: SESSION_ID,
+  }),
+  // msg_2 — Skill invoke, first line has no usage yet
+  JSON.stringify({
+    type: "assistant",
+    message: {
+      id: "msg_2",
+      role: "assistant",
+      model: SONNET,
+      content: [
+        {
+          type: "tool_use",
+          id: "tu_skill",
+          name: "Skill",
+          input: { skill: "shipwright:task-store" },
+        },
+      ],
+    },
+    session_id: SESSION_ID,
+  }),
+  // msg_2 — same message id repeated, now with usage (must count once)
+  JSON.stringify({
+    type: "assistant",
+    message: {
+      id: "msg_2",
+      role: "assistant",
+      model: SONNET,
+      content: [
+        {
+          type: "tool_use",
+          id: "tu_skill",
+          name: "Skill",
+          input: { skill: "shipwright:task-store" },
+        },
+      ],
+      usage: {
+        input_tokens: 3,
+        output_tokens: 20,
+        cache_read_input_tokens: 70_000,
+        cache_creation_input_tokens: 100,
+      },
+    },
+    session_id: SESSION_ID,
+  }),
+  JSON.stringify({
+    type: "user",
+    message: {
+      role: "user",
+      content: [
+        {
+          type: "tool_result",
+          tool_use_id: "tu_skill",
+          content: "<skill body>",
+        },
+      ],
+    },
+    session_id: SESSION_ID,
+  }),
+  // msg_3 — first turn after the invoke: cache_creation carries the skill body
+  JSON.stringify({
+    type: "assistant",
+    message: {
+      id: "msg_3",
+      role: "assistant",
+      model: SONNET,
+      content: [
+        {
+          type: "tool_use",
+          id: "tu_agent",
+          name: "Agent",
+          input: { subagent_type: "Explore", prompt: "find things" },
+        },
+      ],
+      usage: {
+        input_tokens: 4,
+        output_tokens: 30,
+        cache_read_input_tokens: 70_100,
+        cache_creation_input_tokens: 9_000,
+      },
+    },
+    session_id: SESSION_ID,
+  }),
+  // sub_1 — sub-agent turn
+  JSON.stringify({
+    type: "assistant",
+    parent_tool_use_id: "tu_agent",
+    message: {
+      id: "sub_1",
+      role: "assistant",
+      model: SONNET,
+      content: [{ type: "text", text: "found it" }],
+      usage: {
+        input_tokens: 500,
+        output_tokens: 40,
+        cache_read_input_tokens: 0,
+        cache_creation_input_tokens: 0,
+      },
+    },
+    session_id: SESSION_ID,
+  }),
+  JSON.stringify({
+    type: "user",
+    message: {
+      role: "user",
+      content: [
+        { type: "tool_result", tool_use_id: "tu_agent", content: "report" },
+      ],
+    },
+    session_id: SESSION_ID,
+  }),
+  // msg_4 — still attributed to the skill
+  JSON.stringify({
+    type: "assistant",
+    message: {
+      id: "msg_4",
+      role: "assistant",
+      model: SONNET,
+      content: [{ type: "tool_use", id: "tu_read", name: "Read", input: {} }],
+      usage: {
+        input_tokens: 5,
+        output_tokens: 50,
+        cache_read_input_tokens: 79_100,
+        cache_creation_input_tokens: 200,
+      },
+    },
+    session_id: SESSION_ID,
+  }),
+  JSON.stringify({
+    type: "result",
+    subtype: "success",
+    result: "done",
+    session_id: SESSION_ID,
+    is_error: false,
+    usage: {
+      input_tokens: 514,
+      output_tokens: 150,
+      cache_read_input_tokens: 249_200,
+      cache_creation_input_tokens: 49_300,
+    },
+    total_cost_usd: 0.42,
+    modelUsage: {
+      [SONNET]: {
+        inputTokens: 514,
+        outputTokens: 150,
+        cacheReadInputTokens: 249_200,
+        cacheCreationInputTokens: 49_300,
+        costUSD: 0.42,
+      },
+    },
+  }),
+];
+
+export const expectedTelemetry: RunTelemetry = {
+  firstTurn: {
+    model: SONNET,
+    messageId: "msg_1",
+    inputTokens: 2,
+    cacheCreationTokens: 40_000,
+    cacheReadTokens: 30_000,
+    contextTokens: 70_002,
+  },
+  turns: 5,
+  toolCalls: 3,
+  skillUsage: [
+    {
+      kind: "root",
+      name: "root",
+      invocations: 0,
+      turns: 1,
+      inputTokens: 2,
+      outputTokens: 10,
+      cacheReadTokens: 30_000,
+      cacheCreationTokens: 40_000,
+      invokeContextDelta: null,
+    },
+    {
+      kind: "skill",
+      name: "shipwright:task-store",
+      invocations: 1,
+      turns: 3,
+      inputTokens: 12,
+      outputTokens: 100,
+      cacheReadTokens: 219_200,
+      cacheCreationTokens: 9_300,
+      invokeContextDelta: 3 + 100,
+    },
+    {
+      kind: "agent",
+      name: "Explore",
+      invocations: 1,
+      turns: 1,
+      inputTokens: 500,
+      outputTokens: 40,
+      cacheReadTokens: 0,
+      cacheCreationTokens: 0,
+      invokeContextDelta: null,
+    },
+  ],
+};

package/fixtures/stream-json/truncated-no-result.ts

@@ -0,0 +1,84 @@
+/**
+ * Hand-authored synthetic `claude -p --output-format stream-json --verbose`
+ * transcript: a session that is cut off mid-stream — two assistant turns are
+ * emitted but NO terminal `result` line ever arrives (truncated output /
+ * process killed).
+ *
+ * NOTE: hand-authored per the documented public schema, standing in for a real
+ * capture to be validated later. The parser must surface whatever per-model
+ * usage it accumulated from the `assistant` lines rather than discarding it.
+ */
+
+const SONNET = "claude-sonnet-4-6";
+const OPUS = "claude-opus-4-8";
+const SESSION_ID = "sess-truncated";
+
+export const lines: string[] = [
+  JSON.stringify({
+    type: "system",
+    subtype: "init",
+    session_id: SESSION_ID,
+    model: SONNET,
+  }),
+  JSON.stringify({
+    type: "assistant",
+    message: {
+      id: "msg_a",
+      role: "assistant",
+      model: SONNET,
+      content: [{ type: "text", text: "Working on it..." }],
+      usage: {
+        input_tokens: 100,
+        output_tokens: 40,
+        cache_read_input_tokens: 5,
+        cache_creation_input_tokens: 0,
+      },
+    },
+    session_id: SESSION_ID,
+  }),
+  JSON.stringify({
+    type: "assistant",
+    message: {
+      id: "msg_b",
+      role: "assistant",
+      model: OPUS,
+      content: [{ type: "text", text: "Almost done" }],
+      usage: {
+        input_tokens: 150,
+        output_tokens: 60,
+        cache_read_input_tokens: 0,
+        cache_creation_input_tokens: 10,
+      },
+    },
+    session_id: SESSION_ID,
+  }),
+  // stream is cut off here — no `result` line
+];
+
+/**
+ * Everything the parser accumulated before the stream ended. costUSD is 0
+ * (never learned the authoritative cost — no result event arrived).
+ */
+export const expectedAccumulated = {
+  [SONNET]: {
+    inputTokens: 100,
+    outputTokens: 40,
+    cacheReadInputTokens: 5,
+    cacheCreationInputTokens: 0,
+    costUSD: 0,
+  },
+  [OPUS]: {
+    inputTokens: 150,
+    outputTokens: 60,
+    cacheReadInputTokens: 0,
+    cacheCreationInputTokens: 10,
+    costUSD: 0,
+  },
+};
+
+/**
+ * The `session_id` carried on the leading `system`/`init` line — the only
+ * place a session id is available in this fixture, since the stream never
+ * reaches a terminal `result` event.
+ */
+export const expectedSessionId = SESSION_ID;

package/format.ts

@@ -0,0 +1,283 @@
+// Markdown → Slack mrkdwn converter
+// Converts GitHub-flavored markdown to Slack format
+
+const TABLE_REGEX =
+  /(\|[^\n]+\|\n)([ \t]*\|[ \t]*[-:]+[ \t]*(?:\|[ \t]*[-:]+[ \t]*)*\|[ \t]*\n)((?:[ \t]*\|[^\n]+\|\n?)*)/gm;
+
+export function markdownToSlack(text: string): string {
+  return (
+    text
+      // Tables → code blocks (fallback for non-block contexts)
+      .replace(TABLE_REGEX, (match) => {
+        return `\`\`\`\n${match.trimEnd()}\n\`\`\``;
+      })
+      // Italic (but not bold) — must run before heading/bold so the
+      // lookbehind/lookahead can distinguish *italic* from **bold** in
+      // the original markdown input. After the bold pass, **bold** becomes
+      // *bold* and the regex would incorrectly convert it to _bold_ (italic).
+      .replace(/(?<!\*)\*(?!\*)(.+?)(?<!\*)\*(?!\*)/g, "_$1_")
+      // Headings → bold
+      .replace(/^#{1,6}\s+(.+)$/gm, "*$1*")
+      // Bold
+      .replace(/\*\*(.+?)\*\*/g, "*$1*")
+      // Inline code
+      .replace(/`([^`]+)`/g, "`$1`")
+      // Links
+      .replace(/\[([^\]]+)\]\(([^)]+)\)/g, "<$2|$1>")
+      // Horizontal rules
+      .replace(/^---+$/gm, "─────────────────────")
+      .trim()
+  );
+}
+
+const SECTION_TEXT_LIMIT = 3000;
+
+type RawTextCell = { type: "raw_text"; text: string };
+type TableBlock = {
+  type: "table";
+  column_settings: Array<{ align: "left" | "center" | "right" }>;
+  rows: RawTextCell[][];
+};
+type SectionBlock = { type: "section"; text: { type: "mrkdwn"; text: string } };
+export type SlackBlock = TableBlock | SectionBlock;
+
+function pushSectionBlocks(blocks: SlackBlock[], mrkdwn: string): void {
+  if (!mrkdwn) return;
+  for (let i = 0; i < mrkdwn.length; i += SECTION_TEXT_LIMIT) {
+    blocks.push({
+      type: "section",
+      text: { type: "mrkdwn", text: mrkdwn.slice(i, i + SECTION_TEXT_LIMIT) },
+    });
+  }
+}
+
+function parseTableRows(tableStr: string): string[][] {
+  const lines = tableStr.trim().split("\n").filter(Boolean);
+  const parseRow = (line: string) =>
+    line
+      .split("|")
+      .slice(1, -1)
+      .map((c) => c.trim());
+  // lines[0] = header, lines[1] = separator, lines[2+] = data rows
+  return [parseRow(lines[0]), ...lines.slice(2).map(parseRow)];
+}
+
+/**
+ * Converts markdown to Slack Block Kit blocks when tables are present.
+ * Returns null when no tables are found (fall back to markdownToSlack).
+ */
+export function markdownToBlocks(
+  text: string,
+): { text: string; blocks: SlackBlock[] } | null {
+  const regex = new RegExp(TABLE_REGEX.source, TABLE_REGEX.flags);
+  if (!regex.test(text)) return null;
+
+  regex.lastIndex = 0;
+  const blocks: SlackBlock[] = [];
+  let lastIndex = 0;
+
+  for (let match = regex.exec(text); match !== null; match = regex.exec(text)) {
+    const before = text.slice(lastIndex, match.index).trim();
+    if (before) {
+      pushSectionBlocks(blocks, markdownToSlack(before));
+    }
+
+    const rows = parseTableRows(match[0]);
+    const columnCount = rows[0]?.length ?? 0;
+    blocks.push({
+      type: "table",
+      column_settings: Array.from({ length: columnCount }, () => ({
+        align: "left" as const,
+      })),
+      rows: rows.map((row) =>
+        // Slack's table block rejects a cell whose text is empty ("must be more
+        // than 0 characters"), which a blank corner/header cell produces and
+        // fails the whole message with invalid_blocks. Coerce empties to a
+        // non-breaking space: a real character (passes validation) that still
+        // renders blank.
+        row.map((cell) => ({
+          type: "raw_text" as const,
+          text: cell || " ",
+        })),
+      ),
+    });
+
+    lastIndex = match.index + match[0].length;
+  }
+
+  const after = text.slice(lastIndex).trim();
+  if (after) {
+    pushSectionBlocks(blocks, markdownToSlack(after));
+  }
+
+  return { text: markdownToSlack(text), blocks };
+}
+
+// ─── Slack rich_text → Markdown ──────────────────────────────────────────────
+
+type TextStyle = {
+  bold?: boolean;
+  italic?: boolean;
+  code?: boolean;
+  strike?: boolean;
+};
+
+type RichTextElement =
+  | { type: "text"; text: string; style?: TextStyle }
+  | { type: "link"; url: string; text?: string }
+  | { type: "emoji"; name: string }
+  | { type: "user"; user_id: string }
+  | { type: string; [key: string]: unknown };
+
+type RichTextSection = {
+  type: "rich_text_section";
+  elements: RichTextElement[];
+};
+
+type RichTextPreformatted = {
+  type: "rich_text_preformatted";
+  elements: RichTextElement[];
+};
+
+type RichTextList = {
+  type: "rich_text_list";
+  style: "bullet" | "ordered";
+  elements: RichTextSection[];
+};
+
+type RichTextQuote = {
+  type: "rich_text_quote";
+  elements: RichTextElement[];
+};
+
+type RichTextBlockElement =
+  | RichTextSection
+  | RichTextPreformatted
+  | RichTextList
+  | RichTextQuote
+  | { type: string; [key: string]: unknown };
+
+type RichTextBlock = {
+  type: "rich_text";
+  elements: RichTextBlockElement[];
+};
+
+function isTextElement(
+  el: RichTextElement,
+): el is Extract<RichTextElement, { type: "text" }> {
+  return el.type === "text";
+}
+
+function isLinkElement(
+  el: RichTextElement,
+): el is Extract<RichTextElement, { type: "link" }> {
+  return el.type === "link";
+}
+
+function isEmojiElement(
+  el: RichTextElement,
+): el is Extract<RichTextElement, { type: "emoji" }> {
+  return el.type === "emoji";
+}
+
+function isUserElement(
+  el: RichTextElement,
+): el is Extract<RichTextElement, { type: "user" }> {
+  return el.type === "user";
+}
+
+function convertInlineElement(el: RichTextElement): string {
+  if (isTextElement(el)) {
+    const style = el.style ?? {};
+    let text = el.text;
+    if (style.code) text = `\`${text}\``;
+    if (style.bold) text = `**${text}**`;
+    if (style.italic) text = `*${text}*`;
+    if (style.strike) text = `~~${text}~~`;
+    return text;
+  }
+  if (isLinkElement(el)) {
+    return el.text ? `[${el.text}](${el.url})` : el.url;
+  }
+  if (isEmojiElement(el)) {
+    return `:${el.name}:`;
+  }
+  if (isUserElement(el)) {
+    return `<@${el.user_id}>`;
+  }
+  return "";
+}
+
+function convertSection(section: RichTextSection): string {
+  return section.elements.map(convertInlineElement).join("");
+}
+
+function convertRichTextBlock(block: RichTextBlock): string {
+  const parts: string[] = [];
+
+  for (const el of block.elements) {
+    if (el.type === "rich_text_section") {
+      parts.push(convertSection(el as RichTextSection));
+    } else if (el.type === "rich_text_preformatted") {
+      const pre = el as RichTextPreformatted;
+      const inner = pre.elements.map(convertInlineElement).join("");
+      parts.push(`\`\`\`\n${inner}\n\`\`\``);
+    } else if (el.type === "rich_text_list") {
+      const list = el as RichTextList;
+      const items = list.elements.map((item, i) => {
+        const text = convertSection(item);
+        return list.style === "ordered" ? `${i + 1}. ${text}` : `- ${text}`;
+      });
+      parts.push(items.join("\n"));
+    } else if (el.type === "rich_text_quote") {
+      const quote = el as RichTextQuote;
+      const inner = quote.elements.map(convertInlineElement).join("");
+      const quoted = inner
+        .split("\n")
+        .map((line) => `> ${line}`)
+        .join("\n");
+      parts.push(quoted);
+    }
+  }
+
+  return parts.join("\n\n");
+}
+
+/**
+ * Converts Slack rich_text blocks to markdown.
+ * Walks the blocks array for any `{type: "rich_text"}` block and converts
+ * its elements to markdown. Returns empty string if no rich_text block found.
+ */
+export function richTextToMarkdown(blocks: unknown[]): string {
+  const richBlocks = blocks.filter(
+    (b): b is RichTextBlock =>
+      b != null &&
+      typeof b === "object" &&
+      "type" in b &&
+      b.type === "rich_text",
+  );
+
+  if (richBlocks.length === 0) return "";
+
+  return richBlocks.map(convertRichTextBlock).join("\n");
+}
+
+/**
+ * Builds a Slack message for a shareable plan/spec link: a single mrkdwn
+ * section block (`<url|View plan>`, the Slack link form `markdownToSlack`
+ * emits) plus a plain-text fallback. Used by the [plan:url] response marker.
+ */
+export function formatPlanLink(url: string): {
+  text: string;
+  blocks: SlackBlock[];
+} {
+  return {
+    text: `View plan: ${url}`,
+    blocks: [
+      {
+        type: "section",
+        text: { type: "mrkdwn", text: `<${url}|View plan>` },
+      },
+    ],
+  };
+}