@everme/codex 0.6.1 → 0.6.3

package/README.md

@@ -10,8 +10,15 @@ shipped inside the EverMe Codex marketplace plugin and uses the stable
 - `SessionStart`: inject the EverMe profile snapshot.
 - `UserPromptSubmit`: sanitize the prompt, search top 10 memories, and inject
   recall without passive profile rows by default.
-- `Stop`: stream the Codex rollout, save only the latest user turn, and flush
-  extraction every five turns.
+- `Stop`: stream the Codex rollout, save the latest real user turn and its
+  visible assistant/tool trajectory; when no real user exists, retain genuine
+  assistant/tool activity instead of requiring a synthetic user boundary.
+  The hook also discovers completed descendant rollouts in the same session
+  directory and saves each child assistant/tool trajectory under its own
+  conversation id and checkpoint. Flush extraction every five root turns.
+  Developer prompts, injected context, UI notifications, reasoning, and
+  subagent user/task context are excluded; child execution retains the same
+  Codex sender identity without being merged into the root conversation.
 - `PreCompact`: send a flush-only request.
 
 All hook failures are fail-open and credentials are redacted from diagnostics.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everme/codex",
-  "version": "0.6.1",
+  "version": "0.6.3",
   "type": "module",
   "description": "Native EverMe lifecycle hooks for Codex.",
   "license": "Apache-2.0",
@@ -40,7 +40,7 @@
     "registry": "https://registry.npmjs.org"
   },
   "dependencies": {
-    "@everme/agent-sdk": "^0.6.1"
+    "@everme/agent-sdk": "^0.6.3"
   },
   "devDependencies": {
     "esbuild": "^0.28.1"

package/src/adapter.js

@@ -1,11 +1,16 @@
 import os from "node:os";
 import path from "node:path";
+import { readCodexStoreBatches } from "./store-batches.js";
 import { readLastTurn } from "./transcript.js";
 
 const CONTEXT_EVENTS = new Set(["SessionStart", "UserPromptSubmit"]);
 
 export const codexAdapter = {
   platform: "codex",
+  // One hook invocation is one logical turn and the delivery marker runs on
+  // every one of them, so the SDK claims channel="hook" for these writes and
+  // they enter L1-2's denominator. Whole-session hosts leave this unset.
+  turnBoundary: "stop",
 
   envFile() {
     return process.env.EVERME_ENV_FILE_PATH || path.join(os.homedir(), ".codex", "everme.env");
@@ -29,6 +34,10 @@ export const codexAdapter = {
     return readLastTurn(input?.transcriptPath);
   },
 
+  readStoreBatches(input, options) {
+    return readCodexStoreBatches(input, options);
+  },
+
   formatOutput(event, { block = "" } = {}) {
     if (!CONTEXT_EVENTS.has(event) || !block) return {};
     return {

package/src/store-batches.js

@@ -0,0 +1,120 @@
+import { readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+import { readCanonicalTranscript, readLastTurn } from "./transcript.js";
+
+export async function readCodexStoreBatches(input, { checkpointStore } = {}) {
+  if (!input?.sessionId || !input?.transcriptPath) return [];
+  const batches = [];
+  const root = await transcriptBatch({
+    checkpointStore,
+    conversationId: input.sessionId,
+    initialTailOnly: true,
+    transcriptPath: input.transcriptPath,
+  });
+  if (root) batches.push(root);
+
+  for (const child of await completedDescendants(input.transcriptPath, input.sessionId)) {
+    const batch = await transcriptBatch({
+      checkpointStore,
+      conversationId: child.id,
+      initialTailOnly: false,
+      transcriptPath: child.path,
+    });
+    if (batch) batches.push(batch);
+  }
+  return batches;
+}
+
+async function transcriptBatch({ checkpointStore, conversationId, initialTailOnly, transcriptPath }) {
+  const canonical = await readCanonicalTranscript(transcriptPath);
+  const stateId = `codex:${conversationId}`;
+  const checkpoint = checkpointStore
+    ? await checkpointStore.read(stateId)
+    : { initialized: false, uploadedCount: 0 };
+  let messages;
+  if (checkpoint.initialized && checkpoint.uploadedCount <= canonical.length) {
+    messages = canonical.slice(checkpoint.uploadedCount);
+  } else {
+    messages = initialTailOnly ? await readLastTurn(transcriptPath) : canonical;
+  }
+  if (!messages.length) return null;
+  return {
+    conversationId,
+    messages,
+    checkpoint: { stateId, uploadedCount: canonical.length },
+  };
+}
+
+async function completedDescendants(rootPath, rootId) {
+  let names;
+  try {
+    names = await readdir(path.dirname(rootPath));
+  } catch {
+    return [];
+  }
+  const candidates = [];
+  for (const name of names) {
+    if (!name.endsWith(".jsonl")) continue;
+    const candidatePath = path.join(path.dirname(rootPath), name);
+    if (candidatePath === rootPath) continue;
+    const metadata = await rolloutMetadata(candidatePath);
+    if (metadata?.isSubagent && metadata.complete) {
+      candidates.push({ ...metadata, path: candidatePath });
+    }
+  }
+
+  const descendants = [];
+  const parents = new Set([rootId]);
+  let changed = true;
+  while (changed) {
+    changed = false;
+    for (const candidate of candidates) {
+      if (candidate.selected || !parents.has(candidate.parentId)) continue;
+      candidate.selected = true;
+      parents.add(candidate.id);
+      descendants.push(candidate);
+      changed = true;
+    }
+  }
+  return descendants;
+}
+
+async function rolloutMetadata(transcriptPath) {
+  let raw;
+  try {
+    raw = await readFile(transcriptPath, "utf8");
+  } catch {
+    return null;
+  }
+  let metadata;
+  let complete = false;
+  for (const line of raw.split("\n")) {
+    if (!line.trim()) continue;
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    if (!metadata && event?.type === "session_meta") {
+      const payload = event.payload || {};
+      const id = stringValue(payload.id || payload.session_id);
+      const parentId = stringValue(payload.parent_thread_id || payload.forked_from_id);
+      metadata = {
+        id,
+        parentId,
+        isSubagent: payload.thread_source === "subagent" || Boolean(parentId),
+      };
+    }
+    if ((event?.type === "event_msg" && event?.payload?.type === "task_complete")
+      || event?.type === "task_complete") {
+      complete = true;
+    }
+  }
+  if (!metadata?.id || !metadata.parentId) return null;
+  return { ...metadata, complete };
+}
+
+function stringValue(value) {
+  return typeof value === "string" ? value.trim() : "";
+}

package/src/transcript.js

@@ -1,65 +1,351 @@
 import { createReadStream } from "node:fs";
+import { stat } from "node:fs/promises";
 import { createInterface } from "node:readline";
 import { capRunes } from "@everme/agent-sdk";
 
-export async function readLastTurn(transcriptPath) {
+const INJECTED_CONTEXT_TAGS = [
+  "app-context",
+  "apps_instructions",
+  "codex_internal_context",
+  "environment_context",
+  "in-app-browser-context",
+  "multi_agent_mode",
+  "permissions",
+  "plugins_instructions",
+  "recommended_plugins",
+  "skills_instructions",
+];
+
+const SYNTHETIC_USER_TAGS = [
+  "bash-input",
+  "bash-stdout",
+  "command-name",
+  "local-command-stdout",
+  "task-notification",
+  "turn_aborted",
+];
+
+export function readLastTurn(transcriptPath) {
+  return readTranscript(transcriptPath, { lastTurnOnly: true });
+}
+
+export function readCanonicalTranscript(transcriptPath) {
+  return readTranscript(transcriptPath, { lastTurnOnly: false });
+}
+
+async function readTranscript(transcriptPath, { lastTurnOnly }) {
   if (!transcriptPath) return [];
+  const legacyUsersBySegment = await collectLegacyUserMessages(transcriptPath);
+  const fallbackTimestampBase = await transcriptFallbackTimestampBase(transcriptPath);
   const lines = createInterface({
     input: createReadStream(transcriptPath, { encoding: "utf8" }),
     crlfDelay: Infinity,
   });
-  let delta = [];
-  let foundUser = false;
+  const state = newParseState(legacyUsersBySegment);
+  let messages = [];
+  let lineNumber = 0;
 
-  for await (const line of lines) {
+  for await (const rawLine of lines) {
+    const line = rawLine.trim();
+    if (!line) continue;
+    lineNumber += 1;
     let event;
     try {
       event = JSON.parse(line);
     } catch {
       continue;
     }
-    if (event?.type !== "response_item" || !event.payload) continue;
-    const message = mapPayload(event.payload, event.timestamp);
+    const payload = event?.payload;
+    if (event?.type === "session_meta") {
+      observeSessionMeta(state, payload);
+      if (lastTurnOnly) messages = [];
+      continue;
+    }
+    if (skipsInheritedSubagentEvent(state, event)) continue;
+    if (event?.type === "event_msg") {
+      continue;
+    }
+    if (event?.type !== "response_item" || !payload) continue;
+
+    const message = mapPayload(
+      payload,
+      event.timestamp,
+      fallbackTimestampBase + lineNumber,
+      lineNumber,
+      state,
+    );
     if (!message) continue;
+    if (lastTurnOnly && message.role === "user") {
+      messages = [message];
+    } else {
+      messages.push(message);
+    }
+  }
+  return messages;
+}
 
-    if (message.role === "user") {
-      delta = [message];
-      foundUser = true;
-    } else if (foundUser) {
-      delta.push(message);
+async function collectLegacyUserMessages(transcriptPath) {
+  const usersBySegment = [];
+  const lines = createInterface({
+    input: createReadStream(transcriptPath, { encoding: "utf8" }),
+    crlfDelay: Infinity,
+  });
+  const state = newParseState([]);
+  for await (const rawLine of lines) {
+    let event;
+    try {
+      event = JSON.parse(rawLine);
+    } catch {
+      continue;
     }
+    const payload = event?.payload;
+    if (event?.type === "session_meta") {
+      observeSessionMeta(state, payload);
+      continue;
+    }
+    if (skipsInheritedSubagentEvent(state, event) || state.isSubagent
+      || event?.type !== "event_msg" || payload?.type !== "user_message"
+      || typeof payload.message !== "string") continue;
+    const text = payload.message.trim();
+    if (!text) continue;
+    if (!usersBySegment[state.segmentIndex]) usersBySegment[state.segmentIndex] = new Map();
+    const users = usersBySegment[state.segmentIndex];
+    users.set(text, (users.get(text) || 0) + 1);
   }
-  return foundUser ? delta : [];
+  return usersBySegment;
+}
+
+function newParseState(legacyUsersBySegment) {
+  return {
+    historyMode: "",
+    isSubagent: false,
+    segmentIndex: 0,
+    seenSessionMeta: false,
+    outerSessionIsSubagent: false,
+    hasSubagentHistoryStart: false,
+    subagentHistoryStart: 0,
+    legacyUsersBySegment,
+    legacyUserMessages: new Map(legacyUsersBySegment[0] || []),
+    pendingLegacyToolCallIds: [],
+  };
 }
 
-function mapPayload(payload, timestampValue) {
-  const timestamp = normalizeTimestamp(timestampValue);
+function observeSessionMeta(state, payload) {
+  // Root rollouts can contain multiple sections. Child rollouts instead embed
+  // forked root metadata, so their outer subagent identity remains authoritative.
+  const firstSessionMeta = !state.seenSessionMeta;
+  if (!firstSessionMeta) state.segmentIndex += 1;
+  else state.seenSessionMeta = true;
+  if (firstSessionMeta) state.outerSessionIsSubagent = sessionMetaIsSubagent(payload);
+  if (firstSessionMeta || !state.outerSessionIsSubagent) {
+    state.historyMode = typeof payload?.history_mode === "string" ? payload.history_mode : "";
+    state.isSubagent = sessionMetaIsSubagent(payload);
+    state.hasSubagentHistoryStart = Number.isFinite(payload?.subagent_history_start_ordinal);
+    state.subagentHistoryStart = state.hasSubagentHistoryStart
+      ? Math.trunc(payload.subagent_history_start_ordinal)
+      : 0;
+  } else {
+    state.isSubagent = true;
+  }
+  state.legacyUserMessages = new Map(state.legacyUsersBySegment[state.segmentIndex] || []);
+  state.pendingLegacyToolCallIds = [];
+}
+
+function skipsInheritedSubagentEvent(state, event) {
+  return state.isSubagent && state.hasSubagentHistoryStart
+    && Number.isFinite(event?.ordinal)
+    && Math.trunc(event.ordinal) < state.subagentHistoryStart;
+}
+
+function sessionMetaIsSubagent(payload) {
+  if (payload && payload.thread_source !== undefined && payload.thread_source !== null) {
+    return String(payload.thread_source).trim() === "subagent";
+  }
+  return Boolean(payload?.parent_thread_id);
+}
+
+function mapPayload(payload, timestampValue, fallbackTimestamp, lineNumber, state) {
+  const timestamp = normalizeTimestamp(timestampValue, fallbackTimestamp);
   if (payload.type === "message") {
-    if (payload.role !== "user" && payload.role !== "assistant") return null;
-    const content = contentText(payload.content);
-    if (!content) return null;
-    return { role: payload.role, ...stamp(timestamp), content };
+    if (payload.role === "developer") return null;
+    const rawText = contentText(payload.content);
+    if (!rawText) return null;
+    if (payload.role === "user") {
+      // Subagent user records are inherited parent context or an agent-authored
+      // task, not a human utterance.
+      if (state.isSubagent) return null;
+      const content = normalizeUserMessage(state, rawText);
+      if (!content) return null;
+      state.pendingLegacyToolCallIds = [];
+      return { role: "user", ...stamp(timestamp), content: capText(content) };
+    }
+    if (payload.role !== "assistant") return null;
+    const legacyTool = mapLegacyToolMessage(state, rawText, timestamp, lineNumber);
+    if (legacyTool.matched) return legacyTool.message;
+    return { role: "assistant", ...stamp(timestamp), content: capText(rawText) };
   }
-  if (payload.type === "function_call") {
+  if (payload.type === "function_call" || payload.type === "custom_tool_call") {
+    const custom = payload.type === "custom_tool_call";
     return {
       role: "assistant",
       ...stamp(timestamp),
       toolCalls: [{
-        id: payload.call_id || `codex_tool_${timestamp ?? "untimed"}`,
+        id: payload.call_id || `${custom ? "codex_custom_tool" : "codex_tool"}_${timestamp ?? "untimed"}`,
         type: "function",
         name: payload.name || "unknown",
-        arguments: argumentText(payload.arguments),
+        arguments: redactText(argumentText(custom ? payload.input : payload.arguments)),
       }],
     };
   }
-  if (payload.type === "function_call_output" && payload.call_id) {
+  if (["function_call_output", "custom_tool_call_output"].includes(payload.type) && payload.call_id) {
     return {
       role: "tool",
       ...stamp(timestamp),
       toolCallId: payload.call_id,
-      content: capText(payload.output || "tool result"),
+      content: capText(outputText(payload.output) || "tool result"),
+    };
+  }
+  if (payload.type === "web_search_call") {
+    return {
+      role: "assistant",
+      ...stamp(timestamp),
+      toolCalls: [{
+        id: `codex_web_search_${timestamp ?? "untimed"}_${lineNumber}`,
+        type: "function",
+        name: "web_search",
+        arguments: redactText(argumentText(payload.action)),
+      }],
     };
   }
+  if (payload.type === "agent_message") return null;
+  return null;
+}
+
+function normalizeUserMessage(state, text) {
+  if (state.historyMode !== "paginated") {
+    const remaining = state.legacyUserMessages.get(text) || 0;
+    if (remaining === 0) return "";
+    state.legacyUserMessages.set(text, remaining - 1);
+    return text;
+  }
+  return normalizePaginatedUserText(text);
+}
+
+function normalizePaginatedUserText(text) {
+  let trimmed = text.trim();
+  const tags = [...INJECTED_CONTEXT_TAGS, ...SYNTHETIC_USER_TAGS, "command-args", "command-message"];
+  while (trimmed) {
+    const command = commandIntent(trimmed);
+    if (command) return command;
+
+    const agentsRemainder = stripLeadingAgentsInstructions(trimmed);
+    if (agentsRemainder !== null) {
+      trimmed = agentsRemainder;
+      continue;
+    }
+
+    let stripped = false;
+    for (const tag of tags) {
+      const remainder = stripLeadingEnvelope(trimmed, tag);
+      if (remainder !== null) {
+        trimmed = remainder;
+        stripped = true;
+        break;
+      }
+    }
+    if (!stripped) break;
+  }
+  return !trimmed || hasEnvelopePrefix(trimmed, "command-message") ? "" : trimmed;
+}
+
+function commandIntent(text) {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith("<command-message>")) return "";
+  const name = envelopeValue(trimmed, "command-name");
+  if (!name?.startsWith("/")) return "";
+  const args = envelopeValue(trimmed, "command-args");
+  return `${name} ${args}`.trim();
+}
+
+function stripLeadingEnvelope(text, tag) {
+  if (!hasEnvelopePrefix(text, tag)) return null;
+  const openEnd = text.indexOf(">");
+  if (openEnd < 0) return null;
+  const close = `</${tag}>`;
+  const closeStart = text.indexOf(close, openEnd + 1);
+  if (closeStart < 0) return null;
+  return text.slice(closeStart + close.length).trim();
+}
+
+function hasEnvelopePrefix(text, tag) {
+  if (!text.startsWith(`<${tag}`)) return false;
+  return [">", " ", "\t", "\n", "\r"].includes(text.at(tag.length + 1));
+}
+
+function stripLeadingAgentsInstructions(text) {
+  if (!text.startsWith("# AGENTS.md instructions for ") || !text.includes("<INSTRUCTIONS>")) return null;
+  const close = "</INSTRUCTIONS>";
+  const closeStart = text.indexOf(close);
+  return closeStart < 0 ? "" : text.slice(closeStart + close.length).trim();
+}
+
+function envelopeValue(text, tag) {
+  const open = `<${tag}>`;
+  const close = `</${tag}>`;
+  const start = text.indexOf(open);
+  if (start < 0) return "";
+  const valueStart = start + open.length;
+  const end = text.indexOf(close, valueStart);
+  return end < 0 ? "" : text.slice(valueStart, end).trim();
+}
+
+function mapLegacyToolMessage(state, text, timestamp, lineNumber) {
+  const envelope = legacyToolEnvelope(text);
+  if (!envelope) return { matched: false, message: null };
+  if (envelope.kind === "call") {
+    const callId = `codex_legacy_tool_${lineNumber}`;
+    state.pendingLegacyToolCallIds.push(callId);
+    return {
+      matched: true,
+      message: {
+        role: "assistant",
+        ...stamp(timestamp),
+        toolCalls: [{
+          id: callId,
+          type: "function",
+          name: envelope.name,
+          arguments: redactText(envelope.body),
+        }],
+      },
+    };
+  }
+  if (state.pendingLegacyToolCallIds.length !== 1) {
+    // Legacy wrappers carry no call id; ambiguous parallel calls must not be
+    // paired by guesswork and persisted as a false trajectory.
+    state.pendingLegacyToolCallIds = [];
+    return { matched: true, message: null };
+  }
+  const [toolCallId] = state.pendingLegacyToolCallIds;
+  state.pendingLegacyToolCallIds = [];
+  return {
+    matched: true,
+    message: {
+      role: "tool",
+      ...stamp(timestamp),
+      toolCallId,
+      content: capText(envelope.body || "tool result"),
+    },
+  };
+}
+
+function legacyToolEnvelope(text) {
+  const trimmed = text.trim();
+  const callMatch = trimmed.match(/^\[external_agent_tool_call:\s*([^\]]+)\]\s*([\s\S]*?)\s*\[\/external_agent_tool_call\]$/);
+  if (callMatch) {
+    return { kind: "call", name: callMatch[1].trim() || "unknown", body: callMatch[2].trim() };
+  }
+  const resultMatch = trimmed.match(/^\[external_agent_tool_result\]\s*([\s\S]*?)\s*\[\/external_agent_tool_result\]$/);
+  if (resultMatch) return { kind: "result", body: resultMatch[1].trim() };
   return null;
 }
 
@@ -68,7 +354,7 @@ function stamp(timestamp) {
 }
 
 function contentText(content) {
-  if (typeof content === "string") return capText(content);
+  if (typeof content === "string") return content.trim();
   if (!Array.isArray(content)) return "";
   const parts = [];
   for (const item of content) {
@@ -78,7 +364,11 @@ function contentText(content) {
       parts.push(item.text);
     }
   }
-  return capText(parts.join("\n"));
+  return parts.join("\n").trim();
+}
+
+function outputText(value) {
+  return typeof value === "string" ? value.trim() : contentText(value);
 }
 
 function argumentText(value) {
@@ -90,19 +380,34 @@ function argumentText(value) {
   }
 }
 
-function normalizeTimestamp(value) {
+function normalizeTimestamp(value, fallbackTimestamp) {
   if (typeof value === "number" && Number.isFinite(value)) {
     return value > 10_000_000_000 ? Math.trunc(value) : Math.trunc(value * 1000);
   }
   const parsed = Date.parse(value);
-  // undefined — never 0: 0 is a finite epoch (1970) that the SDK would ship
-  // as-is, while a missing timestamp makes the SDK stamp Date.now() instead,
-  // keeping the epoch-ms wire contract honest.
-  return Number.isFinite(parsed) ? parsed : undefined;
+  return Number.isFinite(parsed) ? parsed : fallbackTimestamp;
+}
+
+async function transcriptFallbackTimestampBase(transcriptPath) {
+  try {
+    return Math.trunc((await stat(transcriptPath)).mtimeMs);
+  } catch {
+    return 0;
+  }
 }
 
-// SDK capRunes keeps head AND tail (0.7 head ratio) so the end of a long
-// tool output — exit status, root-cause line — survives truncation.
 function capText(value) {
-  return capRunes(String(value || "").trim());
+  return capRunes(redactText(String(value || "").trim()));
+}
+
+function redactText(value) {
+  return String(value || "")
+    .replace(/sk-[A-Za-z0-9_-]{16,}/g, "[redacted]")
+    .replace(/evt_[A-Za-z0-9_-]{8,}/g, "[redacted]")
+    .replace(/emk_[A-Za-z0-9_-]{8,}/g, "[redacted]")
+    .replace(/ghp_[A-Za-z0-9]{20,}/g, "[redacted]")
+    .replace(/AKIA[0-9A-Z]{16}/g, "[redacted]")
+    .replace(/bearer\s+[A-Za-z0-9._=-]{10,}/gi, "[redacted]")
+    .replace(/X-Amz-Signature=[A-Za-z0-9%]+/g, "[redacted]")
+    .replace(/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g, "[redacted]");
 }