@everme/codex 0.6.4 → 0.6.5

package/README.md

@@ -14,13 +14,56 @@ shipped inside the EverMe Codex marketplace plugin and uses the stable
   visible assistant/tool trajectory; when no real user exists, retain genuine
   assistant/tool activity instead of requiring a synthetic user boundary.
   The hook also discovers completed descendant rollouts in the same session
-  directory and saves each child assistant/tool trajectory under its own
-  conversation id and checkpoint. Flush extraction every five root turns.
+  directory and skips children without a real user after filtering, without
+  advancing their checkpoints. Flush extraction every five root turns.
   Developer prompts, injected context, UI notifications, reasoning, and
-  subagent user/task context are excluded; child execution retains the same
-  Codex sender identity without being merged into the root conversation.
+  subagent user/task context are excluded. Goal continuations retain only the
+  explicitly user-provided objective, not scheduler instructions or budgets.
 - `PreCompact`: send a flush-only request.
 
+Native paginated rollouts isolate transcript checkpoints by the filename's
+thread UUID (`codex-fragment-<uuid>`), even when multiple files share the hook
+session ID. The same scope accompanies every write and matches cold-start
+fragment coordinates; logical conversation attribution stays unchanged.
+Unscoped legacy checkpoints are not copied into fragment state. A newly seen
+root fragment still captures only its last turn, then captures appended messages
+on later Stops; this is not automatic historical replay. Before writing, the SDK
+requires `hookSyncScopeSupported` from the server's memory-state endpoint.
+Deploy the server before the plugin; unsupported servers leave the checkpoint
+unchanged. Separate session flush requests retain their existing contract.
+
+Both realtime capture and Codex cold-start import first plan user tasks. Fitting
+tasks keep their tool results unchanged. Tasks exceeding the 280 KiB message
+budget try 4, 3, then 2 KiB UTF-8 tool-result budgets from original content,
+retaining 70% head and 30% tail around a byte-count marker. Compression changes
+no IDs, arguments, timestamps or ordering. Count-only overflow is not compressed.
+The 64 KiB soft target cannot split a fitting task; remaining byte or 500-message
+overflow prefers text edges, then complete tool pairs, then hard cuts. Hard
+fallbacks cannot guarantee a user or text edges in every batch. Existing
+redaction and ordinary-text budgets remain unchanged, as do checkpoint and
+flush semantics. The shared SDK strategy is opt-in, currently enabled by Codex.
+
+Both paths retain recorded empty tool results as empty content with their
+original call id. Legacy wrappers without native IDs are dropped with bounded
+count diagnostics; neither line-number IDs nor inferred pairing are used.
+A standalone `web_search_call` has no paired result:
+its source payload is retained as JSON text in an assistant message, subject to
+the normal redaction and length limits. It does not create a synthetic tool call
+or result, and an absent source status is not replaced with a success claim.
+
+Tool arguments are JSON-encoded strings: existing JSON is retained, empty
+arguments become `{}`, and unstructured legacy/function input is preserved as
+`{"input":"original text"}`. Local `turnId` and tri-state `turnComplete`
+metadata guide SDK batching without adding fields to the BFF message contract.
+Native function/custom calls and results must carry a non-blank string
+`call_id`. Invalid records are dropped with bounded count diagnostics; no
+fallback ID is generated and no missing ID is inferred from pending calls.
+Valid source IDs are preserved unchanged.
+Explicit final answers and completion events identify completed turns;
+commentary and standalone search records do not close a turn. Unknown or
+malformed records and filtering reasons produce one bounded count summary per
+transcript, without source text or credentials.
+
 All hook failures are fail-open and credentials are redacted from diagnostics.
 `evercli plugin install codex` writes credentials to `~/.codex/everme.env`
 with mode `0600`; the user reviews and trusts the commands in `/hooks`. Hook

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everme/codex",
-  "version": "0.6.4",
+  "version": "0.6.5",
   "type": "module",
   "description": "Native EverMe lifecycle hooks for Codex.",
   "license": "Apache-2.0",
@@ -40,7 +40,7 @@
     "registry": "https://registry.npmjs.org"
   },
   "dependencies": {
-    "@everme/agent-sdk": "^0.6.4"
+    "@everme/agent-sdk": "^0.6.5"
   },
   "devDependencies": {
     "esbuild": "^0.28.1"

package/src/adapter.js

@@ -11,6 +11,7 @@ export const codexAdapter = {
   // every one of them, so the SDK claims channel="hook" for these writes and
   // they enter L1-2's denominator. Whole-session hosts leave this unset.
   turnBoundary: "stop",
+  taskBatching: true,
 
   envFile() {
     return process.env.EVERME_ENV_FILE_PATH || path.join(os.homedir(), ".codex", "everme.env");
@@ -31,11 +32,11 @@ export const codexAdapter = {
   },
 
   readLastTurn(input) {
-    return readLastTurn(input?.transcriptPath);
+    return readLastTurn(input?.transcriptPath, { diagnostic: (line) => console.error(line) });
   },
 
   readStoreBatches(input, options) {
-    return readCodexStoreBatches(input, options);
+    return readCodexStoreBatches(input, { ...options, diagnostic: (line) => console.error(line) });
   },
 
   formatOutput(event, { block = "" } = {}) {

package/src/fragment.js

@@ -0,0 +1,31 @@
+import { createReadStream } from "node:fs";
+import path from "node:path";
+import { createInterface } from "node:readline";
+
+const UUID = "[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}";
+const ROLLOUT = new RegExp(`^rollout-\\d{4}-\\d{2}-\\d{2}T\\d{2}-\\d{2}-\\d{2}-(${UUID})(?:_(${UUID}))?\\.jsonl$`);
+
+export async function readFragmentScope(transcriptPath) {
+  const stream = createReadStream(transcriptPath, { encoding: "utf8" });
+  const lines = createInterface({ input: stream, crlfDelay: Infinity });
+  try {
+    for await (const line of lines) {
+      let event;
+      try { event = JSON.parse(line); } catch { continue; }
+      if (event?.type !== "session_meta") continue;
+      if (event.payload?.history_mode !== "paginated") return "";
+      const match = path.basename(transcriptPath).match(ROLLOUT);
+      if (!match) {
+        if (event.payload?.history_base?.thread_id) {
+          throw new Error("codex paginated fragment has no native rollout filename identity");
+        }
+        return "";
+      }
+      return `codex-fragment-${match[2] || match[1]}`;
+    }
+    return "";
+  } finally {
+    lines.close();
+    stream.destroy();
+  }
+}

package/src/store-batches.js

@@ -2,12 +2,14 @@ import { readFile, readdir } from "node:fs/promises";
 import path from "node:path";
 import { readCanonicalTranscript, readLastTurn } from "./transcript.js";
 import { turnOrdinals } from "@everme/agent-sdk";
+import { readFragmentScope } from "./fragment.js";
 
-export async function readCodexStoreBatches(input, { checkpointStore } = {}) {
+export async function readCodexStoreBatches(input, { checkpointStore, diagnostic = () => {} } = {}) {
   if (!input?.sessionId || !input?.transcriptPath) return [];
   const batches = [];
   const root = await transcriptBatch({
     checkpointStore,
+    diagnostic,
     conversationId: input.sessionId,
     initialTailOnly: true,
     transcriptPath: input.transcriptPath,
@@ -17,6 +19,7 @@ export async function readCodexStoreBatches(input, { checkpointStore } = {}) {
   for (const child of await completedDescendants(input.transcriptPath, input.sessionId)) {
     const batch = await transcriptBatch({
       checkpointStore,
+      diagnostic,
       conversationId: child.id,
       initialTailOnly: false,
       transcriptPath: child.path,
@@ -26,9 +29,11 @@ export async function readCodexStoreBatches(input, { checkpointStore } = {}) {
   return batches;
 }
 
-async function transcriptBatch({ checkpointStore, conversationId, initialTailOnly, transcriptPath }) {
-  const canonical = await readCanonicalTranscript(transcriptPath);
-  const stateId = `codex:${conversationId}`;
+async function transcriptBatch({ checkpointStore, conversationId, initialTailOnly, transcriptPath, diagnostic }) {
+  const canonical = await readCanonicalTranscript(transcriptPath, { diagnostic });
+  if (!canonical.length) return null;
+  const syncScope = await readFragmentScope(transcriptPath);
+  const stateId = syncScope ? `codex:${conversationId}:${syncScope}` : `codex:${conversationId}`;
   const checkpoint = checkpointStore
     ? await checkpointStore.read(stateId)
     : { initialized: false, uploadedCount: 0 };
@@ -39,6 +44,7 @@ async function transcriptBatch({ checkpointStore, conversationId, initialTailOnl
   if (!messages.length) return null;
   return {
     conversationId,
+    ...(syncScope ? { syncScope } : {}),
     messages,
     // Address by absolute turn only when everything before the slice is
     // known to be covered: a checkpoint continuation, or the whole canonical

package/src/transcript.js

@@ -25,15 +25,23 @@ const SYNTHETIC_USER_TAGS = [
   "turn_aborted",
 ];
 
-export function readLastTurn(transcriptPath) {
-  return readTranscript(transcriptPath, { lastTurnOnly: true });
+const KNOWN_EVENT_MESSAGES = new Set([
+  "agent_message", "agent_reasoning", "context_compacted", "item_completed",
+  "mcp_tool_call_begin", "mcp_tool_call_end", "patch_apply_begin", "patch_apply_end",
+  "reasoning", "task_complete", "task_started", "sub_agent_activity", "thread_goal_updated",
+  "thread_rolled_back", "thread_settings_applied", "token_count", "turn_aborted",
+  "turn_complete", "user_message", "web_search_begin", "web_search_end",
+]);
+
+export function readLastTurn(transcriptPath, options = {}) {
+  return readTranscript(transcriptPath, { ...options, lastTurnOnly: true });
 }
 
-export function readCanonicalTranscript(transcriptPath) {
-  return readTranscript(transcriptPath, { lastTurnOnly: false });
+export function readCanonicalTranscript(transcriptPath, options = {}) {
+  return readTranscript(transcriptPath, { ...options, lastTurnOnly: false });
 }
 
-async function readTranscript(transcriptPath, { lastTurnOnly }) {
+async function readTranscript(transcriptPath, { lastTurnOnly, diagnostic = () => {} }) {
   if (!transcriptPath) return [];
   const legacyUsersBySegment = await collectLegacyUserMessages(transcriptPath);
   const fallbackTimestampBase = await transcriptFallbackTimestampBase(transcriptPath);
@@ -53,6 +61,7 @@ async function readTranscript(transcriptPath, { lastTurnOnly }) {
     try {
       event = JSON.parse(line);
     } catch {
+      countDiagnostic(state, "malformed_json");
       continue;
     }
     const payload = event?.payload;
@@ -61,11 +70,34 @@ async function readTranscript(transcriptPath, { lastTurnOnly }) {
       if (lastTurnOnly) messages = [];
       continue;
     }
-    if (skipsInheritedSubagentEvent(state, event)) continue;
+    if (skipsInheritedSubagentEvent(state, event)) {
+      countDiagnostic(state, "inherited_context");
+      continue;
+    }
+    observeTurn(state, payload);
+    if (event?.type === "turn_context") {
+      continue;
+    }
+    if (["task_complete", "turn_complete"].includes(event?.type) || (event?.type === "event_msg"
+      && ["task_complete", "turn_complete"].includes(payload?.type))) {
+      if (event.type !== "event_msg" && typeof event.turn_id === "string" && event.turn_id) {
+        observeTurn(state, event);
+      }
+      const last = messages.at(-1);
+      if (last && last === state.lastAssistant && last.turnComplete !== false
+        && (last.turnId || "") === state.turnId) {
+        setBoundaryMetadata(last, "turnComplete", true);
+      }
+      continue;
+    }
     if (event?.type === "event_msg") {
+      if (!KNOWN_EVENT_MESSAGES.has(payload?.type)) countDiagnostic(state, "unknown_event_message");
+      continue;
+    }
+    if (event?.type !== "response_item" || !payload) {
+      if (!["compacted"].includes(event?.type)) countDiagnostic(state, "unknown_event");
       continue;
     }
-    if (event?.type !== "response_item" || !payload) continue;
 
     const message = mapPayload(
       payload,
@@ -75,15 +107,58 @@ async function readTranscript(transcriptPath, { lastTurnOnly }) {
       state,
     );
     if (!message) continue;
+    if (state.turnId) setBoundaryMetadata(message, "turnId", state.turnId);
     if (lastTurnOnly && message.role === "user") {
       messages = [message];
     } else {
       messages.push(message);
     }
   }
+  if (state.isSubagent && !messages.some((message) => message.role === "user")) {
+    countDiagnostic(state, "subagent_without_user");
+    messages = [];
+  }
+  diagnoseToolPairs(messages, state);
+  if (Object.keys(state.diagnostics).length) {
+    const counts = Object.entries(state.diagnostics).map(([reason, count]) => `${reason}=${count}`).join(" ");
+    diagnostic(`[everme] codex parse lines=${lineNumber} messages=${messages.length} ${counts}`.slice(0, 1024));
+  }
   return messages;
 }
 
+function countDiagnostic(state, reason) {
+  state.diagnostics[reason] = (state.diagnostics[reason] || 0) + 1;
+}
+
+function sourceCallId(payload) {
+  return typeof payload?.call_id === "string" && payload.call_id.trim() ? payload.call_id : "";
+}
+
+function diagnoseToolPairs(messages, state) {
+  const seen = new Set();
+  const pending = new Map();
+  for (const message of messages) {
+    for (const call of message.toolCalls || []) {
+      if (seen.has(call.id)) countDiagnostic(state, "duplicate_call_id");
+      seen.add(call.id);
+      pending.set(call.id, (pending.get(call.id) || 0) + 1);
+    }
+    if (message.role !== "tool") continue;
+    const count = pending.get(message.toolCallId) || 0;
+    if (!count) countDiagnostic(state, "orphan_result");
+    else if (count === 1) pending.delete(message.toolCallId);
+    else pending.set(message.toolCallId, count - 1);
+  }
+  for (const count of pending.values()) {
+    state.diagnostics.missing_result = (state.diagnostics.missing_result || 0) + count;
+  }
+}
+
+function setBoundaryMetadata(message, key, value) {
+  Object.defineProperty(message, key, { value, writable: true, configurable: true });
+  return message;
+}
+
 async function collectLegacyUserMessages(transcriptPath) {
   const usersBySegment = [];
   const lines = createInterface({
@@ -115,6 +190,12 @@ async function collectLegacyUserMessages(transcriptPath) {
   return usersBySegment;
 }
 
+function observeTurn(state, payload) {
+  const turnId = payload?.turn_id || payload?.internal_chat_message_metadata_passthrough?.turn_id;
+  if (typeof turnId !== "string" || !turnId) return;
+  state.turnId = turnId;
+}
+
 function newParseState(legacyUsersBySegment) {
   return {
     historyMode: "",
@@ -126,7 +207,9 @@ function newParseState(legacyUsersBySegment) {
     subagentHistoryStart: 0,
     legacyUsersBySegment,
     legacyUserMessages: new Map(legacyUsersBySegment[0] || []),
-    pendingLegacyToolCallIds: [],
+    diagnostics: {},
+    turnId: "",
+    lastAssistant: null,
   };
 }
 
@@ -148,7 +231,8 @@ function observeSessionMeta(state, payload) {
     state.isSubagent = true;
   }
   state.legacyUserMessages = new Map(state.legacyUsersBySegment[state.segmentIndex] || []);
-  state.pendingLegacyToolCallIds = [];
+  state.turnId = "";
+  state.lastAssistant = null;
 }
 
 function skipsInheritedSubagentEvent(state, event) {
@@ -167,57 +251,94 @@ function sessionMetaIsSubagent(payload) {
 function mapPayload(payload, timestampValue, fallbackTimestamp, lineNumber, state) {
   const timestamp = normalizeTimestamp(timestampValue, fallbackTimestamp);
   if (payload.type === "message") {
-    if (payload.role === "developer") return null;
+    if (["developer", "system"].includes(payload.role)) {
+      countDiagnostic(state, "system_message");
+      return null;
+    }
     const rawText = contentText(payload.content);
-    if (!rawText) return null;
+    if (!rawText) {
+      countDiagnostic(state, "empty_or_nontext_message");
+      return null;
+    }
     if (payload.role === "user") {
       // Subagent user records are inherited parent context or an agent-authored
       // task, not a human utterance.
-      if (state.isSubagent) return null;
+      if (state.isSubagent) {
+        countDiagnostic(state, "subagent_user");
+        return null;
+      }
       const content = normalizeUserMessage(state, rawText);
-      if (!content) return null;
-      state.pendingLegacyToolCallIds = [];
+      if (!content) {
+        countDiagnostic(state, "injected_user");
+        return null;
+      }
+      if (content !== rawText) countDiagnostic(state, "stripped_user_wrapper");
       return { role: "user", ...stamp(timestamp), content: capText(content) };
     }
-    if (payload.role !== "assistant") return null;
-    const legacyTool = mapLegacyToolMessage(state, rawText, timestamp, lineNumber);
-    if (legacyTool.matched) return legacyTool.message;
-    return { role: "assistant", ...stamp(timestamp), content: capText(rawText) };
+    if (payload.role !== "assistant") {
+      countDiagnostic(state, "unknown_role");
+      return null;
+    }
+    if (legacyToolEnvelope(rawText)) {
+      countDiagnostic(state, "dropped_legacy_tool_without_id");
+      return null;
+    }
+    const message = { role: "assistant", ...stamp(timestamp), content: capText(rawText) };
+    if (typeof payload.phase === "string" && payload.phase) {
+      setBoundaryMetadata(message, "turnComplete", payload.phase === "final_answer");
+    }
+    state.lastAssistant = message;
+    return message;
   }
   if (payload.type === "function_call" || payload.type === "custom_tool_call") {
     const custom = payload.type === "custom_tool_call";
+    const callId = sourceCallId(payload);
+    if (!callId) {
+      countDiagnostic(state, "dropped_missing_call_id");
+      return null;
+    }
     return {
       role: "assistant",
       ...stamp(timestamp),
       toolCalls: [{
-        id: payload.call_id || `${custom ? "codex_custom_tool" : "codex_tool"}_${timestamp ?? "untimed"}`,
+        id: callId,
         type: "function",
         name: payload.name || "unknown",
-        arguments: redactText(argumentText(custom ? payload.input : payload.arguments)),
+        arguments: redactText(custom ? customToolArguments(payload.input) : argumentText(payload.arguments)),
       }],
     };
   }
-  if (["function_call_output", "custom_tool_call_output"].includes(payload.type) && payload.call_id) {
+  if (["function_call_output", "custom_tool_call_output"].includes(payload.type)) {
+    const toolCallId = sourceCallId(payload);
+    if (!toolCallId) {
+      countDiagnostic(state, "dropped_missing_result_id");
+      return null;
+    }
+    if (!Object.hasOwn(payload, "output")) countDiagnostic(state, "missing_output_field");
     return {
       role: "tool",
       ...stamp(timestamp),
-      toolCallId: payload.call_id,
-      content: capText(outputText(payload.output) || "tool result"),
+      toolCallId,
+      content: redactText(outputText(payload.output)),
     };
   }
   if (payload.type === "web_search_call") {
-    return {
+    return setBoundaryMetadata({
       role: "assistant",
       ...stamp(timestamp),
-      toolCalls: [{
-        id: `codex_web_search_${timestamp ?? "untimed"}_${lineNumber}`,
-        type: "function",
-        name: "web_search",
-        arguments: redactText(argumentText(payload.action)),
-      }],
-    };
+      // This source record has no paired result; preserve it without inventing one.
+      content: capText(JSON.stringify(payload)),
+    }, "turnComplete", false);
+  }
+  if (payload.type === "agent_message") {
+    countDiagnostic(state, "internal_agent_message");
+    return null;
   }
-  if (payload.type === "agent_message") return null;
+  if (payload.type === "reasoning") {
+    countDiagnostic(state, "internal_reasoning");
+    return null;
+  }
+  countDiagnostic(state, "unknown_payload");
   return null;
 }
 
@@ -235,6 +356,8 @@ function normalizePaginatedUserText(text) {
   let trimmed = text.trim();
   const tags = [...INJECTED_CONTEXT_TAGS, ...SYNTHETIC_USER_TAGS, "command-args", "command-message"];
   while (trimmed) {
+    const objective = goalObjective(trimmed);
+    if (objective) return objective;
     const command = commandIntent(trimmed);
     if (command) return command;
 
@@ -299,43 +422,18 @@ function envelopeValue(text, tag) {
   return end < 0 ? "" : text.slice(valueStart, end).trim();
 }
 
-function mapLegacyToolMessage(state, text, timestamp, lineNumber) {
-  const envelope = legacyToolEnvelope(text);
-  if (!envelope) return { matched: false, message: null };
-  if (envelope.kind === "call") {
-    const callId = `codex_legacy_tool_${lineNumber}`;
-    state.pendingLegacyToolCallIds.push(callId);
-    return {
-      matched: true,
-      message: {
-        role: "assistant",
-        ...stamp(timestamp),
-        toolCalls: [{
-          id: callId,
-          type: "function",
-          name: envelope.name,
-          arguments: redactText(envelope.body),
-        }],
-      },
-    };
-  }
-  if (state.pendingLegacyToolCallIds.length !== 1) {
-    // Legacy wrappers carry no call id; ambiguous parallel calls must not be
-    // paired by guesswork and persisted as a false trajectory.
-    state.pendingLegacyToolCallIds = [];
-    return { matched: true, message: null };
-  }
-  const [toolCallId] = state.pendingLegacyToolCallIds;
-  state.pendingLegacyToolCallIds = [];
-  return {
-    matched: true,
-    message: {
-      role: "tool",
-      ...stamp(timestamp),
-      toolCallId,
-      content: capText(envelope.body || "tool result"),
-    },
-  };
+function goalObjective(text) {
+  const prefix = '<codex_internal_context source="goal">';
+  if (!text.startsWith(prefix)) return "";
+  const close = "</codex_internal_context>";
+  const end = text.indexOf(close, prefix.length);
+  if (end < 0) return "";
+  const body = text.slice(prefix.length, end);
+  if (!body.includes("The objective below is user-provided data.")) return "";
+  const objective = envelopeValue(body, "objective");
+  if (!objective) return "";
+  const suffix = normalizePaginatedUserText(text.slice(end + close.length));
+  return suffix ? `${objective}\n\n${suffix}` : objective;
 }
 
 function legacyToolEnvelope(text) {
@@ -372,7 +470,15 @@ function outputText(value) {
 }
 
 function argumentText(value) {
-  if (typeof value === "string") return value;
+  if (typeof value === "string") {
+    if (!value.trim()) return "{}";
+    try {
+      JSON.parse(value);
+      return value;
+    } catch {
+      return JSON.stringify({ input: value });
+    }
+  }
   try {
     return JSON.stringify(value ?? {});
   } catch {
@@ -380,6 +486,15 @@ function argumentText(value) {
   }
 }
 
+function customToolArguments(value) {
+  if (value == null || value === "") return "{}";
+  try {
+    return JSON.stringify({ input: value });
+  } catch {
+    return "{}";
+  }
+}
+
 function normalizeTimestamp(value, fallbackTimestamp) {
   if (typeof value === "number" && Number.isFinite(value)) {
     return value > 10_000_000_000 ? Math.trunc(value) : Math.trunc(value * 1000);