@useshifu/coding-harness 0.3.23 → 0.3.25

package/bin/shifu-harness.js

@@ -26,6 +26,8 @@ const CREDENTIAL_ASSIGNMENT_PATTERN = /\b(?:token|secret|password|api[_-]?key)\s
 const CLOUD_CREDENTIAL_PATTERN = /\bAKIA[0-9A-Z]{16}\b|AIza[0-9A-Za-z_-]{20,}/;
 const EVIDENCE_KINDS = new Set(["implementation", "decision", "activity"]);
 const EVIDENCE_FIELDS = new Set(["kind", "title", "statement", "scope", "area", "itemRef", "verificationRefs"]);
+const REVIEWED_EVIDENCE_FIELDS = new Set([...EVIDENCE_FIELDS, "workContext", "decisionRefs", "sources"]);
+const SOURCE_FIELDS = new Set(["role", "turn", "locator", "eventAt", "excerptMode"]);
 const SYNC_FIELDS = new Set(["harness", "sessionRef", "fromTurn", "toTurn", "title", "summary", "evidence", "verification", "decisions", "redactionVersion", "supersedesSyncId", "force"]);
 const AI_PROFILE_FIELDS = new Set(["harness", "sessionRef", "fromTurn", "toTurn", "category", "model", "thinking", "evidence"]);
 const AI_PROFILE_EVIDENCE_FIELDS = new Set(["dimension", "statement", "signal", "turn"]);
@@ -40,8 +42,11 @@ const MAX_TURNS_PER_SEGMENT = 12;
 const MAX_TITLE_LENGTH = 160;
 const MAX_WORK_DETAIL_LENGTH = 600;
 const CONFIG_VERSION = 2;
-const RUNNER_VERSION = "0.3.23";
-const REDACTION_VERSION = 4;
+const RUNNER_VERSION = "0.3.25";
+const REDACTION_VERSION = 5;
+const LEGACY_REDACTION_VERSION = 4;
+const MAX_PREPARATIONS_PER_RUN = 4;
+const PREPARATION_RUN_MS = 5 * 60_000;
 const DEFAULT_INTERVAL_HOURS = 1;
 const LEGACY_INTERVAL_HOURS = 4;
 const MIN_INTERVAL_HOURS = 0.25;
@@ -110,11 +115,13 @@ function opencodeQuery(query) {
       throw new Error("Could not read OpenCode session metadata. Ensure the opencode command is installed and its local database is available.");
     }
   }
-  const temporary = path.join(os.tmpdir(), `shifu-opencode-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}.json`);
+  const temporary = path.join(os.tmpdir(), `shifu-opencode-${randomUUID()}.json`);
   let fd;
   try {
-    fd = fs.openSync(temporary, "w");
-    execFileSync(process.env.SHIFU_OPENCODE_BIN || "opencode", ["db", query, "--format", "json"], {
+    const { resolveNativeBinary, nativeEnvironment } = require("./native-preparation.js");
+    fd = fs.openSync(temporary, "wx", 0o600);
+    execFileSync(resolveNativeBinary("opencode"), ["db", query, "--format", "json"], {
+      env: nativeEnvironment(),
       stdio: ["ignore", fd, "pipe"],
       maxBuffer: 64 * 1024 * 1024,
     });
@@ -184,6 +191,7 @@ function validateSyncOptions(command, start = 3) {
   const options = {
     sessions: [...common, "--hours", "--days", "--count", "--turns", "--session-ref"],
     review: [...common, "--hours", "--days", "--turns", "--session-ref", "--from-turn", "--to-turn", "--supersedes-sync-id"],
+    "review-source": [...common, "--session-ref", "--from-turn", "--to-turn"],
     sync: common,
     "sync-files": common,
     "scheduled-sync": common,
@@ -559,6 +567,25 @@ function reviewSegment(harness, sessionRef, fromTurn, toTurn) {
   return harness === "codex" ? codexReview(selectedFile, fromTurn, toTurn) : claudeReview(selectedFile, fromTurn, toTurn);
 }
 
+function reviewSourcePacket(harness, sessionRef, fromTurn, toTurn, scheduled = false) {
+  if (!sessionRefIsSafe(sessionRef) || !OPAQUE_SESSION_REF_PATTERN.test(sessionRef)) throw new Error("Provide an opaque --session-ref for selected local source review.");
+  if (!Number.isInteger(fromTurn) || !Number.isInteger(toTurn) || fromTurn < 1 || toTurn < fromTurn || toTurn > 1_000_000 || toTurn - fromTurn >= MAX_TURNS_PER_SEGMENT) {
+    throw new Error("Select --from-turn and --to-turn explicitly, covering at most 12 original turns.");
+  }
+  const records = sourceRecords(harness, sessionRef);
+  const adapter = require("./source-review.js");
+  return (scheduled ? adapter.scheduledSourcePacket : adapter.selectedSourcePacket)({ harness, sessionRef, fromTurn, toTurn }, records);
+}
+
+function sourceRecords(harness, sessionRef) {
+  if (harness === "opencode") {
+    return opencodeQuery(`SELECT m.id AS messageId, m.time_created AS createdAt, json_extract(m.data, '$.role') AS role, json_extract(m.data, '$.finish') AS finish, json_extract(m.data, '$.error') AS error, p.data AS part FROM message m LEFT JOIN part p ON p.message_id = m.id WHERE m.session_id = ${JSON.stringify(sessionRef)} ORDER BY m.time_created, p.id`);
+  }
+  const file = sessionFile(harness, sessionRef);
+  try { return fs.readFileSync(file, "utf8").split("\n").filter(Boolean).map((line) => JSON.parse(line)); }
+  catch { throw new Error("Could not read selected original messages locally. Nothing was reconstructed or synced."); }
+}
+
 function sessionFile(harness, sessionRef) {
   const files = harness === "codex" ? sessionFiles(codexSessionsRoot()) : sessionFiles(claudeSessionsRoot());
   const selectedFile = files.map((file) => ({ file, session: harness === "codex" ? codexSession(file) : claudeSession(file) }))
@@ -934,6 +961,7 @@ function safeActivityEvidence(notes) {
   return evidence;
 }
 
+// Retained for legacy comparison adapters; schedules use prepareV5Payload.
 function payloadForSegment(harness, segment) {
   if (segment.force || segment.supersedesSyncId) throw new Error("Force sync requires reviewing final notes and preparing meaningful redacted evidence; do not use the unattended payload projection.");
   const { notes } = reviewSegment(harness, segment.sessionRef, segment.fromTurn, segment.toTurn);
@@ -955,7 +983,7 @@ function payloadForSegment(harness, segment) {
     evidence,
     verification: [],
     decisions: evidence.filter((item) => /^(?:Decided|Rejected) /.test(item.title)).slice(0, MAX_SYNC_ITEMS).map((item) => item.statement.length <= 240 ? item.statement : `${item.statement.slice(0, 239).replace(/\s+\S*$/, "")}…`),
-    redactionVersion: REDACTION_VERSION,
+    redactionVersion: LEGACY_REDACTION_VERSION,
   };
 }
 
@@ -1126,8 +1154,7 @@ function unsyncedCodexSessions(harness, options = {}) {
   return selectedSessionSegments(harness, options).filter((session) => session.fromTurn <= session.toTurn);
 }
 
-function pendingSegments(harness) {
-  const state = readState(harness);
+function pendingSegments(harness, state = readState(harness)) {
   const policy = normalizedPolicy(readConfig(harness));
   const excluded = new Set(policy.excludedSessionRefs);
   const segments = [];
@@ -1135,15 +1162,10 @@ function pendingSegments(harness) {
     .filter((candidate) => !excluded.has(candidate.sessionRef))
     .sort((left, right) => left.updatedAt.localeCompare(right.updatedAt))) {
     let fromTurn = (state.sessions[session.sessionRef]?.lastSyncedTurn || 0) + 1;
-    while (fromTurn <= session.turnCount) {
-      let toTurn = Math.min(session.turnCount, fromTurn + MAX_TURNS_PER_SEGMENT - 1);
-      if (Date.parse(session.updatedAt) > Date.now() - SCHEDULER_TICK_MINUTES * 60_000) {
-        const completedTurns = new Set(reviewSegment(harness, session.sessionRef, fromTurn, toTurn).notes.map((note) => note.turn));
-        let completedTo = fromTurn - 1;
-        while (completedTurns.has(completedTo + 1)) completedTo += 1;
-        if (completedTo < fromTurn) break;
-        toTurn = completedTo;
-      }
+    if (fromTurn > session.turnCount) continue;
+    const completedTo = require("./source-review.js").completedSourceTurn({ harness }, sourceRecords(harness, session.sessionRef));
+    while (fromTurn <= completedTo) {
+      const toTurn = Math.min(completedTo, fromTurn + MAX_TURNS_PER_SEGMENT - 1);
       segments.push({ ...session, fromTurn, toTurn });
       fromTurn = toTurn + 1;
     }
@@ -1184,9 +1206,19 @@ async function readSecret(input = process.stdin, output = process.stderr) {
 
 function copyRunner() {
   fs.mkdirSync(configRoot(), { recursive: true, mode: 0o700 });
-  const dependency = path.dirname(require.resolve("jsonc-parser/package.json"));
-  const installedDependency = path.join(configRoot(), "node_modules", "jsonc-parser");
-  if (path.resolve(dependency) !== path.resolve(installedDependency)) copyDirectory(dependency, installedDependency);
+  for (const name of ["jsonc-parser", "smol-toml"]) {
+    const dependency = name === "smol-toml" ? path.dirname(path.dirname(require.resolve(name))) : path.dirname(require.resolve(`${name}/package.json`));
+    const installedDependency = path.join(configRoot(), "node_modules", name);
+    if (path.resolve(dependency) !== path.resolve(installedDependency)) copyDirectory(dependency, installedDependency);
+  }
+  for (const name of ["source-review.js", "v5-payload.js", "native-preparation.js"]) {
+    const source = path.join(__dirname, name);
+    const destination = path.join(configRoot(), name);
+    if (path.resolve(source) !== path.resolve(destination)) {
+      fs.copyFileSync(source, destination);
+      fs.chmodSync(destination, 0o600);
+    }
+  }
   const temporary = `${installedRunnerPath()}.${process.pid}.tmp`;
   try {
     fs.copyFileSync(__filename, temporary);
@@ -1975,8 +2007,8 @@ async function updateConfig(harness) {
   if (conflicts.length) console.error(`Existing Shifu shortcut(s) were left unchanged: ${conflicts.join(", ")}.`);
 }
 
-function textIsSafe(value, maximum) {
-  return typeof value === "string" && value.trim().length > 0 && value.length <= maximum && !/[\x00-\x1f\x7f]/.test(value) && !PHONE_NUMBER_PATTERN.test(value) && !UNSAFE_CODE_PATTERN.test(value) && !CREDENTIAL_ASSIGNMENT_PATTERN.test(value) && !CLOUD_CREDENTIAL_PATTERN.test(value) && !SENSITIVE_PATTERNS.some((pattern) => value.toLowerCase().includes(pattern));
+function textIsSafe(value, maximum, unicode = false) {
+  return typeof value === "string" && value.trim().length > 0 && (unicode ? [...value].length : value.length) <= maximum && !/[\x00-\x1f\x7f]/.test(value) && !PHONE_NUMBER_PATTERN.test(value) && !UNSAFE_CODE_PATTERN.test(value) && !CREDENTIAL_ASSIGNMENT_PATTERN.test(value) && !CLOUD_CREDENTIAL_PATTERN.test(value) && !SENSITIVE_PATTERNS.some((pattern) => value.toLowerCase().includes(pattern));
 }
 
 function sessionRefIsSafe(value) {
@@ -1987,29 +2019,30 @@ function sessionRefIsSafe(value) {
 }
 
 function validateSync(input, harness) {
+  const safeText = (value, maximum) => textIsSafe(value, maximum, input?.redactionVersion === 5);
   if (!input || Object.keys(input).some((field) => !SYNC_FIELDS.has(field))) throw new Error("Remove unsupported fields from the reviewed sync payload.");
   if (!input || input.harness !== harness || !sessionRefIsSafe(input.sessionRef)) throw new Error("Add the current opaque sessionRef before syncing.");
   if (Object.hasOwn(input, "supersedesSyncId") && (!Number.isSafeInteger(input.supersedesSyncId) || input.supersedesSyncId < 1)) throw new Error("Use the positive supersedesSyncId returned by force discovery.");
   if (Object.hasOwn(input, "force") && typeof input.force !== "boolean") throw new Error("Use force: true only for explicitly requested force sync.");
   if (input.force && input.supersedesSyncId) throw new Error("Use force: true without supersedesSyncId, or the legacy supersedesSyncId mode alone.");
   if (!Number.isInteger(input.fromTurn) || !Number.isInteger(input.toTurn) || input.fromTurn < 1 || input.toTurn < input.fromTurn || input.toTurn > 1_000_000 || input.toTurn - input.fromTurn >= MAX_TURNS_PER_SEGMENT) throw new Error("Use a valid incremental range of at most 12 turns.");
-  if (input.redactionVersion >= 3 && !textIsSafe(input.title, MAX_TITLE_LENGTH)) throw new Error("Add a concise, redacted title for this reviewed sync.");
-  if (!textIsSafe(input.summary, 1200)) throw new Error("The summary is empty, too long, or contains sensitive content. Redact it before syncing.");
+  if (input.redactionVersion >= 3 && !safeText(input.title, MAX_TITLE_LENGTH)) throw new Error("Add a concise, redacted title for this reviewed sync.");
+  if (!safeText(input.summary, 1200)) throw new Error("The summary is empty, too long, or contains sensitive content. Redact it before syncing.");
   const forced = input.force || Boolean(input.supersedesSyncId);
   if (forced && Array.isArray(input.evidence) && input.evidence.length === 0) throw new Error("Force sync has no supported evidence. Re-review the final notes and prepare meaningful redacted work items; existing work and checkpoints were left untouched.");
-  const minimumEvidence = forced || input.redactionVersion < REDACTION_VERSION ? 1 : 0;
+  const minimumEvidence = forced || input.redactionVersion < 4 ? 1 : 0;
   if (!Array.isArray(input.evidence) || input.evidence.length < minimumEvidence || input.evidence.length > MAX_EVIDENCE_ITEMS || !input.evidence.every((item) => {
-    if (!item || !EVIDENCE_KINDS.has(item.kind) || (input.redactionVersion >= 3 && !textIsSafe(item.title, MAX_TITLE_LENGTH)) || !textIsSafe(item.statement, MAX_WORK_DETAIL_LENGTH)) return false;
-    if (item.kind === "activity" && input.redactionVersion >= REDACTION_VERSION) return item.scope === undefined || item.scope === "";
+    if (!item || !EVIDENCE_KINDS.has(item.kind) || (input.redactionVersion >= 3 && !safeText(item.title, MAX_TITLE_LENGTH)) || !safeText(item.statement, MAX_WORK_DETAIL_LENGTH)) return false;
+    if (item.kind === "activity" && input.redactionVersion >= 4) return item.scope === undefined || item.scope === "";
     return SCOPE_LEVELS.has(item.scope);
   })) {
 	throw new Error(`evidence must contain ${minimumEvidence}-${MAX_EVIDENCE_ITEMS} redacted activity or work items with valid titles, details, kinds, and scopes.`);
   }
   for (const field of ["verification", "decisions"]) {
-    if (!Array.isArray(input[field]) || input[field].length > MAX_SYNC_ITEMS || !input[field].every((item) => textIsSafe(item, 240))) throw new Error(`${field} must contain at most ${MAX_SYNC_ITEMS} redacted statements.`);
+    if (!Array.isArray(input[field]) || input[field].length > MAX_SYNC_ITEMS || !input[field].every((item) => safeText(item, 240))) throw new Error(`${field} must contain at most ${MAX_SYNC_ITEMS} redacted statements.`);
   }
   if (!input.evidence.every((item) =>
-    Object.keys(item).every((field) => EVIDENCE_FIELDS.has(field)) &&
+    Object.keys(item).every((field) => (input.redactionVersion === 5 ? REVIEWED_EVIDENCE_FIELDS : EVIDENCE_FIELDS).has(field)) &&
     (!Object.hasOwn(item, "area") || item.area === "" || WORK_AREAS.has(item.area)) &&
     (!Object.hasOwn(item, "itemRef") || item.itemRef === "" || (typeof item.itemRef === "string" && ITEM_REF_PATTERN.test(item.itemRef))) &&
     (!Object.hasOwn(item, "verificationRefs") || (Array.isArray(item.verificationRefs) &&
@@ -2018,12 +2051,39 @@ function validateSync(input, harness) {
     throw new Error("Each work item must use a supported area, safe itemRef, and distinct indexes into verification.");
   }
   const identities = new Set();
+  if (input.redactionVersion === 5 && !input.evidence.every((item) => validReviewedContext(item, input))) {
+    throw new Error("Each reviewed work item must use safe bounded workContext, distinct decisionRefs, and synthesized source metadata within its turn range.");
+  }
   for (const item of input.evidence) {
     const identity = item.itemRef || `${item.kind}:${item.scope}:${item.statement.trim()}`;
     if (identities.has(identity)) throw new Error("Each work item must have a distinct identity.");
     identities.add(identity);
   }
-  if (input.redactionVersion !== 2 && input.redactionVersion !== 3 && input.redactionVersion !== REDACTION_VERSION) throw new Error(`Use redactionVersion ${REDACTION_VERSION} for new syncs.`);
+  if (![2, 3, 4, 5].includes(input.redactionVersion)) throw new Error(`Use redactionVersion ${REDACTION_VERSION} for new syncs; existing version-two through version-four retries remain supported.`);
+}
+
+function validUTCEventTime(value) {
+  if (value === "") return true;
+  if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?Z$/.test(value)) return false;
+  if (/^0001-01-01T00:00:00(?:\.0{1,9})?Z$/.test(value)) return false;
+  const instant = Date.parse(value);
+  return Number.isFinite(instant) && new Date(instant).toISOString().slice(0, 19) === value.slice(0, 19);
+}
+
+function validReviewedContext(item, input) {
+  if (Object.hasOwn(item, "workContext")) {
+    if (typeof item.workContext !== "string") return false;
+    const context = item.workContext.trim();
+    if (context !== "" && ([...context].length > 300 || !textIsSafe(context, context.length))) return false;
+  }
+  if (Object.hasOwn(item, "decisionRefs") && (!Array.isArray(item.decisionRefs) || item.decisionRefs.length > MAX_SYNC_ITEMS ||
+    new Set(item.decisionRefs).size !== item.decisionRefs.length || !item.decisionRefs.every((ref) => Number.isInteger(ref) && ref >= 0 && ref < input.decisions.length))) return false;
+  if (!Object.hasOwn(item, "sources")) return true;
+  return Array.isArray(item.sources) && item.sources.length <= 8 && new Set(item.sources.map((source) => source?.locator)).size === item.sources.length &&
+    item.sources.every((source) => source && typeof source === "object" && !Array.isArray(source) && Object.keys(source).every((field) => SOURCE_FIELDS.has(field)) &&
+      ["user", "assistant"].includes(source.role) && Number.isInteger(source.turn) && source.turn >= input.fromTurn && source.turn <= input.toTurn &&
+      typeof source.locator === "string" && ITEM_REF_PATTERN.test(source.locator) && source.excerptMode === "synthesized" &&
+      (!Object.hasOwn(source, "eventAt") || validUTCEventTime(source.eventAt)));
 }
 
 function validateAIProfileObservation(input, harness) {
@@ -2288,7 +2348,12 @@ async function sendSync(harness, input, config = readConfig(harness)) {
   const force = input.force || Boolean(input.supersedesSyncId);
   if (!force && input.fromTurn !== expected) throw new Error(`This local checkpoint expects turn ${expected}. Review the pending segment before retrying.`);
   if (force && reviewSegment(harness, input.sessionRef, input.fromTurn, input.toTurn).turnCount < input.toTurn) throw new Error("The original local turns must still exist before force syncing this range.");
-  const profile = force ? undefined : profileForSegment(harness, input);
+  if (!force) input = queuePayload(harness, input);
+  let profile;
+  if (!force) {
+    try { profile = profileForSegment(harness, input); }
+    catch { console.error("AI Profile source is unavailable; the prepared activity payload remains retryable."); }
+  }
   const receipt = await postSync(config, input);
   if (!validSyncReceipt(receipt, input)) {
     throw new Error("Shifu returned a checkpoint that does not match this reviewed segment. Local state was not advanced.");
@@ -2297,8 +2362,9 @@ async function sendSync(harness, input, config = readConfig(harness)) {
     state.profilePending ||= {};
     state.profilePending[`${profile.sessionRef}:${profile.fromTurn}:${profile.toTurn}`] = profile;
   }
-  state.sessions[input.sessionRef] = { turnsSinceSync: 0, lastSyncedAt: Date.now(), lastSyncedTurn: Math.max(checkpoint?.lastSyncedTurn || 0, receipt.lastSyncedTurn), syncDue: false };
+  state.sessions[input.sessionRef] = { ...checkpoint, turnsSinceSync: 0, lastSyncedAt: Date.now(), lastSyncedTurn: Math.max(checkpoint?.lastSyncedTurn || 0, receipt.lastSyncedTurn), syncDue: false };
   saveState(harness, state);
+  if (!force) prunePendingQueue(harness);
   if (profile) await flushPendingProfiles(harness, config);
   console.error(receipt.duplicate ? "Shifu already had this exact segment. Local checkpoint recovered." : force ? `Force synced the reviewed range for turns ${input.fromTurn}–${input.toTurn}.` : `Synced through turn ${receipt.lastSyncedTurn}.`);
   return receipt;
@@ -2351,13 +2417,142 @@ async function sync(harness) {
   if (result?.skipped) throw new Error("Another Shifu sync is already running. This segment was not sent; retry later.");
 }
 
-function rebuildPendingQueue(harness) {
-  const segments = pendingSegments(harness).map((segment) => payloadForSegment(harness, segment));
-  const queue = { updatedAt: new Date().toISOString(), segments };
+function readPendingQueue(harness) {
+  const queue = readJSON(pendingPath(harness), { segments: [] });
+  if (!Array.isArray(queue.segments)) throw new Error("The prepared sync queue is invalid. Existing work was left untouched.");
+  for (const input of queue.segments) {
+    validateSync(input, harness);
+    if (input.force || input.supersedesSyncId) throw new Error("Historical replacements cannot enter the automatic queue.");
+  }
+  return queue;
+}
+
+function prunePendingQueue(harness) {
+  const queue = readPendingQueue(harness);
+  const state = readState(harness);
+  queue.segments = queue.segments.filter((input) => input.toTurn > (state.sessions[input.sessionRef]?.lastSyncedTurn || 0));
+  queue.updatedAt = new Date().toISOString();
+  writeJSON(pendingPath(harness), queue);
+  return queue;
+}
+
+function queuePayload(harness, input) {
+  const queue = prunePendingQueue(harness);
+  const overlap = queue.segments.find((queued) => queued.sessionRef === input.sessionRef && queued.fromTurn <= input.toTurn && queued.toTurn >= input.fromTurn);
+  if (overlap) {
+    if (JSON.stringify(overlap) !== JSON.stringify(input)) throw new Error("This range already has a different prepared payload. Retry the queued payload before preparing another version.");
+    return overlap;
+  }
+  queue.segments.push(input);
+  writeJSON(pendingPath(harness), queue);
+  return input;
+}
+
+async function reconcilePreparedSession(harness, sessionRef, config) {
+  const state = readState(harness);
+  const checkpoint = state.sessions[sessionRef] || {};
+  if (checkpoint.preparationVersion === REDACTION_VERSION) return;
+  if (readPendingQueue(harness).segments.some((input) => input.sessionRef === sessionRef)) return;
+  // Older runners did not persist automatic payloads before HTTP. Recover an
+  // accepted prefix before switching an uncertain legacy range to v5.
+  const accepted = await acceptedSegments(config, sessionRef);
+  let lastSyncedTurn = checkpoint.lastSyncedTurn || 0;
+  for (const segment of accepted) {
+    if (segment.toTurn <= lastSyncedTurn) continue;
+    if (segment.fromTurn !== lastSyncedTurn + 1) throw new Error("Accepted sync history has a gap. This session remains pending.");
+    lastSyncedTurn = segment.toTurn;
+  }
+  state.sessions[sessionRef] = { ...checkpoint, lastSyncedTurn, preparationVersion: REDACTION_VERSION };
+  saveState(harness, state);
+}
+
+async function prepareV5Payload(harness, segment, prepare = require("./native-preparation.js").prepareNative) {
+  const packet = reviewSourcePacket(harness, segment.sessionRef, segment.fromTurn, segment.toTurn, true);
+  const output = await prepare(harness, packet);
+  const payload = require("./v5-payload.js").preparedPayload(packet, output);
+  validateSync(payload, harness);
+  return payload;
+}
+
+function preparationState(harness, queue) {
+  const state = readState(harness);
+  const virtual = { ...state, sessions: structuredClone(state.sessions) };
+  const blocked = new Set();
+  for (const input of [...queue.segments].sort((a, b) => a.fromTurn - b.fromTurn)) {
+    const prior = virtual.sessions[input.sessionRef] || {};
+    if (input.fromTurn !== (prior.lastSyncedTurn || 0) + 1) blocked.add(input.sessionRef);
+    else virtual.sessions[input.sessionRef] = { ...prior, lastSyncedTurn: input.toTurn };
+  }
+  return { virtual, blocked };
+}
+
+async function preparePendingQueue(harness, config, blockedSessions = new Set()) {
+  let queue = prunePendingQueue(harness);
+  const previousFailures = new Set((queue.preparationErrors || []).map((error) => error.sessionRef));
+  const errors = [];
+  const attempted = new Set();
+  const deadline = Date.now() + PREPARATION_RUN_MS;
+  let attempts = 0;
+  while (attempts < MAX_PREPARATIONS_PER_RUN && Date.now() < deadline) {
+    const { virtual, blocked } = preparationState(harness, queue);
+    const candidates = pendingSegments(harness, virtual).filter((item) => !blocked.has(item.sessionRef) && !blockedSessions.has(item.sessionRef));
+    const segment = candidates.find((item) => !previousFailures.has(item.sessionRef)) || candidates[0];
+    if (!segment) break;
+    try {
+      const original = readState(harness).sessions[segment.sessionRef];
+      if (original?.preparationVersion !== REDACTION_VERSION && queue.segments.some((input) => input.sessionRef === segment.sessionRef)) {
+        blockedSessions.add(segment.sessionRef);
+        continue;
+      }
+      if (!attempted.has(segment.sessionRef)) {
+        attempted.add(segment.sessionRef);
+        await reconcilePreparedSession(harness, segment.sessionRef, config);
+        // Remote acknowledgement can change the next unsynced range.
+        continue;
+      }
+      let payload;
+      attempts += 1;
+      for (let toTurn = segment.toTurn; toTurn >= segment.fromTurn; toTurn -= 1) {
+        try { payload = await prepareV5Payload(harness, { ...segment, toTurn }); break; }
+        catch (error) {
+          if (toTurn > segment.fromTurn && /exceeds 128 messages or 256 KiB/.test(error.message)) continue;
+          throw error;
+        }
+      }
+      queuePayload(harness, payload);
+      queue = readPendingQueue(harness);
+    } catch (error) {
+      blockedSessions.add(segment.sessionRef);
+      errors.push({ sessionRef: segment.sessionRef, fromTurn: segment.fromTurn, toTurn: segment.toTurn, reason: error.message });
+      console.error(`${HARNESS_NAMES[harness]} preparation remains pending: ${error.message}`);
+    }
+  }
+  queue = prunePendingQueue(harness);
+  queue.preparationErrors = errors;
   writeJSON(pendingPath(harness), queue);
   return queue;
 }
 
+async function drainPendingQueue(harness, config, blockedSessions) {
+  let synced = 0;
+  let failures = 0;
+  for (const input of eligibleQueuedSegments(readPendingQueue(harness), config)) {
+    if (blockedSessions.has(input.sessionRef)) continue;
+    try { await sendSync(harness, input, config); synced += 1; }
+    catch (error) {
+      failures += 1;
+      blockedSessions.add(input.sessionRef);
+      console.error(`${HARNESS_NAMES[harness]} prepared segment remains pending: ${error.message}`);
+    }
+  }
+  return { synced, failures };
+}
+
+function eligibleQueuedSegments(queue, config) {
+  const excluded = new Set(normalizedPolicy(config).excludedSessionRefs);
+  return queue.segments.filter((input) => !excluded.has(input.sessionRef));
+}
+
 function approvalNotificationInvocation(harness, count, sessionRef) {
   const prompt = `Shifu sync approval: ${count} redacted segment${count === 1 ? " is" : "s are"} waiting locally. Tell the user to ask you to show the pending Shifu sync and approve it once. Do not inspect files, run tools, or sync anything in this session.`;
   if (harness === "codex") {
@@ -2388,8 +2583,10 @@ function notificationSessionRef(output) {
 
 function notifyManualApproval(harness, count, existingSessionRef) {
   const invocation = approvalNotificationInvocation(harness, count, existingSessionRef);
+  const { resolveNativeBinary, nativeEnvironment } = require("./native-preparation.js");
   const before = existingSessionRef ? new Set() : new Set(localSessions(harness).map((session) => session.sessionRef));
-  const output = execFileSync(invocation.command, invocation.args, {
+  const output = execFileSync(resolveNativeBinary(harness), invocation.args, {
+    env: nativeEnvironment(),
     cwd: os.homedir(),
     encoding: "utf8",
     stdio: ["ignore", "pipe", "pipe"],
@@ -2404,29 +2601,32 @@ function notifyManualApproval(harness, count, existingSessionRef) {
   return created;
 }
 
-function showPending(harness) {
-  readConfig(harness);
-  const queue = rebuildPendingQueue(harness);
-  console.log(JSON.stringify(queue, null, 2));
+async function showPending(harness) {
+  const config = readConfig(harness);
+  const result = await withSyncLock(harness, async () => {
+    console.log(JSON.stringify(await preparePendingQueue(harness, config), null, 2));
+  });
+  if (result?.skipped) throw new Error("Another Shifu sync is already preparing this queue. Retry later.");
 }
 
 async function approvePending(harness) {
   const config = readConfig(harness);
-  const queue = rebuildPendingQueue(harness);
-  if (queue.segments.length === 0) {
-    console.error("There is no pending Shifu activity to approve.");
-    return;
-  }
-  console.error(`\nThese ${queue.segments.length} redacted segment(s) will be sent to Shifu:\n${JSON.stringify(queue.segments, null, 2)}`);
-  if (!hasFlag("--approved") && !(await confirm("Approve all pending segments"))) {
-    console.error("Nothing was sent. The pending list will keep accumulating.");
-    return;
-  }
   const result = await withSyncLock(harness, async () => {
-    for (const input of queue.segments) await sendSync(harness, input, config);
+    const queue = prunePendingQueue(harness);
+    queue.segments = eligibleQueuedSegments(queue, config);
+    if (queue.segments.length === 0) {
+      console.error("There is no prepared Shifu activity to approve. Run pending to prepare the next ranges.");
+      return;
+    }
+    console.error(`\nThese ${queue.segments.length} redacted segment(s) will be sent to Shifu:\n${JSON.stringify(queue.segments, null, 2)}`);
+    if (!hasFlag("--approved") && !(await confirm("Approve all pending segments"))) {
+      console.error("Nothing was sent. The prepared queue is unchanged.");
+      return;
+    }
+    const sent = await drainPendingQueue(harness, config, new Set());
+    if (sent.failures) throw new Error("Some approved segments could not be acknowledged. Their exact payloads remain pending for retry.");
   });
   if (result?.skipped) throw new Error("Another Shifu sync is already running. Nothing was sent; retry later.");
-  rebuildPendingQueue(harness);
 }
 
 async function scheduledSync(harness) {
@@ -2457,14 +2657,15 @@ async function scheduledSync(harness) {
     }
     await flushPendingProfiles(harness, config);
     if (policy.approvalMode === "manual") {
-      const queue = rebuildPendingQueue(harness);
-      const pendingFingerprint = createHash("sha256").update(queue.segments.map((segment) => `${segment.sessionRef}:${segment.fromTurn}:${segment.toTurn}`).join("\n")).digest("hex");
+      const queue = await preparePendingQueue(harness, config);
+      const eligible = eligibleQueuedSegments(queue, config);
+      const pendingFingerprint = createHash("sha256").update(eligible.map((segment) => `${segment.sessionRef}:${segment.fromTurn}:${segment.toTurn}`).join("\n")).digest("hex");
       const changed = state.schedule?.pendingFingerprint !== pendingFingerprint;
       let approvalSessionRef = state.schedule?.approvalSessionRef;
       let notificationError;
-      if (queue.segments.length > 0 && changed) {
+      if (eligible.length > 0 && changed) {
         try {
-          approvalSessionRef = notifyManualApproval(harness, queue.segments.length, approvalSessionRef);
+          approvalSessionRef = notifyManualApproval(harness, eligible.length, approvalSessionRef);
           if (approvalSessionRef && !config.syncPolicy.excludedSessionRefs.includes(approvalSessionRef)) {
             config.syncPolicy.excludedSessionRefs.push(approvalSessionRef);
             writeJSON(configPath(harness), config);
@@ -2474,38 +2675,37 @@ async function scheduledSync(harness) {
         }
       }
       const pendingProfiles = Object.keys(readState(harness).profilePending || {}).length;
+      const unprepared = pendingSegments(harness, preparationState(harness, queue).virtual).length;
+      const preparationFailures = queue.preparationErrors.length;
       state.schedule = {
         ...state.schedule,
         lastAttemptAt: startedAt,
-        lastResult: notificationError ? "awaiting_approval_notification_failed" : failedFiles || pendingProfiles ? "partial_failure" : "awaiting_approval",
+        lastResult: notificationError ? "awaiting_approval_notification_failed" : failedFiles || pendingProfiles || preparationFailures ? "partial_failure" : "awaiting_approval",
         submittedFiles,
         pendingFiles,
         failedFiles,
         unsupportedFiles,
         pendingProfiles,
-        pendingSegments: queue.segments.length,
+        pendingSegments: eligible.length,
+        unpreparedSegments: unprepared,
+        preparationFailures,
         pendingFingerprint,
         approvalSessionRef,
         notificationError,
-        nextRunAt: startedAt + (notificationError || failedFiles || pendingProfiles ? SCHEDULER_TICK_MINUTES / 60 : policy.intervalHours) * 3_600_000,
+        nextRunAt: Date.now() + (notificationError || failedFiles || pendingProfiles || preparationFailures || unprepared ? SCHEDULER_TICK_MINUTES / 60 : policy.intervalHours) * 3_600_000,
       };
     } else {
       const blockedSessions = new Set();
-      for (const segment of pendingSegments(harness)) {
-        if (blockedSessions.has(segment.sessionRef)) continue;
-        try {
-          await sendSync(harness, payloadForSegment(harness, segment), config);
-          synced += 1;
-        } catch (error) {
-          failures += 1;
-          blockedSessions.add(segment.sessionRef);
-          console.error(`${HARNESS_NAMES[harness]} session ${segment.sessionRef} remains pending: ${error.message}`);
-        }
-      }
+      const previous = await drainPendingQueue(harness, config, blockedSessions);
+      const queue = await preparePendingQueue(harness, config, blockedSessions);
+      const current = await drainPendingQueue(harness, config, blockedSessions);
+      synced = previous.synced + current.synced;
+      failures = previous.failures + current.failures + queue.preparationErrors.length;
+      const remaining = pendingSegments(harness).length || eligibleQueuedSegments(readPendingQueue(harness), config).length;
       const pendingProfiles = Object.keys(readState(harness).profilePending || {}).length;
       state.schedule = {
         lastAttemptAt: startedAt,
-        lastResult: failures || failedFiles || pendingProfiles ? "partial_failure" : "complete",
+        lastResult: failures || failedFiles || pendingProfiles ? "partial_failure" : remaining ? "pending_preparation" : "complete",
         syncedSegments: synced,
         failedSessions: failures,
         submittedFiles,
@@ -2513,7 +2713,7 @@ async function scheduledSync(harness) {
         failedFiles,
         unsupportedFiles,
         pendingProfiles,
-        nextRunAt: startedAt + (failures || failedFiles || pendingProfiles ? SCHEDULER_TICK_MINUTES / 60 : policy.intervalHours) * 3_600_000,
+        nextRunAt: Date.now() + (failures || failedFiles || pendingProfiles || remaining ? SCHEDULER_TICK_MINUTES / 60 : policy.intervalHours) * 3_600_000,
       };
     }
     const latest = readState(harness);
@@ -2760,7 +2960,8 @@ async function main() {
   if (command === "status") return status(harness);
   if (command === "sessions" || command === "discover" || (command === "session" && (subCommand === "discover" || subCommand === "list" || !subCommand || subCommand.startsWith("-")))) return sessions(harness);
   if (command === "review") return review(harness);
-  throw new Error("Use install, connect, mcp-setup, mcp-headers, disconnect, config update, status, sessions, review, sync, sync-files, pending, approve, or scheduled-sync.");
+  if (command === "review-source") return console.log(JSON.stringify(reviewSourcePacket(harness, option("--session-ref"), numberOption("--from-turn"), numberOption("--to-turn")), null, 2));
+  throw new Error("Use install, connect, mcp-setup, mcp-headers, disconnect, config update, status, sessions, review, review-source, sync, sync-files, pending, approve, or scheduled-sync.");
 }
 
 if (require.main === module) {
@@ -2772,4 +2973,4 @@ if (require.main === module) {
     });
 }
 
-module.exports = { setupMcp, olderPackageVersion, updateRecommendedPackage, installPackageVersion, approvalNotificationInvocation, claudeReview, claudeSession, codexReview, codexSession, configPath, configuredSyncPolicy, connect, defaultInputFolder, disconnect, finalAssistantNote, install, installDestination, localSessions, normalizedPolicy, notificationSessionRef, opencodeQuery, opencodeReview, opencodeSessions, payloadForSegment, pendingSegments, prepareInputFolder, profileForSegment, readSecret, removeHooks, requireSafeApiUrl, reviewCandidates, schedulerArtifact, statePath, syncWorkFiles, textIsSafe, unsyncedCodexSessions, validateAIProfileObservation, validateConnection, validateInputFolder, validSyncReceipt, validateApprovalMode, validateIntervalHours, validateSync, workFiles };
+module.exports = { setupMcp, olderPackageVersion, updateRecommendedPackage, installPackageVersion, approvalNotificationInvocation, claudeReview, claudeSession, codexReview, codexSession, configPath, configuredSyncPolicy, connect, defaultInputFolder, disconnect, finalAssistantNote, install, installDestination, localSessions, normalizedPolicy, notificationSessionRef, opencodeQuery, opencodeReview, opencodeSessions, payloadForSegment, pendingSegments, prepareV5Payload, prepareInputFolder, profileForSegment, readSecret, removeHooks, requireSafeApiUrl, reviewCandidates, reviewSourcePacket, schedulerArtifact, statePath, syncWorkFiles, textIsSafe, unsyncedCodexSessions, validateAIProfileObservation, validateConnection, validateInputFolder, validSyncReceipt, validateApprovalMode, validateIntervalHours, validateSync, workFiles };