@useshifu/coding-harness 0.3.28 → 0.3.30

package/bin/shifu-harness.js

@@ -46,7 +46,7 @@ const MAX_TURNS_PER_SEGMENT = 12;
 const MAX_TITLE_LENGTH = 160;
 const MAX_WORK_DETAIL_LENGTH = 600;
 const CONFIG_VERSION = 2;
-const RUNNER_VERSION = "0.3.28";
+const RUNNER_VERSION = "0.3.30";
 const REDACTION_VERSION = 5;
 const LEGACY_REDACTION_VERSION = 4;
 const MAX_PREPARATIONS_PER_RUN = 4;
@@ -109,13 +109,9 @@ function claudeSessionsRoot() {
 function opencodeQuery(query) {
   if (process.env.SHIFU_OPENCODE_DB && !process.env.SHIFU_OPENCODE_BIN) {
     try {
-      const sqliteOutput = execFileSync("sqlite3", ["-json", process.env.SHIFU_OPENCODE_DB, query], {
-        encoding: "utf8",
-        stdio: ["ignore", "pipe", "pipe"],
-        maxBuffer: 64 * 1024 * 1024,
-      });
-      return JSON.parse(sqliteOutput);
-    } catch {
+      return openCodeDatabaseQuery(process.env.SHIFU_OPENCODE_DB, query);
+    } catch (error) {
+      if (error.code === "SHIFU_SQLITE_UNAVAILABLE") throw error;
       throw new Error("Could not read OpenCode session metadata. Ensure the opencode command is installed and its local database is available.");
     }
   }
@@ -137,16 +133,13 @@ function opencodeQuery(query) {
       try { fs.closeSync(fd); } catch {}
       fd = undefined;
     }
-    const dbPath = process.env.SHIFU_OPENCODE_DB || path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), ".local", "share"), "opencode", "opencode.db");
+    const dbPath = process.env.SHIFU_OPENCODE_DB || openCodeDatabasePath();
     if (fs.existsSync(dbPath)) {
       try {
-        const sqliteOutput = execFileSync("sqlite3", ["-json", dbPath, query], {
-          encoding: "utf8",
-          stdio: ["ignore", "pipe", "pipe"],
-          maxBuffer: 64 * 1024 * 1024,
-        });
-        return JSON.parse(sqliteOutput);
-      } catch {}
+        return openCodeDatabaseQuery(dbPath, query);
+      } catch (error) {
+        if (error.code === "SHIFU_SQLITE_UNAVAILABLE") throw error;
+      }
     }
     throw new Error("Could not read OpenCode session metadata. Ensure the opencode command is installed and its local database is available.");
   } finally {
@@ -157,6 +150,56 @@ function opencodeQuery(query) {
   }
 }
 
+function openCodeDatabasePath() {
+  try {
+    const { resolveNativeBinary, nativeEnvironment } = require("./native-preparation.js");
+    const location = execFileSync(resolveNativeBinary("opencode"), ["debug", "paths", "db"], {
+      env: nativeEnvironment(), encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 10_000, maxBuffer: 1024 * 1024,
+    }).trim();
+    if (path.isAbsolute(location) && !/[\r\n\0]/.test(location)) return location;
+  } catch {}
+  return path.join(process.env.XDG_DATA_HOME || path.join(os.homedir(), ".local", "share"), "opencode", "opencode.db");
+}
+
+function openCodeDatabaseQuery(dbPath, query) {
+  function read(sql) {
+    let output;
+    try {
+      output = execFileSync("sqlite3", ["-readonly", "-json", "-cmd", ".timeout 5000", dbPath, sql], {
+        encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], timeout: 10_000, maxBuffer: 64 * 1024 * 1024,
+      });
+    } catch (error) {
+      if (error.code !== "ENOENT") throw error;
+      const unavailable = new Error("Reading the local OpenCode database requires sqlite3. Install sqlite3 and retry; no work was synced.");
+      unavailable.code = "SHIFU_SQLITE_UNAVAILABLE";
+      throw unavailable;
+    }
+    return output.trim() ? JSON.parse(output) : [];
+  }
+  const tables = read("SELECT name FROM sqlite_master WHERE type = 'table' AND name IN ('session_v2', 'session_message')");
+  if (tables.length !== 2) return read(query);
+  // V2 stores typed messages, ordered by sequence, instead of separate parts.
+  // Read-only CTEs retain the existing adapter contract without changing its database.
+  const hasDirectory = read("PRAGMA table_info(session_v2)").some((column) => column.name === "directory");
+  const projection = `WITH
+    session AS (SELECT id, time_created, time_updated, ${hasDirectory ? "directory" : "NULL AS directory"} FROM session_v2),
+    message AS (
+      SELECT id, session_id, seq, time_created, json_object(
+        'role', type, 'finish', CASE WHEN json_extract(data, '$.time.completed') IS NOT NULL THEN json_extract(data, '$.finish') END,
+        'error', json_extract(data, '$.error'), 'modelID', json_extract(data, '$.model.id'), 'storageVersion', 2
+      ) AS data FROM session_message WHERE type IN ('user', 'assistant', 'idle')
+    ),
+    part AS (
+      SELECT id || '_user' AS id, id AS message_id, json_object('type', 'text', 'text', json_extract(data, '$.text')) AS data
+      FROM session_message WHERE type = 'user'
+      UNION ALL
+      SELECT m.id || '_' || printf('%08d', p.key), m.id, p.value
+      FROM session_message m, json_each(m.data, '$.content') p WHERE m.type = 'assistant'
+    ) `;
+  return read(projection + query.replaceAll("ORDER BY s.time_updated, m.time_created", "ORDER BY s.time_updated, m.seq")
+    .replaceAll("ORDER BY m.time_created, p.id", "ORDER BY m.seq, p.id"));
+}
+
 function opencodeSessions(rows) {
   const sessions = new Map();
   for (const row of rows) {
@@ -202,6 +245,7 @@ function validateSyncOptions(command, start = 3) {
     "sync-files": common,
     "scheduled-sync": common,
     pending: common,
+    "retry-preparation": [...common, "--session-ref", "--from-turn"],
     approve: common,
     install: common,
     connect: [...common, ...setup, ...syncSettings, "--api-url"],
@@ -254,6 +298,7 @@ function writeJSON(file, value) {
 }
 
 function readConfig(harness) {
+  if (fs.existsSync(path.join(configRoot(), `${harness}-connection-transition.json`))) throw new Error("An interrupted connection replacement is held locally. Run connect again with the selected connection before syncing.");
   const config = readJSON(configPath(harness), undefined);
   if (!config || typeof config.token !== "string" || typeof config.apiUrl !== "string") {
     throw new Error(`No ${HARNESS_NAMES[harness]} connection is configured. Run connect first.`);
@@ -589,7 +634,7 @@ function reviewSourcePacket(harness, sessionRef, fromTurn, toTurn, scheduled = f
 
 function sourceRecords(harness, sessionRef) {
   if (harness === "opencode") {
-    return opencodeQuery(`SELECT m.id AS messageId, m.time_created AS createdAt, json_extract(m.data, '$.role') AS role, json_extract(m.data, '$.finish') AS finish, json_extract(m.data, '$.error') AS error, p.data AS part FROM message m LEFT JOIN part p ON p.message_id = m.id WHERE m.session_id = ${JSON.stringify(sessionRef)} ORDER BY m.time_created, p.id`);
+    return opencodeQuery(`SELECT m.id AS messageId, m.time_created AS createdAt, json_extract(m.data, '$.role') AS role, json_extract(m.data, '$.finish') AS finish, json_extract(m.data, '$.error') AS error, json_extract(m.data, '$.storageVersion') AS storageVersion, p.data AS part FROM message m LEFT JOIN part p ON p.message_id = m.id WHERE m.session_id = ${JSON.stringify(sessionRef)} ORDER BY m.time_created, p.id`);
   }
   const file = sessionFile(harness, sessionRef);
   try { return fs.readFileSync(file, "utf8").split("\n").filter(Boolean).map((line) => JSON.parse(line)); }
@@ -1231,7 +1276,7 @@ function copyRunner() {
     const installedDependency = path.join(configRoot(), "node_modules", name);
     if (!sameFile(dependency, installedDependency)) copyDirectory(dependency, installedDependency);
   }
-  for (const name of ["source-review.js", "v5-payload.js", "native-preparation.js", "context-preferences.js", "setup-options.js", "initial-history.js", "onboarding-ui.js"]) {
+  for (const name of ["source-review.js", "v5-payload.js", "native-preparation.js", "opencode-v2-account.js", "context-preferences.js", "setup-options.js", "initial-history.js", "onboarding-ui.js", "continuity-preparation.js", "continuity-state.js"]) {
     const source = path.join(__dirname, name);
     const destination = path.join(configRoot(), name);
     if (!sameFile(source, destination)) {
@@ -1515,7 +1560,7 @@ async function connect(harness, readKey = readSecret, choices) {
   const token = (await readKey()).trim();
   if (!TOKEN_PATTERN.test(token)) throw new Error("The connection key is invalid.");
   const apiUrl = requireSafeApiUrl(option("--api-url") || existing?.apiUrl || process.env.SHIFU_API_URL || DEFAULT_API_URL);
-  await validateConnection(apiUrl, token, harness);
+  const connectionInfo = await validateConnection(apiUrl, token, harness);
   const savedPolicy = existing && normalizedPolicy(existing);
   const currentPolicy = savedPolicy?.onboardingComplete ? savedPolicy : { intervalHours: DEFAULT_INTERVAL_HOURS, approvalMode: "automatic", excludedSessionRefs: savedPolicy?.excludedSessionRefs || [] };
   const selected = await chooseSyncPolicy(currentPolicy);
@@ -1528,15 +1573,17 @@ async function connect(harness, readKey = readSecret, choices) {
   install(harness, true, choices.contextPreferences);
   const scheduleFile = scheduleEnabled ? installSchedule(harness) : undefined;
   if (!scheduleEnabled && savedPolicy?.scheduleEnabled) removeSchedule(harness);
-  writeJSON(configPath(harness), { ...existing, ...choices, version: CONFIG_VERSION, apiUrl, token, harness, syncPolicy, inputFolder });
-  refreshManagedContextInstructions(harness, choices.contextPreferences);
-  if (refreshContextKey) writePrivateText(openCodeTokenPath(), token);
-  if (existing?.token && existing.token !== token) {
-    for (const file of [statePath(harness), pendingPath(harness)]) {
-      try { fs.unlinkSync(file); } catch (error) { if (error?.code !== "ENOENT") throw error; }
-    }
-  }
-  scheduleNextRun(harness, syncPolicy.intervalHours);
+  const connection = { ...existing, ...choices, version: CONFIG_VERSION, apiUrl, token, harness, syncPolicy, inputFolder, capabilities: connectionInfo.capabilities || {} };
+  const committed = await withSyncLock(harness, async () => {
+    if (JSON.stringify(readJSON(configPath(harness), undefined)) !== JSON.stringify(existing)) throw new Error("The connection changed during setup. Run connect again.");
+    require("./continuity-state.js").commitConnection(configRoot(), harness, existing, connection);
+    refreshManagedContextInstructions(harness, choices.contextPreferences);
+    if (refreshContextKey) writePrivateText(openCodeTokenPath(), token);
+    const continuity = continuityState(harness, connection);
+    continuity.load(); continuity.dictionary();
+    scheduleNextRun(harness, syncPolicy.intervalHours);
+  });
+  if (committed?.skipped) throw new Error("Another Shifu sync is running. Retry connect after it finishes; the connection was not replaced.");
   console.error(`${HARNESS_NAMES[harness]} is connected. ${scheduleEnabled ? `It will sync every ${syncPolicy.intervalHours} hours with ${syncPolicy.approvalMode} approval.` : "The sync schedule is off."}`);
   console.error(`Shifu sync folder: ${inputFolder}. Place .txt, .md, .rtf, or .docx files there. Only grounded contributions will become claims.`);
   if (conflicts.length) console.error(`Existing Shifu shortcut(s) were left unchanged: ${conflicts.join(", ")}.`);
@@ -2175,10 +2222,11 @@ function retryDelay(response, attempt) {
   return Math.min(base * (2 ** attempt), 30_000);
 }
 
-async function postSync(config, input) {
+async function postSync(config, input, beforeAttempt, onPermanentFailure) {
   const body = JSON.stringify(input.force ? { ...input, forceRequestId: randomUUID() } : input);
   let lastError;
   for (let attempt = 0; attempt < MAX_RETRIES; attempt += 1) {
+    beforeAttempt?.();
     let response;
     try {
       response = await fetch(`${config.apiUrl}/v1/connectors/coding-sessions/syncs`, {
@@ -2195,7 +2243,7 @@ async function postSync(config, input) {
     const payload = await response.json().catch(() => undefined);
     if (response.ok) return payload?.data;
     lastError = new Error(payload?.error?.message || `Shifu rejected the sync (${response.status}).`);
-    if (response.status !== 429 && response.status < 500) throw lastError;
+    if (response.status !== 429 && response.status < 500) { onPermanentFailure?.(); throw lastError; }
     if (attempt + 1 < MAX_RETRIES) await sleep(retryDelay(response, attempt));
   }
   throw lastError;
@@ -2404,20 +2452,27 @@ async function sendSync(harness, input, config = readConfig(harness)) {
   validateSync(input, harness);
   const state = readState(harness);
   const checkpoint = state.sessions[input.sessionRef];
+  const store = continuityState(harness, config);
+  const frozen = store.find(input);
   const expected = (checkpoint?.lastSyncedTurn || 0) + 1;
   const force = input.force || Boolean(input.supersedesSyncId);
-  if (!force && input.fromTurn !== expected) throw new Error(`This local checkpoint expects turn ${expected}. Review the pending segment before retrying.`);
+  if (!force && !frozen && input.fromTurn !== expected) throw new Error(`This local checkpoint expects turn ${expected}. Review the pending segment before retrying.`);
   if (force && reviewSegment(harness, input.sessionRef, input.fromTurn, input.toTurn).turnCount < input.toTurn) throw new Error("The original local turns must still exist before force syncing this range.");
-  if (!force) input = queuePayload(harness, input);
+  if (!force && (!frozen || input.toTurn > (checkpoint?.lastSyncedTurn || 0))) input = queuePayload(harness, input);
   let profile;
   if (!force) {
     try { profile = profileForSegment(harness, input); }
     catch { console.error("AI Profile source is unavailable; the prepared activity payload remains retryable."); }
   }
-  const receipt = await postSync(config, input);
-  if (!validSyncReceipt(receipt, input)) {
-    throw new Error("Shifu returned a checkpoint that does not match this reviewed segment. Local state was not advanced.");
+  let receipt;
+  try {
+    receipt = frozen?.receipt || await postSync(config, input, frozen ? () => store.reserveBase(input) : undefined, frozen ? () => store.update(frozen.key, (entry) => { entry.baseStatus = "needs_review"; entry.reason = "base_rejected"; }) : undefined);
+    if (!validSyncReceipt(receipt, input)) throw new Error("Shifu returned a checkpoint that does not match this reviewed segment. Local state was not advanced.");
+  } catch (error) {
+    if (frozen && store.find(input).baseAttempts >= 10) store.update(frozen.key, (entry) => { entry.baseStatus = "needs_review"; entry.reason = "base_transport_budget"; });
+    throw error;
   }
+  if (frozen) store.bind(input, receipt);
   if (profile) {
     state.profilePending ||= {};
     state.profilePending[`${profile.sessionRef}:${profile.fromTurn}:${profile.toTurn}`] = profile;
@@ -2526,27 +2581,107 @@ async function reconcilePreparedSession(harness, sessionRef, config) {
   saveState(harness, state);
 }
 
+function continuityState(harness, config = readConfig(harness)) {
+  return require("./continuity-state.js").continuityStore(configRoot(), harness, config);
+}
+
+function preparationPacket(harness, segment, config) {
+  if (normalizedPolicy(config).excludedSessionRefs.includes(segment.sessionRef)) throw new Error("This session is excluded from preparation.");
+  if (!initialHistoryEligibility(segment, segment.fromTurn, config).eligible) throw new Error("This range is outside the saved initial-history selection.");
+  const records = sourceRecords(harness, segment.sessionRef);
+  let minimumTurn = Math.max(1, segment.fromTurn - 6);
+  for (let turn = segment.fromTurn - 1; turn >= minimumTurn; turn -= 1) {
+    if (!initialHistoryEligibility(segment, turn, config).eligible) { minimumTurn = turn + 1; break; }
+  }
+  const packet = require("./source-review.js").continuitySourcePacket({ harness, sessionRef: segment.sessionRef, fromTurn: segment.fromTurn, toTurn: segment.toTurn }, records, minimumTurn);
+  let workspace;
+  if (harness === "codex") workspace = records.find((record) => record.type === "session_meta")?.payload?.cwd;
+  else if (harness === "claude_code") workspace = records.find((record) => typeof record.cwd === "string")?.cwd;
+  else {
+    try { workspace = opencodeQuery(`SELECT directory FROM session WHERE id = ${JSON.stringify(segment.sessionRef)}`)[0]?.directory; }
+    catch { /* No trusted directory means a session-specific scope, never the scheduler workspace. */ }
+  }
+  return { packet, workspace: createHash("sha256").update(typeof workspace === "string" && workspace ? `workspace:${workspace}` : `${harness}:session:${segment.sessionRef}`).digest("hex") };
+}
+
 async function prepareV5Payload(harness, segment, prepare = require("./native-preparation.js").prepareNative) {
-  const packet = reviewSourcePacket(harness, segment.sessionRef, segment.fromTurn, segment.toTurn, true);
+  const config = readConfig(harness);
+  const store = continuityState(harness, config);
+  const prior = store.load().attempts[`${segment.sessionRef}:${segment.fromTurn}`];
+  if (prior?.payloadKey) {
+    const frozen = store.load().outbox.find((entry) => entry.key === prior.payloadKey);
+    if (!frozen) throw new Error("Frozen preparation is unavailable; continuity needs review.");
+    return frozen.payload;
+  }
+  if (prior?.toTurn) segment = { ...segment, toTurn: prior.toTurn };
+  const { packet, workspace } = preparationPacket(harness, segment, config);
+  const dictionary = store.dictionary();
+  dictionary.scopes[workspace] ||= { scopeRef: `s_${randomUUID().replaceAll("-", "")}`, entities: {} };
+  store.saveDictionary(dictionary);
+  const relevantBindings = () => {
+    const content = [...packet.sources, ...packet.referenceSources].map((source) => source.content).join("\n").toLocaleLowerCase();
+    return Object.values(dictionary.scopes[workspace].entities).filter((entry) => entry.createdBy !== `${packet.sessionRef}:${packet.fromTurn}` && content.includes(entry.mention.toLocaleLowerCase()))
+      .map(({ key, mention, kind }) => ({ bindingRef: key, mention, kind }));
+  };
+  packet.entityBindings = relevantBindings();
+  while (packet.referenceSources.length && Buffer.byteLength(JSON.stringify(packet)) + 1 > 256 * 1024) {
+    const oldest = packet.referenceSources[0].turn;
+    packet.referenceSources = packet.referenceSources.filter((source) => source.turn > oldest);
+    packet.referenceCoverage.limited = true;
+    packet.referenceCoverage.droppedTurns += 1;
+    packet.referenceCoverage.fromTurn = packet.referenceSources[0]?.turn || packet.fromTurn;
+    packet.entityBindings = relevantBindings();
+  }
+  if (Buffer.byteLength(JSON.stringify(packet)) + 1 > 256 * 1024) throw new Error("The selected source packet exceeds 128 messages or 256 KiB. Nothing was truncated or synced.");
+  if (packet.entityBindings.length > 32) throw new Error("Relevant private entity bindings exceed the bounded packet; this range needs review.");
+  const bindings = { workspace, scopeRef: dictionary.scopes[workspace].scopeRef, catalog: require("./continuity-preparation.js").CATALOG };
   const deadline = Date.now() + 180_000;
   let validationFeedback;
   for (let attempt = 0; attempt < 2; attempt += 1) {
     const remainingMs = deadline - Date.now();
     if (remainingMs <= 0) throw new Error("Native preparation timed out; the source remains pending.");
-    const output = await prepare(harness, packet, { validationFeedback, timeoutMs: remainingMs });
+    const reservation = store.reservation(packet, bindings);
+    let output;
     try {
-      const payload = require("./v5-payload.js").preparedPayload(packet, output);
-      validateSync(payload, harness);
-      return payload;
+      output = await prepare(harness, packet, { validationFeedback, timeoutMs: remainingMs, continuity: true });
+      const candidateDictionary = structuredClone(dictionary);
+      const result = require("./continuity-preparation.js").bindContinuity(packet, output, candidateDictionary, workspace);
+      validateSync(result.payload, harness);
+      if (result.assessment.disposition === "no_safe_context" && reservation.attempt < 2 && packet.sources.length) throw new Error("Recheck this proposed empty result against the original sources; preserve meaningful context or confirm a legitimate neutral empty result.");
+      store.saveDictionary(candidateDictionary);
+      return store.freeze(packet, result).payload;
     } catch (error) {
-      if (attempt === 1) throw error;
-      // Only local validator diagnostics enter the isolated correction prompt.
-      // Failed model output is never queued, echoed, or sent to Shifu.
+      store.failure(packet);
+      if (reservation.attempt >= 2 || output === undefined) throw error;
       validationFeedback = preparationValidationFeedback(error, output);
     }
   }
 }
 
+async function retryPreparation(harness) {
+  const sessionRef = option("--session-ref");
+  const fromTurn = numberOption("--from-turn");
+  if (!sessionRefIsSafe(sessionRef) || !Number.isSafeInteger(fromTurn) || fromTurn < 1) throw new Error("Choose the held session and original from-turn explicitly.");
+  const result = await withSyncLock(harness, async () => {
+    const store = continuityState(harness);
+    const state = store.load();
+    const identity = `${sessionRef}:${fromTurn}`;
+    const held = state.attempts[identity];
+    if (!held || held.payloadKey || state.outbox.some((entry) => entry.sessionRef === sessionRef && entry.fromTurn === fromTurn) || readPendingQueue(harness).segments.some((input) => input.sessionRef === sessionRef && input.toTurn >= fromTurn)) {
+      throw new Error("Only an unqueued preparation hold can be retried; frozen payloads keep their exact transport path.");
+    }
+    if ((readState(harness).sessions[sessionRef]?.lastSyncedTurn || 0) >= fromTurn) throw new Error("Accepted ranges require separately authorized historical recovery.");
+    const session = localSessions(harness).find((item) => item.sessionRef === sessionRef);
+    if (!session) throw new Error("The original session is unavailable; the hold was preserved.");
+    held.attempts = 0; held.status = "unresolved"; delete held.reason;
+    store.save(state);
+    const payload = await prepareV5Payload(harness, { ...session, fromTurn, toTurn: held.toTurn });
+    queuePayload(harness, payload);
+    console.error("The selected preparation is queued for its existing approval and transport policy.");
+  });
+  if (result?.skipped) throw new Error("Another sync is running; the hold was preserved.");
+}
+
 function preparationValidationFeedback(error, output) {
   const fields = [["title", output?.title, 160], ["summary", output?.summary, 1100]];
   for (const name of ["decisions", "verification"]) {
@@ -2589,6 +2724,7 @@ function preparationState(harness, queue) {
 }
 
 async function preparePendingQueue(harness, config, blockedSessions = new Set()) {
+  restoreContinuityQueue(harness, config);
   let queue = prunePendingQueue(harness);
   const previousFailures = new Set((queue.preparationErrors || []).map((error) => error.sessionRef));
   const errors = [];
@@ -2638,8 +2774,13 @@ async function preparePendingQueue(harness, config, blockedSessions = new Set())
 async function drainPendingQueue(harness, config, blockedSessions) {
   let synced = 0;
   let failures = 0;
-  for (const input of eligibleQueuedSegments(readPendingQueue(harness), config)) {
-    if (blockedSessions.has(input.sessionRef)) continue;
+  const store = continuityState(harness, config);
+  const entries = store.load().outbox.filter((entry) => entry.baseStatus !== "needs_review" && (entry.baseStatus !== "accepted" || entry.toTurn > (readState(harness).sessions[entry.sessionRef]?.lastSyncedTurn || 0)));
+  const queued = eligibleQueuedSegments(readPendingQueue(harness), config);
+  const excluded = new Set(normalizedPolicy(config).excludedSessionRefs);
+  const inputs = [...queued, ...entries.filter((entry) => !excluded.has(entry.sessionRef) && !queued.some((input) => JSON.stringify(input) === JSON.stringify(entry.payload))).map((entry) => entry.payload)];
+  for (const input of inputs) {
+    if (blockedSessions.has(input.sessionRef) || store.find(input)?.baseStatus === "needs_review") continue;
     try { await sendSync(harness, input, config); synced += 1; }
     catch (error) {
       failures += 1;
@@ -2650,11 +2791,35 @@ async function drainPendingQueue(harness, config, blockedSessions) {
   return { synced, failures };
 }
 
+async function flushContinuityAttachments(harness, config) {
+  const store = continuityState(harness, config);
+  if (!store.load().outbox.some((entry) => entry.baseStatus === "accepted" && !["attached", "not_applicable", "needs_review"].includes(entry.continuityStatus))) return;
+  let capable = false;
+  try {
+    const connection = await validateConnection(config.apiUrl, config.token, harness);
+    capable = connection.capabilities?.continuityAttachment === 1;
+  } catch { /* Base receipts remain valid while capability discovery is unavailable. */ }
+  await require("./continuity-state.js").flushContinuity(store, config, capable);
+}
+
+function restoreContinuityQueue(harness, config) {
+  const queue = readPendingQueue(harness);
+  const state = readState(harness);
+  for (const entry of continuityState(harness, config).load().outbox) {
+    if (entry.toTurn > (state.sessions[entry.sessionRef]?.lastSyncedTurn || 0) && entry.baseStatus !== "needs_review" && !queue.segments.some((payload) => JSON.stringify(payload) === JSON.stringify(entry.payload))) queuePayload(harness, entry.payload);
+  }
+}
+
 function eligibleQueuedSegments(queue, config) {
   const excluded = new Set(normalizedPolicy(config).excludedSessionRefs);
   return queue.segments.filter((input) => !excluded.has(input.sessionRef));
 }
 
+function continuityRecoveries(harness, config, queued) {
+  const excluded = new Set(normalizedPolicy(config).excludedSessionRefs);
+  return continuityState(harness, config).load().outbox.filter((entry) => entry.baseStatus === "awaiting_receipt" && !excluded.has(entry.sessionRef) && !queued.some((payload) => JSON.stringify(payload) === JSON.stringify(entry.payload))).map((entry) => entry.payload);
+}
+
 function approvalNotificationInvocation(harness, count, sessionRef) {
   const prompt = `Shifu sync approval: ${count} redacted segment${count === 1 ? " is" : "s are"} waiting locally. Tell the user to ask you to show the pending Shifu sync and approve it once. Do not inspect files, run tools, or sync anything in this session.`;
   if (harness === "codex") {
@@ -2706,7 +2871,9 @@ function notifyManualApproval(harness, count, existingSessionRef) {
 async function showPending(harness) {
   const config = readConfig(harness);
   const result = await withSyncLock(harness, async () => {
-    console.log(JSON.stringify(await preparePendingQueue(harness, config), null, 2));
+    const queue = await preparePendingQueue(harness, config);
+    queue.segments = [...eligibleQueuedSegments(queue, config), ...continuityRecoveries(harness, config, queue.segments)];
+    console.log(JSON.stringify(queue, null, 2));
   });
   if (result?.skipped) throw new Error("Another Shifu sync is already preparing this queue. Retry later.");
 }
@@ -2714,18 +2881,21 @@ async function showPending(harness) {
 async function approvePending(harness) {
   const config = readConfig(harness);
   const result = await withSyncLock(harness, async () => {
+    restoreContinuityQueue(harness, config);
     const queue = prunePendingQueue(harness);
     queue.segments = eligibleQueuedSegments(queue, config);
-    if (queue.segments.length === 0) {
+    const recoveries = continuityRecoveries(harness, config, queue.segments);
+    if (queue.segments.length === 0 && recoveries.length === 0) {
       console.error("There is no prepared Shifu activity to approve. Run pending to prepare the next ranges.");
       return;
     }
-    console.error(`\nThese ${queue.segments.length} redacted segment(s) will be sent to Shifu:\n${JSON.stringify(queue.segments, null, 2)}`);
+    console.error(`\nThese ${queue.segments.length + recoveries.length} redacted segment(s) will be sent to Shifu:\n${JSON.stringify([...queue.segments, ...recoveries], null, 2)}`);
     if (!hasFlag("--approved") && !(await confirm("Approve all pending segments"))) {
       console.error("Nothing was sent. The prepared queue is unchanged.");
       return;
     }
     const sent = await drainPendingQueue(harness, config, new Set());
+    await flushContinuityAttachments(harness, config);
     if (sent.failures) throw new Error("Some approved segments could not be acknowledged. Their exact payloads remain pending for retry.");
   });
   if (result?.skipped) throw new Error("Another Shifu sync is already running. Nothing was sent; retry later.");
@@ -2758,9 +2928,12 @@ async function scheduledSync(harness) {
       }
     }
     await flushPendingProfiles(harness, config);
+    restoreContinuityQueue(harness, config);
+    await flushContinuityAttachments(harness, config);
     if (policy.approvalMode === "manual") {
       const queue = await preparePendingQueue(harness, config);
-      const eligible = eligibleQueuedSegments(queue, config);
+      const queued = eligibleQueuedSegments(queue, config);
+      const eligible = [...queued, ...continuityRecoveries(harness, config, queued)];
       const pendingFingerprint = createHash("sha256").update(eligible.map((segment) => `${segment.sessionRef}:${segment.fromTurn}:${segment.toTurn}`).join("\n")).digest("hex");
       const changed = state.schedule?.pendingFingerprint !== pendingFingerprint;
       let approvalSessionRef = state.schedule?.approvalSessionRef;
@@ -2818,6 +2991,7 @@ async function scheduledSync(harness) {
         nextRunAt: Date.now() + (failures || failedFiles || pendingProfiles || remaining ? SCHEDULER_TICK_MINUTES / 60 : policy.intervalHours) * 3_600_000,
       };
     }
+    await flushContinuityAttachments(harness, config);
     const latest = readState(harness);
     latest.schedule = state.schedule;
     saveState(harness, latest);
@@ -2848,7 +3022,7 @@ function status(harness) {
       if (!eligibility.eligible) initialHistoryBlocked.push({ sessionRef: session.sessionRef, reason: eligibility.reason, cutoffAt: eligibility.cutoffAt });
     }
   }
-  console.log(JSON.stringify({ harness, runnerVersion: RUNNER_VERSION, apiUrl: config.apiUrl, inputFolder: folder, inputFolderError, contextPreferences: config.contextPreferences === undefined ? null : normalizeContextPreferences(config.contextPreferences), historicalContext: true, initialBackfill: config.initialBackfill || null, initialHistoryBlocked, syncPolicy: normalizedPolicy(config), schedule: state.schedule || null, pendingSegments: pending.segments.length, completedFiles: entries.filter((entry) => typeof entry === "string").length, pendingFiles: entries.filter((entry) => typeof entry === "object" && entry?.jobId).length, unsupportedFiles, sessions: state.sessions }, null, 2));
+  console.log(JSON.stringify({ continuity: continuityState(harness, config).summary(), harness, runnerVersion: RUNNER_VERSION, apiUrl: config.apiUrl, inputFolder: folder, inputFolderError, contextPreferences: config.contextPreferences === undefined ? null : normalizeContextPreferences(config.contextPreferences), historicalContext: true, initialBackfill: config.initialBackfill || null, initialHistoryBlocked, syncPolicy: normalizedPolicy(config), schedule: state.schedule || null, pendingSegments: pending.segments.length, completedFiles: entries.filter((entry) => typeof entry === "string").length, pendingFiles: entries.filter((entry) => typeof entry === "object" && entry?.jobId).length, unsupportedFiles, sessions: state.sessions }, null, 2));
 }
 
 async function syncFilesNow(harness) {
@@ -3088,21 +3262,21 @@ async function main() {
   if (command === "sync-files") return syncFilesNow(harness);
   if (command === "scheduled-sync") return scheduledSync(harness);
   if (command === "pending") return showPending(harness);
+  if (command === "retry-preparation") return retryPreparation(harness);
   if (command === "approve") return approvePending(harness);
   if (command === "status") return status(harness);
   if (command === "sessions" || command === "discover" || (command === "session" && (subCommand === "discover" || subCommand === "list" || !subCommand || subCommand.startsWith("-")))) return sessions(harness);
   if (command === "review") return review(harness);
   if (command === "review-source") return console.log(JSON.stringify(reviewSourcePacket(harness, option("--session-ref"), numberOption("--from-turn"), numberOption("--to-turn")), null, 2));
-  throw new Error("Use install, connect, mcp-setup, mcp-headers, disconnect, config update, status, sessions, review, review-source, sync, sync-files, pending, approve, or scheduled-sync.");
+  throw new Error("Use install, connect, mcp-setup, mcp-headers, disconnect, config update, status, sessions, review, review-source, sync, sync-files, pending, retry-preparation, approve, or scheduled-sync.");
 }
 
 if (require.main === module) {
   main()
-    .then(() => process.exit(0))
     .catch((error) => {
       console.error(error.message);
-      process.exit(1);
+      process.exitCode = 1;
     });
 }
 
-module.exports = { setupMcp, olderPackageVersion, updateRecommendedPackage, installPackageVersion, approvalNotificationInvocation, claudeReview, claudeSession, codexReview, codexSession, configPath, configuredSyncPolicy, connect, defaultInputFolder, disconnect, finalAssistantNote, install, installDestination, localSessions, normalizedPolicy, notificationSessionRef, opencodeQuery, opencodeReview, opencodeSessions, payloadForSegment, pendingSegments, prepareV5Payload, prepareInputFolder, profileForSegment, readSecret, removeHooks, requireSafeApiUrl, reviewCandidates, reviewSourcePacket, schedulerArtifact, statePath, syncWorkFiles, textIsSafe, unsyncedCodexSessions, validateAIProfileObservation, validateConnection, validateInputFolder, validSyncReceipt, validateApprovalMode, validateIntervalHours, validateSync, workFiles };
+module.exports = { setupMcp, olderPackageVersion, updateRecommendedPackage, installPackageVersion, approvalNotificationInvocation, claudeReview, claudeSession, codexReview, codexSession, configPath, configuredSyncPolicy, connect, defaultInputFolder, disconnect, finalAssistantNote, install, installDestination, localSessions, normalizedPolicy, notificationSessionRef, opencodeQuery, opencodeReview, opencodeSessions, payloadForSegment, pendingSegments, prepareV5Payload, preparationPacket, continuityState, flushContinuityAttachments, prepareInputFolder, profileForSegment, readSecret, removeHooks, requireSafeApiUrl, reviewCandidates, reviewSourcePacket, schedulerArtifact, statePath, syncWorkFiles, textIsSafe, unsyncedCodexSessions, validateAIProfileObservation, validateConnection, validateInputFolder, validSyncReceipt, validateApprovalMode, validateIntervalHours, validateSync, workFiles };

package/bin/source-review.js

@@ -191,10 +191,36 @@ function completedSourceTurn(selection, records) {
       if (message.role === "user") {
         turnCount += 1;
         closed = false;
-      } else if (message.role === "assistant") closed = message.finish === "stop" && message.error == null;
+      } else if (message.role === "assistant") closed = message.storageVersion !== 2 && message.finish === "stop" && message.error == null;
+      else if (message.storageVersion === 2 && message.role === "idle") closed = true;
     }
   } else throw new Error("Unsupported original-source harness.");
   return Math.max(0, turnCount - (closed ? 0 : 1));
 }
 
-module.exports = { selectedSourcePacket, scheduledSourcePacket, completedSourceTurn };
+function continuitySourcePacket(selection, records, minimumTurn = 1) {
+  const packet = scheduledSourcePacket(selection, records);
+  let first = Math.max(1, selection.fromTurn - 6, minimumTurn);
+  const adapters = { codex: codexSources, claude_code: claudeSources, opencode: openCodeSources };
+  const references = first < selection.fromTurn
+    ? adapters[selection.harness]({ ...selection, fromTurn: first, toTurn: selection.fromTurn - 1 }, records, true).sources : [];
+  const originalFirst = first;
+  let referenceSources = references;
+  function oversized() {
+    return Buffer.byteLength(JSON.stringify(referenceSources)) > 64 * 1024 ||
+      packet.sources.length + referenceSources.length > MAX_SOURCE_MESSAGES ||
+      Buffer.byteLength(JSON.stringify({ ...packet, referenceSources })) + 512 > MAX_PACKET_BYTES;
+  }
+  while (referenceSources.length && oversized()) {
+    first = referenceSources[0].turn + 1;
+    referenceSources = references.filter((source) => source.turn >= first);
+  }
+  return { ...packet, referenceSources, referenceCoverage: {
+    fromTurn: referenceSources.length ? referenceSources[0].turn : selection.fromTurn,
+    toTurn: selection.fromTurn - 1,
+    limited: first > Math.max(1, selection.fromTurn - 6) || minimumTurn > Math.max(1, selection.fromTurn - 6),
+    droppedTurns: Math.max(0, first - originalFirst),
+  } };
+}
+
+module.exports = { selectedSourcePacket, scheduledSourcePacket, continuitySourcePacket, completedSourceTurn };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@useshifu/coding-harness",
-  "version": "0.3.28",
+  "version": "0.3.30",
   "description": "Scheduled, redacted coding-harness activity sync for Shifu",
   "bin": {
     "coding-harness": "bin/shifu-harness.js",

package/skills/shifu-context/SKILL.md

@@ -12,7 +12,7 @@ Before first connection or MCP personalization, ask for any choices that are not
 When asked to connect Shifu context, run:
 
 ```sh
-npx --yes @useshifu/coding-harness@0.3.28 mcp-setup --harness codex
+npx --yes @useshifu/coding-harness@0.3.30 mcp-setup --harness codex
 ```
 
 MCP setup verifies the connection and tools, saves the choices and installs instructions; it does not sync sessions or upload the selected history. History selection governs eligible work for subsequent sync. Do not start sync as part of this setup unless the user separately asks for it. If a saved connection exists, reuse its key and do not run `connect` again.

package/skills/shifu-sync/SKILL.md

@@ -117,7 +117,7 @@ For an explicit sync request, first read `status` to determine the saved approva
 
 Use `kind: "activity"` for general harness work; it does not require a technical scope. Evidence may be empty when the reviewed source contains no durable context. Never invent impact, ownership, verification, or outcomes. A requested check, assistant proposal, or reported result must retain that status even when it is useful context.
 
-Make each supported item useful for later recall: retain the described action, feature or project purpose, specific problem, approach, and reported verification or outcome with its status and ownership intact. Preserve supported technologies, design patterns, configuration choices, and measured results; these are useful work details, not identifiers to redact. A descriptive project label such as "payment reconciliation service" is useful; synthesize it from the work's purpose instead of copying a raw project label that might contain a company or person's name. Preserve supported decisions and their rationale, accepted and rejected tradeoffs, lessons, and observed outcomes. Keep proposals explicitly labeled as proposals; do not turn an agent suggestion into a user-adopted decision or infer personal ownership. Use statements, per-item work context, linked decisions and verification, and permitted source metadata. Keep distinct facts separate, including two fixes in one feature that addressed different problems. Omit names of companies and people, customer details, proprietary identifiers, paths, code snippets, credentials, and unsupported impact. If a detail cannot be safely described, keep the supported parts. Prefer a summary that names the main distinct work items over a generic count alone.
+Make each supported item useful for later recall: retain the described action, feature or project purpose, specific problem, approach, and reported verification or outcome with its status and ownership intact. Preserve supported technologies, design patterns, configuration choices, and measured results; these are useful work details, not identifiers to redact. A descriptive project label such as "payment reconciliation service" is useful; synthesize it from the work's purpose instead of copying a raw project label that might contain a company or person's name. Preserve supported decisions and their rationale, accepted and rejected tradeoffs, lessons, and observed outcomes. Keep proposals explicitly labeled as proposals; do not turn an agent suggestion into a user-adopted decision or infer personal ownership. Use statements, per-item work context, linked decisions and verification, and permitted source metadata. Keep distinct facts separate, including two fixes in one feature that addressed different problems. Preserve the code-permitted public technology labels Turbopuffer, PostgreSQL, Go, TypeScript, React and Node.js when grounded and relevant. This does not authorize confidential customer or employer relationships. Omit other company and person names, customer details, proprietary identifiers, paths, code snippets, credentials, and unsupported impact. If a detail cannot be safely described, keep the supported parts. Prefer a summary that names the main distinct work items over a generic count alone.
 
 ```json
 {
@@ -157,3 +157,9 @@ Only when the user explicitly asks to force-sync, add `--force` to `sessions` wi
 Re-review each returned range with `review --harness codex --session-ref <sessionRef> --force --from-turn <fromTurn> --to-turn <toTurn>` and the same window (or `--all` for explicitly selected backfill/count selection). No prior sync ID is required. Read the returned final notes semantically and synthesize meaningful, attributable work evidence; do not use `payloadForSegment` or its unattended keyword projection for force sync. Completed work remains supported when the note uses Markdown such as `**Implemented**` or names a technique in inline code such as `Rust`. Retain supported technologies, design patterns, the problem, approach, and verified result while redacting names and sensitive identifiers. Never send an empty evidence list or a title-only replacement. If the notes support no work items, report that range as not replaced and leave its existing work and checkpoints untouched. Keep the returned reference and range unchanged and include `force: true` in the prepared payload. Send with `sync --harness codex --force` (plus `--approved` only after manual approval). If accepted ranges changed and partially overlap the selection, stop and re-discover/re-review the complete ranges instead of guessing boundaries.
 
 Force sync creates missing ranges or replaces existing exact ranges, preserves sync history, and never resets or lowers checkpoints. Automatic retries preserve the accepted result even if another force sync finishes meanwhile. A new invocation is a new explicit force request. Existing AI Profile observations stay unchanged. Do not add force to scheduled sync, approval queues, or file sync. Existing conversations keep their context; start a new chat in Shifu to use the latest work context.
+
+## Continuity preparation and review holds
+
+Scheduled preparation and pending review use bounded original lookback, code-bound entity placeholders and a separate local continuity outbox. Never add continuity fields to a v5 payload or rewrite a frozen queued body. `status` separates preparation, base receipt and companion attachment. A base receipt does not establish successful continuity attachment. Missing server capability leaves the frozen metadata pending. Do not guess a target sync from turn ranges.
+
+A preparation hold survives restarts and stops after two native attempts for unchanged input. Inspect its code-owned reason. Genuine neutral empties stay valid; never create filler evidence to clear a hold. A specific authorized retry of an unqueued held range can use `retry-preparation --harness codex --session-ref <opaque-session> --from-turn <original-start>`; this cannot replay accepted history or regenerate frozen transport. Replacing the key or API endpoint quarantines old checkpoints, queues, local entity dictionaries and continuity metadata. An interrupted replacement holds sync until connect resumes with the same selected connection. Do not upload, display or copy the raw-name dictionary into chat.