negotium 0.6.17 → 0.7.0

package/dist/agent-helpers.js

@@ -2053,12 +2053,12 @@ ${toolBuffer.join(`
         break;
       case "tool_use": {
         const u = ev;
-        toolBuffer.push(`<!-- Tool: ${u.name} ${truncate(JSON.stringify(u.input), 200)} -->`);
+        toolBuffer.push(opts.formatToolUse?.(u) ?? `<!-- Tool: ${u.name} ${truncate(JSON.stringify(u.input), 200)} -->`);
         break;
       }
       case "tool_result": {
         const u = ev;
-        toolBuffer.push(`<!-- Tool result: ${truncate(u.content, 200)} -->`);
+        toolBuffer.push(opts.formatToolResult?.(u) ?? `<!-- Tool result: ${truncate(u.content, 200)} -->`);
         break;
       }
       case "error": {
@@ -3001,7 +3001,7 @@ var init_codex = __esm(async () => {
 });
 
 // ../../packages/core/src/version.ts
-var NEGOTIUM_VERSION = "0.6.17";
+var NEGOTIUM_VERSION = "0.7.0";
 
 // ../../packages/core/src/agents/codex-native-multi-agent.ts
 import { spawn as spawn2 } from "child_process";
@@ -12123,8 +12123,117 @@ RULES:
 - Do NOT invent file paths or facts not present in the transcript.
 - Keep the entire summary under 1500 words.`, CHARS_PER_TEXT_TOKEN = 3.5, CHARS_PER_CJK_TOKEN = 0.9;
 
-// ../../packages/core/src/storage/token-stats.ts
+// ../../packages/core/src/agents/compaction-tool-projection.ts
 import { createHash as createHash5 } from "crypto";
+function digest(value) {
+  return createHash5("sha256").update(value).digest("hex").slice(0, 16);
+}
+function boundedEdges(value, edgeChars) {
+  if (value.length <= edgeChars * 2)
+    return value;
+  const omitted = value.length - edgeChars * 2;
+  return `${value.slice(0, edgeChars)}
+[\u2026 omitted ${omitted.toLocaleString()} chars \u2026]
+${value.slice(-edgeChars)}`;
+}
+function quoted(value) {
+  return JSON.stringify(value);
+}
+function salientLines(value) {
+  const selected = [];
+  let chars = 0;
+  for (const line of value.split(`
+`)) {
+    if (!SALIENT_LINE_RE.test(line))
+      continue;
+    const trimmed = line.trim();
+    if (!trimmed || selected.includes(trimmed))
+      continue;
+    if (chars + trimmed.length + 1 > TOOL_OUTPUT_SALIENT_CHARS)
+      break;
+    selected.push(trimmed);
+    chars += trimmed.length + 1;
+  }
+  return selected.join(`
+`);
+}
+function formatToolUse2(event) {
+  const input = JSON.stringify(event.input ?? {});
+  const projected = estimateTextTokens(input) <= MAX_TOOL_INPUT_TOKENS ? input : boundedEdges(input, TOOL_INPUT_EDGE_CHARS);
+  return [
+    "[Negotium tool use \u2014 quoted untrusted data]",
+    `id_json: ${quoted(event.toolUseId ?? "unknown")}`,
+    `name_json: ${quoted(event.name)}`,
+    "quoted_untrusted_data: true",
+    `input_json: ${quoted(projected)}`,
+    "[/Negotium tool use]"
+  ].join(`
+`);
+}
+function formatToolResult(event, hash, duplicate) {
+  const metadata = event.metadata;
+  const header = [
+    "[Negotium tool result \u2014 quoted untrusted data]",
+    `id_json: ${quoted(event.toolUseId)}`,
+    `status: ${event.isError ? "error" : "success"}`,
+    "quoted_untrusted_data: true",
+    `hash: sha256:${hash}`,
+    `original_bytes: ${metadata?.originalBytes ?? Buffer.byteLength(event.content)}`,
+    ...metadata?.returnedBytes !== undefined ? [`returned_bytes: ${metadata.returnedBytes}`] : [],
+    ...metadata?.omittedBytes !== undefined ? [`omitted_bytes: ${metadata.omittedBytes}`] : [],
+    ...metadata?.outputPath ? [`output_path_json: ${quoted(metadata.outputPath)}`] : []
+  ];
+  if (duplicate) {
+    return [
+      ...header,
+      "content: [duplicate of the most recent result with this hash]",
+      "[/Negotium tool result]"
+    ].join(`
+`);
+  }
+  if (estimateTextTokens(event.content) <= MAX_TOOL_OUTPUT_TOKENS) {
+    return [...header, `content_json: ${quoted(event.content)}`, "[/Negotium tool result]"].join(`
+`);
+  }
+  const salient = salientLines(event.content);
+  return [
+    ...header,
+    "content: [bounded projection; full output remains in the raw conversation log]",
+    `head_tail_json: ${quoted(boundedEdges(event.content, TOOL_OUTPUT_EDGE_CHARS))}`,
+    ...salient ? [`salient_lines_json: ${quoted(salient)}`] : [],
+    "[/Negotium tool result]"
+  ].join(`
+`);
+}
+function extractCompactionChatPairs(entries) {
+  const remainingByHash = new Map;
+  const hashByEvent = new WeakMap;
+  for (const entry of entries) {
+    if (entry.event.type !== "tool_result")
+      continue;
+    const hash = digest(entry.event.content);
+    hashByEvent.set(entry.event, hash);
+    remainingByHash.set(hash, (remainingByHash.get(hash) ?? 0) + 1);
+  }
+  return extractChatPairs(entries, {
+    includeToolAnnotations: true,
+    formatToolUse: formatToolUse2,
+    formatToolResult: (event) => {
+      const hash = hashByEvent.get(event) ?? digest(event.content);
+      const remaining = Math.max(0, (remainingByHash.get(hash) ?? 1) - 1);
+      remainingByHash.set(hash, remaining);
+      return formatToolResult(event, hash, remaining > 0);
+    }
+  });
+}
+var MAX_TOOL_INPUT_TOKENS = 2000, MAX_TOOL_OUTPUT_TOKENS = 8000, TOOL_INPUT_EDGE_CHARS = 1000, TOOL_OUTPUT_EDGE_CHARS = 2000, TOOL_OUTPUT_SALIENT_CHARS = 2000, SALIENT_LINE_RE;
+var init_compaction_tool_projection = __esm(() => {
+  init_shared();
+  SALIENT_LINE_RE = /\b(error|failed?|failure|warning|exit(?:ed)?|expected|received|not found|denied|passed|tests?|assert|timeout|exception)\b/i;
+});
+
+// ../../packages/core/src/storage/token-stats.ts
+import { createHash as createHash6 } from "crypto";
 import { mkdirSync as mkdirSync12, writeFileSync as writeFileSync10 } from "fs";
 import { join as join21 } from "path";
 function emptyBucket() {
@@ -12139,7 +12248,7 @@ function emptyBucket() {
 }
 function tokenStatsFileId(userId) {
   const rawUserId = String(userId);
-  return /^[A-Za-z0-9][A-Za-z0-9_.@-]{0,255}$/.test(rawUserId) && !rawUserId.includes("..") ? rawUserId : `sha256-${createHash5("sha256").update(rawUserId).digest("hex")}`;
+  return /^[A-Za-z0-9][A-Za-z0-9_.@-]{0,255}$/.test(rawUserId) && !rawUserId.includes("..") ? rawUserId : `sha256-${createHash6("sha256").update(rawUserId).digest("hex")}`;
 }
 function queriesPath(userId) {
   const fileId = tokenStatsFileId(userId);
@@ -12270,11 +12379,14 @@ var init_token_stats = __esm(async () => {
 // ../../packages/core/src/agents/idle-compact.ts
 function cancelIdleCompactForTopic(topicId) {
   const timer = timers2.get(topicId);
-  if (!timer)
-    return false;
-  clearTimeout(timer);
+  const contextLimitTimer = contextLimitTimers.get(topicId);
+  if (timer)
+    clearTimeout(timer);
+  if (contextLimitTimer)
+    clearTimeout(contextLimitTimer);
   timers2.delete(topicId);
-  return true;
+  contextLimitTimers.delete(topicId);
+  return Boolean(timer || contextLimitTimer);
 }
 function envFlagEnabled2(name, fallback) {
   const raw = process.env[name]?.trim().toLowerCase();
@@ -12316,6 +12428,38 @@ function scheduleIdleCompactForTopic(topicId, userId) {
   timers2.set(topicId, timer);
   return "scheduled";
 }
+function scheduleContextLimitCompactForTopic(topicId, userId, contextTokens, contextWindow) {
+  if (!idleCompactEnabled())
+    return "disabled";
+  if (!Number.isFinite(contextWindow) || contextWindow <= 0)
+    return "below-threshold";
+  const percent = contextTokens / contextWindow * 100;
+  if (!Number.isFinite(percent) || percent < CONTEXT_LIMIT_COMPACT_PERCENT) {
+    return "below-threshold";
+  }
+  const topic = getTopic(topicId);
+  if (!topic)
+    return "topic-not-found";
+  if (!topic.agent)
+    return "not-ai-invited";
+  if (topic.aiMode === "mention")
+    return "mention-only-channel";
+  const existing = contextLimitTimers.get(topicId);
+  if (existing)
+    clearTimeout(existing);
+  const retry = () => scheduleContextLimitCompactForTopic(topicId, userId, contextTokens, contextWindow);
+  const timer = setTimeout(() => {
+    contextLimitTimers.delete(topicId);
+    runIdleCompactForTopic(topicId, userId, {
+      minContextPercent: CONTEXT_LIMIT_COMPACT_PERCENT,
+      reason: "context-limit-compact",
+      onBusy: retry
+    });
+  }, DEFAULT_CONTEXT_LIMIT_DELAY_MS);
+  timer.unref?.();
+  contextLimitTimers.set(topicId, timer);
+  return "scheduled";
+}
 async function runIdleCompactForTopic(topicId, userId, options = {}) {
   if (!idleCompactEnabled())
     return "disabled";
@@ -12354,7 +12498,7 @@ async function runIdleCompactForTopic(topicId, userId, options = {}) {
     return compactTopicSession(id, actorId, reason, { preemptive: false });
   });
   try {
-    const result = await compact(topicId, owner, "idle-compact");
+    const result = await compact(topicId, owner, options.reason ?? "idle-compact");
     if (result.busy) {
       logger.debug({ topicId, percent: Math.round(percent) }, "idle-compact: topic became busy, rescheduling");
       if (options.onBusy)
@@ -12374,7 +12518,7 @@ async function runIdleCompactForTopic(topicId, userId, options = {}) {
     return "failed";
   }
 }
-var DEFAULT_IDLE_DELAY_MS2, DEFAULT_MIN_CONTEXT_PERCENT = 50, timers2;
+var DEFAULT_IDLE_DELAY_MS2, DEFAULT_CONTEXT_LIMIT_DELAY_MS = 1000, CONTEXT_LIMIT_COMPACT_PERCENT = 90, DEFAULT_MIN_CONTEXT_PERCENT = 50, timers2, contextLimitTimers;
 var init_idle_compact = __esm(async () => {
   init_logger();
   await init_active_rooms();
@@ -12383,6 +12527,7 @@ var init_idle_compact = __esm(async () => {
   await init_token_stats();
   DEFAULT_IDLE_DELAY_MS2 = 6 * 60 * 60 * 1000;
   timers2 = new Map;
+  contextLimitTimers = new Map;
 });
 
 // ../../packages/core/src/agents/topic-cleanup.ts
@@ -12907,11 +13052,13 @@ var init_runtime_turn_requests = __esm(async () => {
 // ../../packages/core/src/topics/session.ts
 var exports_session = {};
 __export(exports_session, {
+  splitCompactionPairs: () => splitCompactionPairs,
   shouldUseCompactionLog: () => shouldUseCompactionLog,
   shouldCompactForkEntries: () => shouldCompactForkEntries,
   restartTopicSession: () => restartTopicSession,
   createCompactedRolloutEntries: () => createCompactedRolloutEntries,
   compactTopicSession: () => compactTopicSession,
+  COMPACTION_RETAINED_TAIL_TOKENS: () => COMPACTION_RETAINED_TAIL_TOKENS,
   AUTO_FORK_COMPACTION_TOKENS: () => AUTO_FORK_COMPACTION_TOKENS
 });
 import { randomUUID as randomUUID11 } from "crypto";
@@ -13059,6 +13206,61 @@ The assistant response is the authoritative summary of all earlier context.` &&
   }
   return;
 }
+function withoutCompactionSentinels(entries) {
+  const filtered = [];
+  for (let index = 0;index < entries.length; index++) {
+    const current2 = entries[index]?.event;
+    const next = entries[index + 1]?.event;
+    if (current2?.type === "user_message" && current2.synthetic === "compaction" && next?.type === "result") {
+      index += 1;
+      continue;
+    }
+    const entry = entries[index];
+    if (entry)
+      filtered.push(entry);
+  }
+  return filtered;
+}
+function pairTokens(pair) {
+  return estimateConversationTokens([{ content: pair.userText }, { content: pair.assistantText }]);
+}
+function splitCompactionPairs(entries, retainedTailTokens = COMPACTION_RETAINED_TAIL_TOKENS) {
+  const pairs = extractCompactionChatPairs(withoutCompactionSentinels(entries));
+  let retainedStart = pairs.length;
+  let retainedTokens = 0;
+  for (let index = pairs.length - 1;index >= 0; index--) {
+    const pair = pairs[index];
+    if (!pair)
+      continue;
+    const tokens = pairTokens(pair);
+    if (retainedTokens + tokens > retainedTailTokens)
+      break;
+    retainedTokens += tokens;
+    retainedStart = index;
+  }
+  if (retainedStart === 0)
+    return { summaryPairs: pairs, retainedPairs: [] };
+  return {
+    summaryPairs: pairs.slice(0, retainedStart),
+    retainedPairs: pairs.slice(retainedStart)
+  };
+}
+function retainedPairEntries(agent, pairs) {
+  const entries = [];
+  for (const pair of pairs) {
+    const now = new Date().toISOString();
+    entries.push({
+      ts: now,
+      agent,
+      event: { type: "user_message", content: pair.userText }
+    }, {
+      ts: now,
+      agent,
+      event: { type: "result", content: pair.assistantText, stopReason: "end_turn" }
+    });
+  }
+  return entries;
+}
 function fitCompactionChunk(text2, limit) {
   if (text2.length <= limit)
     return text2;
@@ -13072,7 +13274,7 @@ function fitCompactionChunk(text2, limit) {
   const tailChars = available - headChars;
   return `${text2.slice(0, headChars)}${marker}${text2.slice(-tailChars)}`;
 }
-function buildCompactionSource(topicId, userId, entries, visibleMessages) {
+function buildCompactionSource(topicId, userId, entries, visibleMessages, providerPairs) {
   const sections = [];
   const previous = previousCompactedSummary(entries);
   if (previous)
@@ -13084,10 +13286,10 @@ ${previous}`);
     sections.push(`## Durable topic memory
 ${fitCompactionChunk(memory, COMPACTION_MEMORY_CHARS)}`);
   }
-  const rows = (visibleMessages ?? getAllMessagesForTopic(topicId)).filter((row) => row.author_id !== "system" && row.kind !== "system" && row.kind !== "tool" && !row.id.startsWith("tasks-") && row.text.trim());
+  const rows = (previous ? [] : visibleMessages ?? getAllMessagesForTopic(topicId)).filter((row) => row.author_id !== "system" && row.kind !== "system" && row.kind !== "tool" && !row.id.startsWith("tasks-") && row.text.trim());
   const usedByContext = sections.reduce((sum, section) => sum + section.length + 2, 0);
   const conversationBudget = Math.max(0, COMPACTION_SOURCE_CHARS - usedByContext);
-  const pairs = extractChatPairs(entries);
+  const pairs = providerPairs ?? extractCompactionChatPairs(entries);
   const hasVisibleRows = rows.length > 0;
   const providerBudget = pairs.length > 0 ? hasVisibleRows ? Math.floor(conversationBudget * 0.65) : conversationBudget : 0;
   const providerTranscript = [];
@@ -13400,12 +13602,14 @@ function shouldCompactForkEntries(entries, thresholdTokens = AUTO_FORK_COMPACTIO
   return estimateConversationTokens(messages) >= thresholdTokens;
 }
 async function createCompactedRolloutEntries(request, summarize = summarizeTopicContext) {
-  const source = buildCompactionSource(request.topicId, request.userId, request.entries, request.visibleMessages);
+  const { summaryPairs, retainedPairs } = splitCompactionPairs(request.entries, request.retainedTailTokens);
+  const source = buildCompactionSource(request.topicId, request.userId, request.entries, request.visibleMessages, summaryPairs);
   if (!source)
     throw new Error(`Nothing to compact in "${request.topicTitle}".`);
   const {
     entries: _entries,
     visibleMessages: _visibleMessages,
+    retainedTailTokens: _retainedTailTokens,
     timeoutMs = shouldUseCompactionLog(source) ? COMPACTION_LOG_TIMEOUT_MS : COMPACTION_TIMEOUT_MS,
     summaryModel,
     summaryEffort,
@@ -13419,7 +13623,10 @@ async function createCompactedRolloutEntries(request, summarize = summarizeTopic
   }, summarize, timeoutMs)).trim();
   if (!summary)
     throw new Error("Context compaction returned an empty summary.");
-  return compactEntries(request.agent, summary.slice(0, COMPACTION_OUTPUT_CHARS));
+  return [
+    ...compactEntries(request.agent, summary.slice(0, COMPACTION_OUTPUT_CHARS)),
+    ...retainedPairEntries(request.agent, retainedPairs)
+  ];
 }
 async function cleanupNewRollout(agent, cwd, sessionId) {
   try {
@@ -13576,8 +13783,9 @@ async function compactTopicSession(topicId, userId, reason = "topic-session-comp
     maintenance.finish();
   }
 }
-var RESET_TURN_WAIT_MS = 5000, RESET_MEMORY_ARCHIVE_WAIT_MS, COMPACTION_INLINE_CHARS = 1e5, COMPACTION_SOURCE_CHARS, COMPACTION_MEMORY_CHARS = 80000, COMPACTION_OUTPUT_CHARS = 30000, COMPACTION_TIMEOUT_MS, COMPACTION_LOG_TIMEOUT_MS, COMPACTION_LOG_MAX_CALLS = 12, COMPACTION_LOG_MAX_TOTAL_BYTES, COMPACTION_LOG_MAX_CHUNK_BYTES, COMPACT_CONTEXT_MARKER = "[Negotium compacted context]", AUTO_FORK_COMPACTION_TOKENS = 28000;
+var RESET_TURN_WAIT_MS = 5000, RESET_MEMORY_ARCHIVE_WAIT_MS, COMPACTION_INLINE_CHARS = 1e5, COMPACTION_SOURCE_CHARS, COMPACTION_MEMORY_CHARS = 80000, COMPACTION_OUTPUT_CHARS = 30000, COMPACTION_RETAINED_TAIL_TOKENS = 64000, COMPACTION_TIMEOUT_MS, COMPACTION_LOG_TIMEOUT_MS, COMPACTION_LOG_MAX_CALLS = 12, COMPACTION_LOG_MAX_TOTAL_BYTES, COMPACTION_LOG_MAX_CHUNK_BYTES, COMPACT_CONTEXT_MARKER = "[Negotium compacted context]", AUTO_FORK_COMPACTION_TOKENS = 28000;
 var init_session = __esm(async () => {
+  init_compaction_tool_projection();
   await init_idle_archiver();
   await init_idle_compact();
   await init_agents();
@@ -13622,7 +13830,7 @@ __export(exports_derive, {
   TopicForkCompactionError: () => TopicForkCompactionError,
   TopicDeriveBusyError: () => TopicDeriveBusyError
 });
-import { createHash as createHash6, randomUUID as randomUUID12 } from "crypto";
+import { createHash as createHash7, randomUUID as randomUUID12 } from "crypto";
 import { mkdirSync as mkdirSync14, rmSync as rmSync5, unlinkSync as unlinkSync13 } from "fs";
 function getTopics(opts = {}) {
   return listTopics(opts).filter((topic) => !isLegacySharedGeneral(topic.id));
@@ -13688,7 +13896,7 @@ function captureForkSnapshot(sourceTopicId, userId, topicTitle) {
       maxRowid: messageRows.at(-1)?.rowid ?? 0
     },
     active: isTopicRunning(sourceTopicId),
-    canonicalDigest: createHash6("sha256").update(entries.map((entry) => JSON.stringify(entry)).join(`
+    canonicalDigest: createHash7("sha256").update(entries.map((entry) => JSON.stringify(entry)).join(`
 `)).digest("hex")
   };
 }
@@ -14227,7 +14435,7 @@ __export(exports_session_asks, {
   clearPendingAsk: () => clearPendingAsk,
   PENDING_ASK_TTL_MS: () => PENDING_ASK_TTL_MS
 });
-import { createHash as createHash7 } from "crypto";
+import { createHash as createHash8 } from "crypto";
 import {
   closeSync as closeSync2,
   mkdirSync as mkdirSync16,
@@ -14241,15 +14449,15 @@ import {
 import { dirname as dirname14, join as join27 } from "path";
 function pendingAskDir(userId) {
   const rawUserId = String(userId);
-  const safeUserId = /^[A-Za-z0-9][A-Za-z0-9_.@-]{0,255}$/.test(rawUserId) && !rawUserId.includes("..") ? rawUserId : `sha256-${createHash7("sha256").update(rawUserId).digest("hex")}`;
+  const safeUserId = /^[A-Za-z0-9][A-Za-z0-9_.@-]{0,255}$/.test(rawUserId) && !rawUserId.includes("..") ? rawUserId : `sha256-${createHash8("sha256").update(rawUserId).digest("hex")}`;
   return join27(resolveStorageSessionAsksDir(), safeUserId);
 }
 function encodeAskKey(key) {
   return JSON.stringify([key.from, key.to]);
 }
 function pendingAskPath(key) {
-  const digest = createHash7("sha256").update(encodeAskKey(key)).digest("hex");
-  return join27(pendingAskDir(key.userId), `${ASK_FILENAME_PREFIX}${digest}.pending`);
+  const digest2 = createHash8("sha256").update(encodeAskKey(key)).digest("hex");
+  return join27(pendingAskDir(key.userId), `${ASK_FILENAME_PREFIX}${digest2}.pending`);
 }
 function v2PendingAskPath(key) {
   const encoded = Buffer.from(encodeAskKey(key), "utf8").toString("base64url");
@@ -16348,6 +16556,9 @@ ${JSON.stringify(event.input ?? {})}`);
           if (!silent) {
             scheduleIdleArchiveForTopic(topicId, execution?.actorUserId ?? userId);
             scheduleIdleCompactForTopic(topicId, execution?.actorUserId ?? userId);
+            if (event.usage?.contextTokens !== undefined && event.usage.contextWindow !== undefined) {
+              scheduleContextLimitCompactForTopic(topicId, execution?.actorUserId ?? userId, event.usage.contextTokens, event.usage.contextWindow);
+            }
             hub.broadcastDone(topicId, queryId, event.usage ? {
               input: event.usage.inputTokens,
               output: event.usage.outputTokens,
@@ -19957,4 +20168,4 @@ export {
   DEFAULT_SELF_CONFIG_PRODUCT
 };
 
-//# debugId=823FBA6C6CCA73FD64756E2164756E21
+//# debugId=5098AD5EB3D2D82464756E2164756E21