@the-open-engine/zeroshot 6.38.1 → 6.39.0

package/src/agent/context-pack-truncation.ts

@@ -0,0 +1,208 @@
+import contextMetrics = require('./context-metrics');
+import contextPackCore = require('./context-pack-core');
+
+import type {
+  ContextPackTruncationStage,
+  MutableContextPackDecision,
+  NormalizedContextPack,
+  RenderedVariant,
+  SelectedVariant,
+} from './context-pack-types';
+
+interface MaxCharsGuardParams {
+  packs: NormalizedContextPack[];
+  selected: Map<string, SelectedVariant>;
+  decisions: Map<string, MutableContextPackDecision>;
+  cache: Map<string, RenderedVariant>;
+  maxChars: number | undefined;
+  totalChars: number;
+}
+
+interface MaxCharsMutationParams {
+  selected: Map<string, SelectedVariant>;
+  decisions: Map<string, MutableContextPackDecision>;
+  cache: Map<string, RenderedVariant>;
+  maxChars: number;
+}
+
+interface RequiredTruncationParams {
+  packs: readonly NormalizedContextPack[];
+  selected: Map<string, SelectedVariant>;
+  decisions: Map<string, MutableContextPackDecision>;
+  currentChars: number;
+  maxChars: number;
+}
+
+const { estimateTokensFromChars } = contextMetrics;
+const { renderVariant, sortByPriorityDescThenOrderDesc } = contextPackCore;
+
+const TRUNCATION_SUFFIX = '\n\n[Context truncated to fit limit]\n';
+
+function truncateText(text: string, targetChars: number): { text: string; truncated: boolean } {
+  if (text.length <= targetChars) {
+    return { text, truncated: false };
+  }
+
+  if (targetChars <= 0) {
+    return { text: '', truncated: true };
+  }
+
+  if (targetChars <= TRUNCATION_SUFFIX.length) {
+    return { text: text.slice(0, targetChars), truncated: true };
+  }
+
+  const sliceLength = targetChars - TRUNCATION_SUFFIX.length;
+  return {
+    text: text.slice(0, sliceLength) + TRUNCATION_SUFFIX,
+    truncated: true,
+  };
+}
+
+function compactOptionalPacks(
+  packs: readonly NormalizedContextPack[],
+  { selected, decisions, cache, maxChars }: MaxCharsMutationParams,
+  startingChars: number
+): number {
+  let currentChars = startingChars;
+  for (const pack of packs) {
+    if (currentChars <= maxChars) break;
+    const decision = decisions.get(pack.id);
+    if (!decision || decision.variant === 'compact' || !pack.compact) continue;
+
+    const compact = renderVariant(pack, 'compact', cache);
+    if (compact.chars === 0 || compact.chars >= decision.chars) continue;
+
+    const previousChars = decision.chars;
+    selected.set(pack.id, { ...compact, variant: 'compact' });
+    decision.variant = 'compact';
+    decision.chars = compact.chars;
+    decision.estimatedTokens = compact.estimatedTokens;
+    decision.reason = decision.reason || 'max_chars';
+
+    currentChars -= previousChars - compact.chars;
+    currentChars = Math.max(0, currentChars);
+  }
+  return currentChars;
+}
+
+function dropOptionalPacks(
+  packs: readonly NormalizedContextPack[],
+  { selected, decisions, maxChars }: MaxCharsMutationParams,
+  startingChars: number
+): number {
+  let currentChars = startingChars;
+  for (const pack of packs) {
+    if (currentChars <= maxChars) break;
+    if (!selected.has(pack.id)) continue;
+
+    const decision = decisions.get(pack.id);
+    currentChars -= decision?.chars || 0;
+    selected.delete(pack.id);
+    if (decision) {
+      decision.status = 'skipped';
+      decision.reason = decision.reason || 'max_chars';
+      decision.chars = 0;
+      decision.estimatedTokens = 0;
+    }
+  }
+  return currentChars;
+}
+
+function sortRequiredTruncationCandidates(
+  packs: readonly NormalizedContextPack[],
+  selected: ReadonlyMap<string, SelectedVariant>
+): NormalizedContextPack[] {
+  return packs
+    .filter((pack) => selected.has(pack.id) && pack.required)
+    .sort((a, b) => {
+      const preserveDelta = (a.preserve ? 1 : 0) - (b.preserve ? 1 : 0);
+      if (preserveDelta !== 0) return preserveDelta;
+      const sizeDelta = (selected.get(b.id)?.chars || 0) - (selected.get(a.id)?.chars || 0);
+      if (sizeDelta !== 0) return sizeDelta;
+      return b.order - a.order;
+    });
+}
+
+function truncateRequiredPacks({
+  packs,
+  selected,
+  decisions,
+  currentChars,
+  maxChars,
+}: RequiredTruncationParams): void {
+  let overage = currentChars - maxChars;
+  const requiredCandidates = sortRequiredTruncationCandidates(packs, selected);
+
+  for (const pack of requiredCandidates) {
+    if (overage <= 0) break;
+    const decision = decisions.get(pack.id);
+    const selectedPack = selected.get(pack.id);
+    if (!decision || !selectedPack) continue;
+
+    const targetChars = Math.max(0, selectedPack.chars - overage);
+    const truncated = truncateText(selectedPack.text, targetChars);
+    if (truncated.text.length === selectedPack.chars) continue;
+
+    const newChars = truncated.text.length;
+    overage -= selectedPack.chars - newChars;
+    selected.set(pack.id, {
+      text: truncated.text,
+      chars: newChars,
+      estimatedTokens: estimateTokensFromChars(newChars),
+      variant: selectedPack.variant,
+    });
+
+    decision.chars = newChars;
+    decision.estimatedTokens = estimateTokensFromChars(newChars);
+    decision.truncated = true;
+    decision.reason = decision.reason || 'max_chars';
+  }
+}
+
+function selectedChars(selected: ReadonlyMap<string, SelectedVariant>): number {
+  return Array.from(selected.values()).reduce((sum, item) => sum + item.chars, 0);
+}
+
+function applyMaxCharsGuard({
+  packs,
+  selected,
+  decisions,
+  cache,
+  maxChars,
+  totalChars,
+}: MaxCharsGuardParams): ContextPackTruncationStage {
+  if (typeof maxChars !== 'number' || !Number.isFinite(maxChars) || totalChars <= maxChars) {
+    return {
+      applied: false,
+      beforeChars: totalChars,
+      afterChars: totalChars,
+    };
+  }
+
+  const includedOptional = packs
+    .filter((pack) => selected.has(pack.id) && !pack.required)
+    .sort(sortByPriorityDescThenOrderDesc);
+  const mutationParams = { selected, decisions, cache, maxChars };
+  let currentChars = compactOptionalPacks(includedOptional, mutationParams, totalChars);
+  currentChars = dropOptionalPacks(includedOptional, mutationParams, currentChars);
+
+  if (currentChars > maxChars) {
+    truncateRequiredPacks({
+      packs,
+      selected,
+      decisions,
+      currentChars,
+      maxChars,
+    });
+  }
+
+  return {
+    applied: true,
+    beforeChars: totalChars,
+    afterChars: selectedChars(selected),
+  };
+}
+
+export = {
+  applyMaxCharsGuard,
+};

package/src/agent/context-pack-types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/src/agent/context-pack-types.ts

@@ -0,0 +1,98 @@
+export type ContextPackPriority = 'required' | 'high' | 'medium' | 'low';
+export type ContextPackVariant = 'full' | 'compact';
+export type ContextPackStatus = 'included' | 'skipped';
+
+export interface ContextPack {
+  id: string;
+  section?: string | null;
+  priority?: string;
+  required?: boolean;
+  order?: number;
+  preserve?: boolean;
+  render?: () => unknown;
+  compact?: () => unknown;
+}
+
+export interface NormalizedContextPack extends ContextPack {
+  priority: ContextPackPriority;
+  required: boolean;
+  order: number;
+}
+
+export interface RenderedVariant {
+  text: string;
+  chars: number;
+  estimatedTokens: number;
+}
+
+export interface SelectedVariant extends RenderedVariant {
+  variant: ContextPackVariant;
+}
+
+export interface IncludedSelection extends SelectedVariant {
+  status: 'included';
+}
+
+export interface SkippedSelection {
+  status: 'skipped';
+  variant: null;
+  reason: 'empty' | 'budget';
+  chars: 0;
+  estimatedTokens: 0;
+}
+
+export type PackSelection = IncludedSelection | SkippedSelection;
+
+export interface ContextPackDecision {
+  id: string;
+  section: string | null;
+  priority: ContextPackPriority;
+  required: boolean;
+  status: ContextPackStatus;
+  variant: ContextPackVariant | null;
+  chars: number;
+  estimatedTokens: number;
+  order: number;
+  reason: string | null;
+  truncated: boolean;
+}
+
+export interface MutableContextPackDecision extends Omit<ContextPackDecision, 'truncated'> {
+  truncated?: boolean;
+}
+
+export interface PackSelectionState {
+  renderCache: Map<string, RenderedVariant>;
+  decisions: Map<string, MutableContextPackDecision>;
+  selected: Map<string, SelectedVariant>;
+  remainingTokens: number;
+  overBudgetTokens: number;
+}
+
+export interface ContextPackBudget {
+  maxTokens: number | undefined;
+  remainingTokens: number | null;
+  overBudgetTokens: number;
+  finalTokens: number;
+}
+
+export interface ContextPackTruncationStage {
+  applied: boolean;
+  beforeChars: number;
+  afterChars: number;
+}
+
+export interface ContextPackResult {
+  context: string;
+  packDecisions: ContextPackDecision[];
+  budget: ContextPackBudget;
+  truncation: {
+    maxContextChars: ContextPackTruncationStage;
+  };
+}
+
+export interface BuildContextPacksParams {
+  packs: ContextPack[];
+  maxTokens?: number | undefined;
+  maxChars?: number | undefined;
+}

package/src/agent/context-replay-policy.js

@@ -1,51 +1,42 @@
+"use strict";
 const RAW_LOG_ONLY_REPLAY_POLICY = 'raw_log_only';
 const CONTEXT_REPLAY_POLICY = 'context';
-
 function buildRawLogOnlyMetadata(extra = {}) {
-  return {
-    ...extra,
-    contextSafe: false,
-    replayPolicy: RAW_LOG_ONLY_REPLAY_POLICY,
-  };
+    return {
+        ...extra,
+        contextSafe: false,
+        replayPolicy: RAW_LOG_ONLY_REPLAY_POLICY,
+    };
 }
-
 function getReplayPolicy(message) {
-  return message?.metadata?.replayPolicy ?? message?.content?.data?.replayPolicy;
+    return message?.metadata?.replayPolicy ?? message?.content?.data?.replayPolicy;
 }
-
 function getContextSafe(message) {
-  if (typeof message?.metadata?.contextSafe === 'boolean') {
-    return message.metadata.contextSafe;
-  }
-
-  if (typeof message?.content?.data?.contextSafe === 'boolean') {
-    return message.content.data.contextSafe;
-  }
-
-  return null;
+    if (typeof message?.metadata?.contextSafe === 'boolean') {
+        return message.metadata.contextSafe;
+    }
+    if (typeof message?.content?.data?.contextSafe === 'boolean') {
+        return message.content.data.contextSafe;
+    }
+    return null;
 }
-
 function isReplayableMessage(message) {
-  const contextSafe = getContextSafe(message);
-  if (contextSafe !== null) {
-    return contextSafe;
-  }
-
-  const replayPolicy = getReplayPolicy(message);
-  if (replayPolicy === RAW_LOG_ONLY_REPLAY_POLICY) {
-    return false;
-  }
-
-  if (replayPolicy === CONTEXT_REPLAY_POLICY) {
-    return true;
-  }
-
-  return message?.topic !== 'AGENT_OUTPUT';
+    const contextSafe = getContextSafe(message);
+    if (contextSafe !== null) {
+        return contextSafe;
+    }
+    const replayPolicy = getReplayPolicy(message);
+    if (replayPolicy === RAW_LOG_ONLY_REPLAY_POLICY) {
+        return false;
+    }
+    if (replayPolicy === CONTEXT_REPLAY_POLICY) {
+        return true;
+    }
+    return message?.topic !== 'AGENT_OUTPUT';
 }
-
 module.exports = {
-  RAW_LOG_ONLY_REPLAY_POLICY,
-  CONTEXT_REPLAY_POLICY,
-  buildRawLogOnlyMetadata,
-  isReplayableMessage,
+    RAW_LOG_ONLY_REPLAY_POLICY,
+    CONTEXT_REPLAY_POLICY,
+    buildRawLogOnlyMetadata,
+    isReplayableMessage,
 };

package/src/agent/context-replay-policy.ts

@@ -0,0 +1,71 @@
+const RAW_LOG_ONLY_REPLAY_POLICY = 'raw_log_only';
+const CONTEXT_REPLAY_POLICY = 'context';
+
+type ReplayMetadata = Record<string, unknown> & {
+  readonly contextSafe?: unknown;
+  readonly replayPolicy?: unknown;
+};
+
+interface ReplayMessage {
+  readonly topic?: unknown;
+  readonly metadata?: ReplayMetadata | null;
+  readonly content?: {
+    readonly data?: {
+      readonly contextSafe?: unknown;
+      readonly replayPolicy?: unknown;
+    } | null;
+  } | null;
+}
+
+function buildRawLogOnlyMetadata(extra: Record<string, unknown> = {}): {
+  contextSafe: false;
+  replayPolicy: typeof RAW_LOG_ONLY_REPLAY_POLICY;
+  [key: string]: unknown;
+} {
+  return {
+    ...extra,
+    contextSafe: false,
+    replayPolicy: RAW_LOG_ONLY_REPLAY_POLICY,
+  };
+}
+
+function getReplayPolicy(message: ReplayMessage | null | undefined): unknown {
+  return message?.metadata?.replayPolicy ?? message?.content?.data?.replayPolicy;
+}
+
+function getContextSafe(message: ReplayMessage | null | undefined): boolean | null {
+  if (typeof message?.metadata?.contextSafe === 'boolean') {
+    return message.metadata.contextSafe;
+  }
+
+  if (typeof message?.content?.data?.contextSafe === 'boolean') {
+    return message.content.data.contextSafe;
+  }
+
+  return null;
+}
+
+function isReplayableMessage(message: ReplayMessage | null | undefined): boolean {
+  const contextSafe = getContextSafe(message);
+  if (contextSafe !== null) {
+    return contextSafe;
+  }
+
+  const replayPolicy = getReplayPolicy(message);
+  if (replayPolicy === RAW_LOG_ONLY_REPLAY_POLICY) {
+    return false;
+  }
+
+  if (replayPolicy === CONTEXT_REPLAY_POLICY) {
+    return true;
+  }
+
+  return message?.topic !== 'AGENT_OUTPUT';
+}
+
+export = {
+  RAW_LOG_ONLY_REPLAY_POLICY,
+  CONTEXT_REPLAY_POLICY,
+  buildRawLogOnlyMetadata,
+  isReplayableMessage,
+};

package/src/agent/critical-agent-policy.js

@@ -1,17 +1,13 @@
-// @ts-nocheck
-
+"use strict";
 // Validators publish a rejection when their task exhausts retries. Every other named role can
 // execute provider work, including user-defined and orchestrator roles, so exhaustion must
 // terminalize the cluster by default instead of requiring this policy to know each future role.
 const NON_TERMINAL_AGENT_ROLES = new Set(['validator']);
-
 function isCriticalAgent(agent) {
-  if (agent?.id === 'consensus-coordinator') return true;
-  return (
-    typeof agent?.role === 'string' &&
-    agent.role.length > 0 &&
-    !NON_TERMINAL_AGENT_ROLES.has(agent.role)
-  );
+    if (agent?.id === 'consensus-coordinator')
+        return true;
+    return (typeof agent?.role === 'string' &&
+        agent.role.length > 0 &&
+        !NON_TERMINAL_AGENT_ROLES.has(agent.role));
 }
-
 module.exports = { isCriticalAgent };

package/src/agent/critical-agent-policy.ts

@@ -0,0 +1,20 @@
+interface AgentIdentity {
+  readonly id?: unknown;
+  readonly role?: unknown;
+}
+
+// Validators publish a rejection when their task exhausts retries. Every other named role can
+// execute provider work, including user-defined and orchestrator roles, so exhaustion must
+// terminalize the cluster by default instead of requiring this policy to know each future role.
+const NON_TERMINAL_AGENT_ROLES: ReadonlySet<string> = new Set(['validator']);
+
+function isCriticalAgent(agent: AgentIdentity | null | undefined): boolean {
+  if (agent?.id === 'consensus-coordinator') return true;
+  return (
+    typeof agent?.role === 'string' &&
+    agent.role.length > 0 &&
+    !NON_TERMINAL_AGENT_ROLES.has(agent.role)
+  );
+}
+
+export = { isCriticalAgent };

package/src/agent/guidance-queue.js

@@ -1,99 +1,86 @@
+"use strict";
+const ledgerSequence = require("../ledger-sequence");
 const GUIDANCE_BLOCK_START = '<<GUIDANCE_QUEUE_START>>';
 const GUIDANCE_BLOCK_END = '<<GUIDANCE_QUEUE_END>>';
-const { compareMessageSequences } = require('../ledger-sequence');
-
+const { compareMessageSequences } = ledgerSequence;
 function formatGuidanceMessage(message) {
-  const timestamp = Number.isFinite(message.timestamp)
-    ? new Date(message.timestamp).toISOString()
-    : new Date().toISOString();
-  const sender = message.sender || 'unknown';
-  const topic = message.topic || 'GUIDANCE';
-  const target = message.receiver || message.target_agent_id;
-  const targetSuffix = target ? ` -> ${target}` : '';
-
-  let formatted = `[${timestamp}] ${sender} (${topic}${targetSuffix})\n`;
-  if (message.content?.text) {
-    formatted += `${message.content.text}\n`;
-  }
-  if (message.content?.data) {
-    formatted += `${JSON.stringify(message.content.data, null, 2)}\n`;
-  }
-
-  return formatted.trimEnd();
+    const timestamp = typeof message.timestamp === 'number' && Number.isFinite(message.timestamp)
+        ? new Date(message.timestamp).toISOString()
+        : new Date().toISOString();
+    const sender = message.sender || 'unknown';
+    const topic = message.topic || 'GUIDANCE';
+    const target = message.receiver || message.target_agent_id;
+    const targetSuffix = target ? ` -> ${target}` : '';
+    let formatted = `[${timestamp}] ${sender} (${topic}${targetSuffix})\n`;
+    if (message.content?.text) {
+        formatted += `${message.content.text}\n`;
+    }
+    if (message.content?.data) {
+        formatted += `${JSON.stringify(message.content.data, null, 2)}\n`;
+    }
+    return formatted.trimEnd();
 }
-
 function orderGuidanceMessages(messages, orderBySequence) {
-  return messages.slice().sort((a, b) => {
-    if (orderBySequence) {
-      return compareMessageSequences(a.sequence || '0', b.sequence || '0');
-    }
-    return (a.timestamp || 0) - (b.timestamp || 0);
-  });
+    return messages.slice().sort((left, right) => {
+        if (orderBySequence) {
+            return compareMessageSequences(left.sequence || '0', right.sequence || '0');
+        }
+        return (left.timestamp || 0) - (right.timestamp || 0);
+    });
 }
-
 function formatGuidanceBlock(messages, { orderBySequence = false } = {}) {
-  if (!Array.isArray(messages) || messages.length === 0) return '';
-
-  const ordered = orderGuidanceMessages(messages, orderBySequence);
-
-  let block = '## Guidance (Queued)\n\n';
-  block += `${GUIDANCE_BLOCK_START}\n`;
-
-  ordered.forEach((message, index) => {
-    block += `${formatGuidanceMessage(message)}\n`;
-    if (index < ordered.length - 1) {
-      block += '\n';
-    }
-  });
-
-  block += `\n${GUIDANCE_BLOCK_END}\n\n`;
-  return block;
+    if (!Array.isArray(messages) || messages.length === 0)
+        return '';
+    const ordered = orderGuidanceMessages(messages, orderBySequence);
+    let block = '## Guidance (Queued)\n\n';
+    block += `${GUIDANCE_BLOCK_START}\n`;
+    ordered.forEach((message, index) => {
+        block += `${formatGuidanceMessage(message)}\n`;
+        if (index < ordered.length - 1) {
+            block += '\n';
+        }
+    });
+    block += `\n${GUIDANCE_BLOCK_END}\n\n`;
+    return block;
 }
-
-function collectQueuedGuidance({
-  messageBus,
-  clusterId,
-  agentId,
-  afterId,
-  throughId,
-  lastDeliveredAt,
-  limit,
-}) {
-  if (!messageBus) {
-    throw new Error('collectQueuedGuidance: messageBus is required');
-  }
-  if (!clusterId) {
-    throw new Error('collectQueuedGuidance: clusterId is required');
-  }
-  if (!agentId) {
-    throw new Error('collectQueuedGuidance: agentId is required');
-  }
-
-  const messages = messageBus.queryGuidanceMailbox({
-    cluster_id: clusterId,
-    target_agent_id: agentId,
-    afterId,
-    throughId,
-    lastDeliveredAt,
-    limit,
-  });
-
-  if (!messages.length) {
+function emptyQueuedGuidance() {
     return { messages: [], latestSequence: null, latestTimestamp: null, guidanceBlock: '' };
-  }
-
-  const sequenceBounded = afterId !== undefined || throughId !== undefined;
-  const ordered = orderGuidanceMessages(messages, sequenceBounded);
-  const latestSequence = ordered[ordered.length - 1].sequence;
-  const latestTimestamp = ordered[ordered.length - 1].timestamp;
-  const guidanceBlock = formatGuidanceBlock(ordered, { orderBySequence: sequenceBounded });
-
-  return { messages: ordered, latestSequence, latestTimestamp, guidanceBlock };
 }
-
+function collectQueuedGuidance({ messageBus, clusterId, agentId, afterId, throughId, lastDeliveredAt, limit, }) {
+    if (!messageBus) {
+        throw new Error('collectQueuedGuidance: messageBus is required');
+    }
+    if (!clusterId) {
+        throw new Error('collectQueuedGuidance: clusterId is required');
+    }
+    if (!agentId) {
+        throw new Error('collectQueuedGuidance: agentId is required');
+    }
+    const messages = messageBus.queryGuidanceMailbox({
+        cluster_id: clusterId,
+        target_agent_id: agentId,
+        afterId,
+        throughId,
+        lastDeliveredAt,
+        limit,
+    });
+    if (!messages.length) {
+        return emptyQueuedGuidance();
+    }
+    const sequenceBounded = afterId !== undefined || throughId !== undefined;
+    const ordered = orderGuidanceMessages(messages, sequenceBounded);
+    const latestMessage = ordered.at(-1);
+    if (!latestMessage) {
+        return emptyQueuedGuidance();
+    }
+    const latestSequence = latestMessage.sequence;
+    const latestTimestamp = latestMessage.timestamp;
+    const guidanceBlock = formatGuidanceBlock(ordered, { orderBySequence: sequenceBounded });
+    return { messages: ordered, latestSequence, latestTimestamp, guidanceBlock };
+}
 module.exports = {
-  GUIDANCE_BLOCK_START,
-  GUIDANCE_BLOCK_END,
-  formatGuidanceBlock,
-  collectQueuedGuidance,
+    GUIDANCE_BLOCK_START,
+    GUIDANCE_BLOCK_END,
+    formatGuidanceBlock,
+    collectQueuedGuidance,
 };