@the-open-engine/zeroshot 6.38.1 → 6.39.0

package/src/agent/context-pack-builder.js

@@ -1,367 +1,81 @@
-const { estimateTokensFromChars } = require('./context-metrics');
-
-const PRIORITY_RANK = {
-  required: 0,
-  high: 1,
-  medium: 2,
-  low: 3,
-};
-
-const DEFAULT_PRIORITY = 'medium';
-const TRUNCATION_SUFFIX = '\n\n[Context truncated to fit limit]\n';
-
-function normalizePriority(priority, required) {
-  if (required) return 'required';
-  if (priority && PRIORITY_RANK[priority] !== undefined) return priority;
-  return DEFAULT_PRIORITY;
-}
-
-function normalizePack(pack, index) {
-  const priority = normalizePriority(pack.priority, pack.required);
-  return {
-    ...pack,
-    priority,
-    required: pack.required || priority === 'required',
-    order: pack.order ?? index,
-  };
-}
-
-function renderVariant(pack, variant, cache) {
-  const cacheKey = `${pack.id}:${variant}`;
-  if (cache.has(cacheKey)) return cache.get(cacheKey);
-
-  let text = '';
-  if (variant === 'full') {
-    text = typeof pack.render === 'function' ? pack.render() : '';
-  } else if (variant === 'compact') {
-    text = typeof pack.compact === 'function' ? pack.compact() : '';
-  }
-
-  if (typeof text !== 'string') {
-    text = '';
-  }
-
-  const chars = text.length;
-  const estimatedTokens = estimateTokensFromChars(chars);
-  const rendered = { text, chars, estimatedTokens };
-  cache.set(cacheKey, rendered);
-  return rendered;
-}
-
-function sortByPriorityThenOrder(a, b) {
-  const priorityDelta = PRIORITY_RANK[a.priority] - PRIORITY_RANK[b.priority];
-  if (priorityDelta !== 0) return priorityDelta;
-  return a.order - b.order;
-}
-
-function sortByOrder(a, b) {
-  return a.order - b.order;
-}
-
-function sortByPriorityDescThenOrderDesc(a, b) {
-  const priorityDelta = PRIORITY_RANK[b.priority] - PRIORITY_RANK[a.priority];
-  if (priorityDelta !== 0) return priorityDelta;
-  return b.order - a.order;
-}
-
-function selectVariant(pack, remainingTokens, cache) {
-  const full = renderVariant(pack, 'full', cache);
-  const compact = pack.compact ? renderVariant(pack, 'compact', cache) : null;
-
-  const hasFull = full.chars > 0;
-  const hasCompact = compact && compact.chars > 0;
-
-  if (!hasFull && !hasCompact) {
-    return {
-      status: 'skipped',
-      variant: null,
-      reason: 'empty',
-      chars: 0,
-      estimatedTokens: 0,
-    };
-  }
-
-  if (pack.required) {
-    let chosen = full;
-    let variant = 'full';
-
-    if (!hasFull && hasCompact) {
-      chosen = compact;
-      variant = 'compact';
-    } else if (
-      Number.isFinite(remainingTokens) &&
-      hasCompact &&
-      full.estimatedTokens > remainingTokens
-    ) {
-      if (
-        compact.estimatedTokens <= remainingTokens ||
-        compact.estimatedTokens < full.estimatedTokens
-      ) {
-        chosen = compact;
-        variant = 'compact';
-      }
+"use strict";
+const contextMetrics = require("./context-metrics");
+const contextPackCore = require("./context-pack-core");
+const contextPackSelection = require("./context-pack-selection");
+const contextPackTruncation = require("./context-pack-truncation");
+const { estimateTokensFromChars } = contextMetrics;
+const { normalizePack, sortByOrder } = contextPackCore;
+const { selectPacks } = contextPackSelection;
+const { applyMaxCharsGuard } = contextPackTruncation;
+function renderSelectedContext(packs, selected) {
+    let context = '';
+    for (const pack of packs) {
+        const selectedPack = selected.get(pack.id);
+        if (selectedPack) {
+            context += selectedPack.text;
+        }
     }
-
-    return {
-      status: 'included',
-      variant,
-      chars: chosen.chars,
-      estimatedTokens: chosen.estimatedTokens,
-      text: chosen.text,
-    };
-  }
-
-  if (!Number.isFinite(remainingTokens) || full.estimatedTokens <= remainingTokens) {
-    return {
-      status: 'included',
-      variant: 'full',
-      chars: full.chars,
-      estimatedTokens: full.estimatedTokens,
-      text: full.text,
-    };
-  }
-
-  if (hasCompact && compact.estimatedTokens <= remainingTokens) {
-    return {
-      status: 'included',
-      variant: 'compact',
-      chars: compact.chars,
-      estimatedTokens: compact.estimatedTokens,
-      text: compact.text,
-    };
-  }
-
-  return {
-    status: 'skipped',
-    variant: null,
-    reason: 'budget',
-    chars: 0,
-    estimatedTokens: 0,
-  };
-}
-
-function truncateText(text, targetChars) {
-  if (text.length <= targetChars) {
-    return { text, truncated: false };
-  }
-
-  if (targetChars <= 0) {
-    return { text: '', truncated: true };
-  }
-
-  if (targetChars <= TRUNCATION_SUFFIX.length) {
-    return { text: text.slice(0, targetChars), truncated: true };
-  }
-
-  const sliceLength = targetChars - TRUNCATION_SUFFIX.length;
-  return { text: text.slice(0, sliceLength) + TRUNCATION_SUFFIX, truncated: true };
+    return context;
 }
-
-function applyMaxCharsGuard({ packs, selected, decisions, cache, maxChars, totalChars }) {
-  let currentChars = totalChars;
-  if (!Number.isFinite(maxChars) || currentChars <= maxChars) {
-    return { applied: false, beforeChars: totalChars, afterChars: totalChars };
-  }
-
-  const includedOptional = packs
-    .filter((pack) => selected.has(pack.id) && !pack.required)
-    .sort(sortByPriorityDescThenOrderDesc);
-
-  for (const pack of includedOptional) {
-    if (currentChars <= maxChars) break;
-    const decision = decisions.get(pack.id);
-    if (!decision || decision.variant === 'compact' || !pack.compact) continue;
-
-    const compact = renderVariant(pack, 'compact', cache);
-    if (compact.chars === 0 || compact.chars >= decision.chars) continue;
-
-    const previousChars = decision.chars;
-    selected.set(pack.id, { ...compact, variant: 'compact' });
-    decision.variant = 'compact';
-    decision.chars = compact.chars;
-    decision.estimatedTokens = compact.estimatedTokens;
-    decision.reason = decision.reason || 'max_chars';
-
-    currentChars -= previousChars - compact.chars;
-    currentChars = Math.max(0, currentChars);
-  }
-
-  for (const pack of includedOptional) {
-    if (currentChars <= maxChars) break;
-    if (!selected.has(pack.id)) continue;
-
-    const decision = decisions.get(pack.id);
-    currentChars -= decision?.chars || 0;
-    selected.delete(pack.id);
-    if (decision) {
-      decision.status = 'skipped';
-      decision.reason = decision.reason || 'max_chars';
-      decision.chars = 0;
-      decision.estimatedTokens = 0;
-    }
-  }
-
-  if (currentChars > maxChars) {
-    let overage = currentChars - maxChars;
-    const requiredCandidates = packs
-      .filter((pack) => selected.has(pack.id) && pack.required)
-      .sort((a, b) => {
-        const preserveDelta = (a.preserve ? 1 : 0) - (b.preserve ? 1 : 0);
-        if (preserveDelta !== 0) return preserveDelta;
-        const sizeDelta = (selected.get(b.id)?.chars || 0) - (selected.get(a.id)?.chars || 0);
-        if (sizeDelta !== 0) return sizeDelta;
-        return b.order - a.order;
-      });
-
-    for (const pack of requiredCandidates) {
-      if (overage <= 0) break;
-      const decision = decisions.get(pack.id);
-      const selectedPack = selected.get(pack.id);
-      if (!decision || !selectedPack) continue;
-
-      const targetChars = Math.max(0, selectedPack.chars - overage);
-      const truncated = truncateText(selectedPack.text, targetChars);
-      if (truncated.text.length === selectedPack.chars) continue;
-
-      const newChars = truncated.text.length;
-      const reduced = selectedPack.chars - newChars;
-      overage -= reduced;
-
-      selected.set(pack.id, {
-        text: truncated.text,
-        chars: newChars,
-        estimatedTokens: estimateTokensFromChars(newChars),
-        variant: selectedPack.variant,
-      });
-
-      decision.chars = newChars;
-      decision.estimatedTokens = estimateTokensFromChars(newChars);
-      decision.truncated = true;
-      decision.reason = decision.reason || 'max_chars';
-    }
-  }
-
-  const afterChars = Array.from(selected.values()).reduce((sum, item) => sum + item.chars, 0);
-  return { applied: true, beforeChars: totalChars, afterChars };
+function finalizePackDecisions(packs, decisions) {
+    return packs.map((pack) => {
+        const decision = decisions.get(pack.id);
+        if (!decision) {
+            throw new Error(`Missing context-pack decision for "${pack.id}"`);
+        }
+        return {
+            id: decision.id,
+            section: decision.section,
+            priority: decision.priority,
+            required: decision.required,
+            status: decision.status,
+            variant: decision.variant,
+            chars: decision.chars,
+            estimatedTokens: decision.estimatedTokens,
+            order: decision.order,
+            reason: decision.reason,
+            truncated: decision.truncated || false,
+        };
+    });
 }
-
-function buildContextPacks({ packs, maxTokens, maxChars }) {
-  const normalized = packs.map(normalizePack);
-  const selectionOrder = normalized.slice().sort(sortByPriorityThenOrder);
-  const renderCache = new Map();
-  const decisions = new Map();
-  const selected = new Map();
-
-  let remainingTokens = Number.isFinite(maxTokens) ? maxTokens : Infinity;
-  let overBudgetTokens = 0;
-
-  for (const pack of selectionOrder) {
-    const selection = selectVariant(pack, remainingTokens, renderCache);
-    const decision = {
-      id: pack.id,
-      section: pack.section || null,
-      priority: pack.priority,
-      required: pack.required,
-      status: selection.status,
-      variant: selection.variant,
-      chars: selection.chars,
-      estimatedTokens: selection.estimatedTokens,
-      order: pack.order,
-      reason: selection.reason || null,
-    };
-
-    decisions.set(pack.id, decision);
-
-    if (selection.status !== 'included') {
-      continue;
-    }
-
-    selected.set(pack.id, {
-      text: selection.text,
-      chars: selection.chars,
-      estimatedTokens: selection.estimatedTokens,
-      variant: selection.variant,
+function buildContextPacks({ packs, maxTokens, maxChars, }) {
+    const normalized = packs.map(normalizePack);
+    const ordered = normalized.slice().sort(sortByOrder);
+    const selection = selectPacks(normalized, maxTokens);
+    const initialContext = renderSelectedContext(ordered, selection.selected);
+    const truncation = applyMaxCharsGuard({
+        packs: ordered,
+        selected: selection.selected,
+        decisions: selection.decisions,
+        cache: selection.renderCache,
+        maxChars,
+        totalChars: initialContext.length,
     });
-
-    if (!Number.isFinite(remainingTokens)) {
-      continue;
-    }
-
-    if (selection.estimatedTokens > remainingTokens) {
-      overBudgetTokens += selection.estimatedTokens - remainingTokens;
-      remainingTokens = 0;
-    } else {
-      remainingTokens -= selection.estimatedTokens;
-    }
-  }
-
-  const ordered = normalized.slice().sort(sortByOrder);
-  let context = '';
-  for (const pack of ordered) {
-    const selectedPack = selected.get(pack.id);
-    if (selectedPack) {
-      context += selectedPack.text;
-    }
-  }
-
-  const totalChars = context.length;
-  const truncation = applyMaxCharsGuard({
-    packs: ordered,
-    selected,
-    decisions,
-    cache: renderCache,
-    maxChars,
-    totalChars,
-  });
-
-  if (truncation.applied) {
-    context = '';
-    for (const pack of ordered) {
-      const selectedPack = selected.get(pack.id);
-      if (selectedPack) {
-        context += selectedPack.text;
-      }
-    }
-  }
-
-  const finalChars = context.length;
-  const finalTokens = estimateTokensFromChars(finalChars);
-  const packDecisions = ordered.map((pack) => {
-    const decision = decisions.get(pack.id);
+    const context = truncation.applied
+        ? renderSelectedContext(ordered, selection.selected)
+        : initialContext;
+    const finalChars = context.length;
+    const finalTokens = estimateTokensFromChars(finalChars);
     return {
-      id: decision.id,
-      section: decision.section,
-      priority: decision.priority,
-      required: decision.required,
-      status: decision.status,
-      variant: decision.variant,
-      chars: decision.chars,
-      estimatedTokens: decision.estimatedTokens,
-      order: decision.order,
-      reason: decision.reason,
-      truncated: decision.truncated || false,
+        context,
+        packDecisions: finalizePackDecisions(ordered, selection.decisions),
+        budget: {
+            maxTokens,
+            remainingTokens: Number.isFinite(selection.remainingTokens)
+                ? selection.remainingTokens
+                : null,
+            overBudgetTokens: selection.overBudgetTokens,
+            finalTokens,
+        },
+        truncation: {
+            maxContextChars: {
+                applied: truncation.applied,
+                beforeChars: truncation.beforeChars,
+                afterChars: truncation.afterChars,
+            },
+        },
     };
-  });
-
-  return {
-    context,
-    packDecisions,
-    budget: {
-      maxTokens,
-      remainingTokens: Number.isFinite(remainingTokens) ? remainingTokens : null,
-      overBudgetTokens,
-      finalTokens,
-    },
-    truncation: {
-      maxContextChars: {
-        applied: truncation.applied,
-        beforeChars: truncation.beforeChars,
-        afterChars: truncation.afterChars,
-      },
-    },
-  };
 }
-
 module.exports = {
-  buildContextPacks,
+    buildContextPacks,
 };

package/src/agent/context-pack-builder.ts

@@ -0,0 +1,105 @@
+import contextMetrics = require('./context-metrics');
+import contextPackCore = require('./context-pack-core');
+import contextPackSelection = require('./context-pack-selection');
+import contextPackTruncation = require('./context-pack-truncation');
+
+import type {
+  BuildContextPacksParams,
+  ContextPackDecision,
+  ContextPackResult,
+  MutableContextPackDecision,
+  NormalizedContextPack,
+  SelectedVariant,
+} from './context-pack-types';
+
+const { estimateTokensFromChars } = contextMetrics;
+const { normalizePack, sortByOrder } = contextPackCore;
+const { selectPacks } = contextPackSelection;
+const { applyMaxCharsGuard } = contextPackTruncation;
+
+function renderSelectedContext(
+  packs: readonly NormalizedContextPack[],
+  selected: ReadonlyMap<string, SelectedVariant>
+): string {
+  let context = '';
+  for (const pack of packs) {
+    const selectedPack = selected.get(pack.id);
+    if (selectedPack) {
+      context += selectedPack.text;
+    }
+  }
+  return context;
+}
+
+function finalizePackDecisions(
+  packs: readonly NormalizedContextPack[],
+  decisions: ReadonlyMap<string, MutableContextPackDecision>
+): ContextPackDecision[] {
+  return packs.map((pack): ContextPackDecision => {
+    const decision = decisions.get(pack.id);
+    if (!decision) {
+      throw new Error(`Missing context-pack decision for "${pack.id}"`);
+    }
+    return {
+      id: decision.id,
+      section: decision.section,
+      priority: decision.priority,
+      required: decision.required,
+      status: decision.status,
+      variant: decision.variant,
+      chars: decision.chars,
+      estimatedTokens: decision.estimatedTokens,
+      order: decision.order,
+      reason: decision.reason,
+      truncated: decision.truncated || false,
+    };
+  });
+}
+
+function buildContextPacks({
+  packs,
+  maxTokens,
+  maxChars,
+}: BuildContextPacksParams): ContextPackResult {
+  const normalized = packs.map(normalizePack);
+  const ordered = normalized.slice().sort(sortByOrder);
+  const selection = selectPacks(normalized, maxTokens);
+  const initialContext = renderSelectedContext(ordered, selection.selected);
+  const truncation = applyMaxCharsGuard({
+    packs: ordered,
+    selected: selection.selected,
+    decisions: selection.decisions,
+    cache: selection.renderCache,
+    maxChars,
+    totalChars: initialContext.length,
+  });
+  const context = truncation.applied
+    ? renderSelectedContext(ordered, selection.selected)
+    : initialContext;
+  const finalChars = context.length;
+  const finalTokens = estimateTokensFromChars(finalChars);
+
+  return {
+    context,
+    packDecisions: finalizePackDecisions(ordered, selection.decisions),
+    budget: {
+      maxTokens,
+      remainingTokens: Number.isFinite(selection.remainingTokens)
+        ? selection.remainingTokens
+        : null,
+      overBudgetTokens: selection.overBudgetTokens,
+      finalTokens,
+    },
+    truncation: {
+      maxContextChars: {
+        applied: truncation.applied,
+        beforeChars: truncation.beforeChars,
+        afterChars: truncation.afterChars,
+      },
+    },
+  };
+}
+
+export = {
+  buildContextPacks,
+};

package/src/agent/context-pack-core.js

@@ -0,0 +1,70 @@
+"use strict";
+const contextMetrics = require("./context-metrics");
+const { estimateTokensFromChars } = contextMetrics;
+const PRIORITY_RANK = {
+    required: 0,
+    high: 1,
+    medium: 2,
+    low: 3,
+};
+const DEFAULT_PRIORITY = 'medium';
+function isContextPackPriority(priority) {
+    return (priority === 'required' || priority === 'high' || priority === 'medium' || priority === 'low');
+}
+function normalizePriority(priority, required) {
+    if (required)
+        return 'required';
+    if (priority && isContextPackPriority(priority))
+        return priority;
+    return DEFAULT_PRIORITY;
+}
+function normalizePack(pack, index) {
+    const priority = normalizePriority(pack.priority, pack.required);
+    return {
+        ...pack,
+        priority,
+        required: pack.required || priority === 'required',
+        order: pack.order ?? index,
+    };
+}
+function renderVariant(pack, variant, cache) {
+    const cacheKey = `${pack.id}:${variant}`;
+    const cached = cache.get(cacheKey);
+    if (cached)
+        return cached;
+    let text = '';
+    if (variant === 'full') {
+        text = typeof pack.render === 'function' ? pack.render() : '';
+    }
+    else if (variant === 'compact') {
+        text = typeof pack.compact === 'function' ? pack.compact() : '';
+    }
+    const safeText = typeof text === 'string' ? text : '';
+    const chars = safeText.length;
+    const estimatedTokens = estimateTokensFromChars(chars);
+    const rendered = { text: safeText, chars, estimatedTokens };
+    cache.set(cacheKey, rendered);
+    return rendered;
+}
+function sortByPriorityThenOrder(a, b) {
+    const priorityDelta = PRIORITY_RANK[a.priority] - PRIORITY_RANK[b.priority];
+    if (priorityDelta !== 0)
+        return priorityDelta;
+    return a.order - b.order;
+}
+function sortByOrder(a, b) {
+    return a.order - b.order;
+}
+function sortByPriorityDescThenOrderDesc(a, b) {
+    const priorityDelta = PRIORITY_RANK[b.priority] - PRIORITY_RANK[a.priority];
+    if (priorityDelta !== 0)
+        return priorityDelta;
+    return b.order - a.order;
+}
+module.exports = {
+    normalizePack,
+    renderVariant,
+    sortByPriorityThenOrder,
+    sortByOrder,
+    sortByPriorityDescThenOrderDesc,
+};