@brunobracaioli/aibrain 0.1.0-beta.1 → 0.3.0-beta.13

package/dist/assets/context-selection-fitting.mjs

@@ -0,0 +1,194 @@
+export const SELECTION_FIT_OVERHEAD_TOKENS = 256;
+export const SELECTION_FIT_MAX_BATCHES = 4;
+export const SELECTION_FIT_ESTIMATOR = 'utf8-bytes-v1';
+function utf8ByteLength(value) {
+  return new TextEncoder().encode(value).byteLength;
+}
+export function estimateSelectionTokens(value) {
+  return utf8ByteLength(JSON.stringify(value));
+}
+function freezeProviderState(value) {
+  if (typeof value === 'object' && value !== null) {
+    Object.freeze(value);
+    for (const nested of Object.values(value)) freezeProviderState(nested);
+  }
+  return value;
+}
+function marker(text, size) {
+  if (text.length <= size) return text;
+  const head = Math.floor(size * 0.7);
+  const tail = Math.floor(size * 0.3);
+  return `${text.slice(0, head)}…[omitted ${text.length - size} chars]…${text.slice(-tail)}`;
+}
+export function normalizeSelectionItems(projection, preserveRecentItems) {
+  const protectedIds = new Set(
+    projection.items
+      .slice(Math.max(0, projection.items.length - preserveRecentItems))
+      .map((item) => item.id),
+  );
+  for (const item of projection.items) {
+    if (
+      item.protected ||
+      item.kind === 'user_message' ||
+      item.kind === 'plan' ||
+      item.kind === 'reference' ||
+      item.pinReasons.length > 0
+    )
+      protectedIds.add(item.id);
+  }
+  return projection.items.map((item) =>
+    protectedIds.has(item.id)
+      ? {
+          ...item,
+          protected: true,
+          pinReasons:
+            item.pinReasons.length > 0
+              ? item.pinReasons
+              : [
+                  item.kind === 'user_message'
+                    ? 'user_message'
+                    : item.kind === 'plan'
+                      ? 'plan'
+                      : item.kind === 'reference'
+                        ? 'reference'
+                        : 'recent',
+                ],
+        }
+      : item,
+  );
+}
+export function eligibleSelectionItems(normalized) {
+  return normalized.filter(
+    (item) => !item.protected && (item.kind === 'assistant_message' || item.kind === 'tool_result'),
+  );
+}
+export function selectionQuestion(item) {
+  return {
+    type: 'noul',
+    instructions: `Should recovery item ${item.id} remain useful for continuing the approved goal? Keep explicit constraints and evidence when uncertain.`,
+  };
+}
+export function selectionQuestions(items) {
+  return Object.fromEntries(items.map((item) => [`item_${item.id}`, selectionQuestion(item)]));
+}
+function stateFor(projection, items) {
+  return freezeProviderState({
+    goal: projection.goal,
+    items: items.map((item) => ({
+      id: item.id,
+      kind: item.kind,
+      text: item.text,
+      local_chars: item.localChars,
+    })),
+  });
+}
+function completeBatches(batches, eligible, stateTokens, maxRequestTokens) {
+  const expected = new Set(eligible.map((item) => item.id));
+  const seen = new Set();
+  for (const batch of batches) {
+    for (const item of batch) {
+      if (!expected.has(item.id) || seen.has(item.id)) return null;
+      seen.add(item.id);
+    }
+  }
+  if (seen.size !== expected.size) return null;
+  const requestTokens = Math.max(
+    ...batches.map(
+      (batch) =>
+        stateTokens +
+        SELECTION_FIT_OVERHEAD_TOKENS +
+        estimateSelectionTokens(selectionQuestions(batch)),
+    ),
+  );
+  return requestTokens <= maxRequestTokens ? { requestTokens } : null;
+}
+export function preflightSelectionState(input) {
+  const normalized = normalizeSelectionItems(input.projection, input.options.preserveRecentItems);
+  const eligible = eligibleSelectionItems(normalized);
+  const originalState = stateFor(input.projection, normalized);
+  const beforeStateTokens = estimateSelectionTokens(originalState);
+  if (eligible.length === 0)
+    return {
+      ok: true,
+      normalized,
+      eligible,
+      state: originalState,
+      batches: [],
+      fitting: {
+        version: 'state-fit-v1',
+        stage: 'original',
+        estimator: SELECTION_FIT_ESTIMATOR,
+        beforeEstimatedStateTokens: beforeStateTokens,
+        afterEstimatedStateTokens: beforeStateTokens,
+      },
+      beforeStateTokens,
+      stateTokens: beforeStateTokens,
+      requestTokens: 0,
+    };
+  const stages = [
+    { stage: 'original', items: normalized },
+    {
+      stage: 'text_head_tail',
+      items: normalized.map((item) =>
+        eligible.some((candidate) => candidate.id === item.id)
+          ? { ...item, text: marker(item.text, 300) }
+          : item,
+      ),
+    },
+    {
+      stage: 'text_omitted',
+      items: normalized.map((item) =>
+        eligible.some((candidate) => candidate.id === item.id)
+          ? { ...item, text: `[item omitted; ${item.text.length} chars]` }
+          : item,
+      ),
+    },
+  ];
+  for (const candidate of stages) {
+    const state = stateFor(input.projection, candidate.items);
+    const stateTokens = estimateSelectionTokens(state);
+    if (stateTokens > input.options.maxStateTokens) continue;
+    const available = input.options.maxRequestTokens - stateTokens - SELECTION_FIT_OVERHEAD_TOKENS;
+    if (available <= 0) continue;
+    const batches = [];
+    let current = [];
+    let currentTokens = 0;
+    let complete = true;
+    for (const item of eligible) {
+      const itemTokens = estimateSelectionTokens(selectionQuestions([item]));
+      if (itemTokens > available) {
+        complete = false;
+        break;
+      }
+      if (current.length > 0 && currentTokens + itemTokens > available) {
+        batches.push(current);
+        current = [];
+        currentTokens = 0;
+      }
+      current.push(item);
+      currentTokens += itemTokens;
+    }
+    if (current.length > 0) batches.push(current);
+    if (!complete || batches.length === 0 || batches.length > SELECTION_FIT_MAX_BATCHES) continue;
+    const checked = completeBatches(batches, eligible, stateTokens, input.options.maxRequestTokens);
+    if (checked === null) continue;
+    return {
+      ok: true,
+      normalized,
+      eligible,
+      state,
+      batches,
+      fitting: {
+        version: 'state-fit-v1',
+        stage: candidate.stage,
+        estimator: SELECTION_FIT_ESTIMATOR,
+        beforeEstimatedStateTokens: beforeStateTokens,
+        afterEstimatedStateTokens: stateTokens,
+      },
+      beforeStateTokens,
+      stateTokens,
+      requestTokens: checked.requestTokens,
+    };
+  }
+  return { ok: false, reason: 'state_budget_exceeded' };
+}

package/dist/assets/context-state-fitting.mjs

@@ -0,0 +1,324 @@
+export const UTF8_ESTIMATOR_VERSION = 'utf8-bytes-v1';
+export const FIT_OVERHEAD_TOKENS = 256;
+export const FIT_MAX_BATCHES = 4;
+const textEncoder = typeof TextEncoder === 'function' ? new TextEncoder() : undefined;
+function utf8ByteLength(value) {
+  if (textEncoder !== undefined) return textEncoder.encode(value).byteLength;
+  const buffer = globalThis.Buffer;
+  if (buffer !== undefined) return buffer.byteLength(value, 'utf8');
+  throw new Error('utf8_encoder_unavailable');
+}
+export function estimateUtf8Tokens(value) {
+  return utf8ByteLength(JSON.stringify(value));
+}
+export function freezeProviderState(value) {
+  if (typeof value === 'object' && value !== null) {
+    Object.freeze(value);
+    for (const nested of Object.values(value)) freezeProviderState(nested);
+  }
+  return value;
+}
+function marker(text, head, tail = head) {
+  const codePoints = Array.from(text);
+  if (codePoints.length <= head + tail + 32) return text;
+  return `${codePoints.slice(0, head).join('')}…[omitted ${codePoints.length - head - tail} chars]…${codePoints.slice(-tail).join('')}`;
+}
+function takeCodePoints(text, count) {
+  return Array.from(text).slice(0, count).join('');
+}
+function codePointLength(text) {
+  return Array.from(text).length;
+}
+function stateFor(
+  projection,
+  pairs,
+  messages = projection.messages,
+  compactIds = new Set(),
+  omittedProtectedPairCount = 0,
+) {
+  return freezeProviderState({
+    goal: projection.goal,
+    messages: messages.map((message) => ({
+      id: message.id,
+      role: message.role,
+      text: message.text,
+      call_ids: [...message.callIds],
+      result_ids: [...message.resultIds],
+    })),
+    pairs: pairs.map((pair) =>
+      compactIds.has(pair.id)
+        ? {
+            id: pair.id,
+            compact_record: `${pair.id}|${pair.tool}|${takeCodePoints(pair.inputText, 12)}|result_chars=${pair.resultChars}|error=${pair.resultIsError}`,
+          }
+        : {
+            id: pair.id,
+            tool: pair.tool,
+            input_text: pair.inputText,
+            result_chars: pair.resultChars,
+            result_is_error: pair.resultIsError,
+          },
+    ),
+    protected_item_count: projection.protectedItemCount,
+    ...(omittedProtectedPairCount > 0
+      ? { omitted_protected_pair_count: omittedProtectedPairCount }
+      : {}),
+  });
+}
+export function compactionQuestions(pairs) {
+  return Object.fromEntries(
+    pairs.flatMap((pair) => [
+      [
+        `call_${pair.id}`,
+        {
+          type: 'noul',
+          instructions: `Should the call occurrence and input for pair ${pair.id} (${pair.tool}) remain useful for continuing the approved goal? Answer yes only when retaining the call is needed.`,
+        },
+      ],
+      [
+        `result_${pair.id}`,
+        {
+          type: 'noul',
+          instructions: `Should the full result for pair ${pair.id} (${pair.resultChars} characters) remain useful for continuing the approved goal? Answer yes only when the result itself is needed.`,
+        },
+      ],
+    ]),
+  );
+}
+export function estimateCompactionQuestions(pairs) {
+  return estimateUtf8Tokens(compactionQuestions(pairs));
+}
+function makeBatches(eligible, stateTokens, maxRequestTokens) {
+  const available = maxRequestTokens - stateTokens - FIT_OVERHEAD_TOKENS;
+  if (available <= 0) return null;
+  const batches = [];
+  let current = [];
+  let currentTokens = 0;
+  for (const pair of eligible) {
+    const pairTokens = estimateCompactionQuestions([pair]);
+    if (pairTokens > available) return null;
+    if (current.length > 0 && currentTokens + pairTokens > available) {
+      batches.push(current);
+      current = [];
+      currentTokens = 0;
+    }
+    current.push(pair);
+    currentTokens += pairTokens;
+  }
+  if (current.length > 0) batches.push(current);
+  if (batches.length > FIT_MAX_BATCHES) return null;
+  return {
+    batches,
+    requestTokens: batches.reduce(
+      (max, batch) =>
+        Math.max(max, stateTokens + FIT_OVERHEAD_TOKENS + estimateCompactionQuestions(batch)),
+      0,
+    ),
+  };
+}
+function isPinnedMessage(projection, messageIndex, recent, protectedIds, pinnedMessageIds) {
+  if (pinnedMessageIds.has(projection.messages[messageIndex]?.id ?? '')) return true;
+  return (
+    projection.messages[messageIndex]?.callIds.some((id) => protectedIds.has(id)) === true ||
+    projection.messages[messageIndex]?.resultIds.some((id) => protectedIds.has(id)) === true
+  );
+}
+function stageProjection(projection, eligibleIds, recent, stage, pinnedMessageIds) {
+  const protectedIds = new Set(
+    projection.pairs.filter((pair) => !eligibleIds.has(pair.id)).map((pair) => pair.id),
+  );
+  const pairs = projection.pairs.map((pair) => {
+    if (!eligibleIds.has(pair.id)) return pair;
+    if (stage === 'inputs_1000')
+      return { ...pair, inputText: takeCodePoints(pair.inputText, 1_000) };
+    if (stage === 'inputs_200') return { ...pair, inputText: takeCodePoints(pair.inputText, 200) };
+    if (stage === 'inputs_60') return { ...pair, inputText: takeCodePoints(pair.inputText, 60) };
+    if (stage === 'text_head_tail') return { ...pair, inputText: marker(pair.inputText, 30, 30) };
+    if (stage === 'text_omitted')
+      return codePointLength(pair.inputText) > 60
+        ? { ...pair, inputText: `[input omitted; ${codePointLength(pair.inputText)} chars]` }
+        : pair;
+    if (stage === 'compact_calls')
+      return { ...pair, inputText: `${pair.tool}(${marker(pair.inputText, 24, 12)})` };
+    return pair;
+  });
+  let messages = projection.messages;
+  if (stage === 'text_head_tail') {
+    messages = projection.messages.map((message, index) =>
+      isPinnedMessage(projection, index, recent, protectedIds, pinnedMessageIds)
+        ? message
+        : { ...message, text: marker(message.text, 80, 40) },
+    );
+  } else if (stage === 'text_omitted') {
+    messages = projection.messages.map((message, index) =>
+      isPinnedMessage(projection, index, recent, protectedIds, pinnedMessageIds)
+        ? message
+        : codePointLength(message.text) > 120
+          ? { ...message, text: `[message omitted; ${codePointLength(message.text)} chars]` }
+          : message,
+    );
+  } else if (stage === 'compact_calls') {
+    messages = projection.messages.map((message, index) =>
+      // At this stage pair identity and question binding live in `pairs`. Keep
+      // protected message wiring available while abbreviating only its text.
+      // A later fitting stage folds this wiring only if the complete state still
+      // cannot fit, so ordinary histories retain more useful provider context.
+      isPinnedMessage(projection, index, recent, protectedIds, pinnedMessageIds)
+        ? message
+        : {
+            ...message,
+            text: `call:${message.callIds.join(',')};result:${message.resultIds.join(',')}`,
+          },
+    );
+  } else if (stage === 'callless_omitted') {
+    messages = projection.messages.filter(
+      (message, index) =>
+        isPinnedMessage(projection, index, recent, protectedIds, pinnedMessageIds) ||
+        message.callIds.length > 0 ||
+        message.resultIds.length > 0,
+    );
+  } else if (stage === 'runs_folded') {
+    const folded = [];
+    const usedIds = new Set(projection.messages.map((message) => message.id));
+    let markerSequence = 0;
+    let omittedRun;
+    let omittedCount = 0;
+    const flushOmitted = () => {
+      if (omittedRun === undefined) return;
+      let id = 'provider-history-omitted';
+      while (usedIds.has(id)) id = `provider-history-omitted-${++markerSequence}`;
+      usedIds.add(id);
+      folded.push({
+        id,
+        role: omittedRun.role,
+        text: `[${omittedCount} historical messages omitted from provider metadata]`,
+        callIds: [],
+        resultIds: [],
+      });
+      omittedRun = undefined;
+      omittedCount = 0;
+    };
+    for (let index = 0; index < projection.messages.length; index += 1) {
+      const message = projection.messages[index];
+      if (message === undefined) continue;
+      if (pinnedMessageIds.has(message.id)) {
+        flushOmitted();
+        folded.push(message);
+      } else {
+        omittedRun ??= message;
+        omittedCount += 1;
+      }
+    }
+    flushOmitted();
+    messages = folded;
+  }
+  return { pairs, messages };
+}
+export function fitProviderState(projection, eligible, options) {
+  const beforeState = stateFor(projection, projection.pairs);
+  const beforeTokens = estimateUtf8Tokens(beforeState);
+  const eligibleIds = new Set(eligible.map((pair) => pair.id));
+  const pinnedMessageIds = new Set(
+    projection.messages
+      .slice(0, 1)
+      .concat(
+        projection.messages.slice(
+          Math.max(0, projection.messages.length - options.preserveRecentMessages),
+        ),
+      )
+      .map((message) => message.id),
+  );
+  const stages = [
+    'original',
+    'inputs_1000',
+    'inputs_200',
+    'inputs_60',
+    'text_head_tail',
+    'text_omitted',
+    'compact_calls',
+    'callless_omitted',
+    'runs_folded',
+  ];
+  let current = { pairs: projection.pairs, messages: projection.messages };
+  for (const [stageIndex, stage] of stages.entries()) {
+    if (stage !== 'original')
+      current = stageProjection(
+        { ...projection, pairs: current.pairs, messages: current.messages },
+        eligibleIds,
+        options.preserveRecentMessages,
+        stage,
+        pinnedMessageIds,
+      );
+    let transformed = current;
+    let omittedProtectedPairCount = 0;
+    if (stage === 'runs_folded') {
+      const retainedPairIds = new Set(
+        transformed.messages.flatMap((message) => [...message.callIds, ...message.resultIds]),
+      );
+      const retainedPairs = transformed.pairs.filter(
+        (pair) => eligibleIds.has(pair.id) || retainedPairIds.has(pair.id),
+      );
+      omittedProtectedPairCount = transformed.pairs.length - retainedPairs.length;
+      transformed = { ...transformed, pairs: retainedPairs };
+    }
+    // Once we reach the explicit compact-calls stage, abbreviate pair metadata
+    // while retaining identity, tool, size and outcome. Eligible inputs have
+    // already passed through bounded input stages, and the compact record keeps
+    // a short prefix for the provider's final question context.
+    const compactIds =
+      stageIndex >= stages.indexOf('compact_calls')
+        ? new Set(projection.pairs.map((pair) => pair.id))
+        : new Set();
+    const state = stateFor(
+      projection,
+      transformed.pairs,
+      transformed.messages,
+      compactIds,
+      omittedProtectedPairCount,
+    );
+    const stateTokens = estimateUtf8Tokens(state);
+    if (stateTokens > options.maxStateTokens) continue;
+    const packed = makeBatches(eligible, stateTokens, options.maxRequestTokens);
+    if (packed === null) continue;
+    return {
+      state,
+      fitting: {
+        version: 'state-fit-v1',
+        stage,
+        estimator: UTF8_ESTIMATOR_VERSION,
+        beforeEstimatedStateTokens: beforeTokens,
+        afterEstimatedStateTokens: stateTokens,
+      },
+      batches: packed.batches,
+      stateTokens,
+      requestTokens: packed.requestTokens,
+    };
+  }
+  throw new Error('state_fit_budget_exceeded');
+}
+/**
+ * Narrow, side-effect-free contract for the Claude provider-window planner.
+ * It intentionally does not expose the prepared provider state or transcript.
+ */
+export function preflightProviderState(projection, eligible, options) {
+  try {
+    const fitted = fitProviderState(projection, eligible, options);
+    return {
+      fits: true,
+      reason: 'fits',
+      stage: fitted.fitting.stage,
+      stateTokens: fitted.stateTokens,
+      requestTokens: fitted.requestTokens,
+      batchCount: fitted.batches.length,
+    };
+  } catch {
+    return {
+      fits: false,
+      reason: 'budget_exceeded',
+      stage: null,
+      stateTokens: null,
+      requestTokens: null,
+      batchCount: 0,
+    };
+  }
+}