@pi-unipi/background-tasks 2.6.3 → 2.9.0

package/src/__tests__/delegate-budget.test.ts

@@ -1,456 +0,0 @@
-import { describe, it } from 'node:test';
-import assert from 'node:assert/strict';
-import { readFileSync } from 'node:fs';
-import {
-  DELEGATE_FRAMING_RESERVE_TOKENS,
-  DELEGATE_INLINE_ANSWER_BYTES,
-  DELEGATE_MIN_CONTEXT_WINDOW_TOKENS,
-  DELEGATE_MIN_USABLE_INPUT_TOKENS,
-  DELEGATE_RESERVED_OUTPUT_TOKENS,
-  DELEGATE_SAFETY_RESERVE_TOKENS,
-  assertDelegateAdmission,
-  delegateAllowedInputTokens,
-  evaluateDelegateRuntimeBudget,
-  planDelegateAdmission,
-  resolveDelegateLimits,
-} from '../delegate/budget.js';
-import { DelegateError, type DelegatePinnedRoute } from '../delegate/types.js';
-import {
-  TOKEN_BUDGET_AFFINE_F_TOKENS,
-  TOKEN_BUDGET_FAMILY_CALIBRATIONS,
-  TOKEN_BUDGET_PROVABLE_RATE_X100,
-  estimateInputTokens,
-  maxKnownTextBytesForTokens,
-} from '../context-token-budget.js';
-
-function route(
-  contextWindow: number,
-  provider = 'openai-codex',
-  model = 'gpt-5.5',
-): DelegatePinnedRoute {
-  return {
-    provider,
-    model,
-    qualified_id: `${provider}/${model}`,
-    context_window_tokens: contextWindow,
-    thinking_level: 'medium',
-    origin: 'explicit',
-  };
-}
-
-const INCIDENT = JSON.parse(
-  readFileSync(new URL('../fixtures/delegate-context-incident.json', import.meta.url), 'utf8'),
-) as Record<string, unknown>;
-
-function incidentNumber(key: string): number {
-  const value = INCIDENT[key];
-  if (typeof value !== 'number') throw new Error(`incident fixture ${key} must be numeric`);
-  return value;
-}
-
-const LIMITS = {
-  max_turns: 24,
-  max_tool_calls: 120,
-  timeout_seconds: 900,
-  max_tool_result_bytes: 65_536,
-  max_total_tool_output_bytes: 67_108_864,
-  max_answer_bytes: 4_194_304,
-  allowed_input_tokens: 0,
-};
-
-function maxDelegateKnownBytes(pinnedRoute: DelegatePinnedRoute): number {
-  const allowed = delegateAllowedInputTokens(pinnedRoute);
-  const probe = planDelegateAdmission({
-    route: pinnedRoute,
-    childPrompt: 'x '.repeat(30_000),
-    childSystemPrompt: '',
-    limits: LIMITS,
-  });
-  const rate = probe.route.rate_source;
-  return Math.floor(
-    ((allowed - rate.affine_f_tokens) * rate.effective_rate_bytes_per_token_x100) /
-      100,
-  );
-}
-
-function normalText(bytes: number): string {
-  return `${'x '.repeat(Math.floor(bytes / 2))}${bytes % 2 === 0 ? '' : 'x'}`;
-}
-
-function delegateTokenForecast(bytes: number, allowedInputTokens: number): number {
-  return estimateInputTokens({
-    family: 'openai-codex',
-    allowedInputTokens,
-    scope: 'delegate_launch',
-    segments: [{ kind: 'known_text', bytes, multibyteBytes: 0, denseBytes: 0 }],
-  }).tokens;
-}
-
-void describe('delegate route capacity', () => {
-  void it('subtracts every documented reserve from the context window', () => {
-    const window = 200_000;
-    assert.equal(
-      delegateAllowedInputTokens(route(window)),
-      window -
-        DELEGATE_RESERVED_OUTPUT_TOKENS -
-        DELEGATE_FRAMING_RESERVE_TOKENS -
-        DELEGATE_SAFETY_RESERVE_TOKENS,
-    );
-  });
-
-  void it('accepts exactly the documented minimum window and rejects one token below', () => {
-    assert.equal(
-      delegateAllowedInputTokens(route(DELEGATE_MIN_CONTEXT_WINDOW_TOKENS)),
-      DELEGATE_MIN_USABLE_INPUT_TOKENS,
-    );
-    assert.throws(
-      () => delegateAllowedInputTokens(route(DELEGATE_MIN_CONTEXT_WINDOW_TOKENS - 1)),
-      (error: unknown) =>
-        error instanceof DelegateError &&
-        error.code === 'route_capacity_unknown' &&
-        error.childCreated === false,
-    );
-  });
-
-  void it('never assumes a default window for unusable capacity', () => {
-    for (const window of [0, -1, Number.NaN, Number.POSITIVE_INFINITY, 1.5]) {
-      assert.throws(
-        () => delegateAllowedInputTokens(route(window)),
-        (error: unknown) =>
-          error instanceof DelegateError && error.code === 'route_capacity_unknown',
-        `window ${String(window)} must be refused, not defaulted`,
-      );
-    }
-  });
-});
-
-void describe('delegate launch admission', () => {
-  void it('fits when the seed is comfortably under the route allowance', () => {
-    const pinnedRoute = route(200_000);
-    const plan = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt: 'x'.repeat(1000),
-      childSystemPrompt: 'y'.repeat(500),
-      limits: LIMITS,
-    });
-    assert.equal(plan.fits, true);
-    assert.equal(plan.schema_version, 'pi-background-tasks.delegate-budget-plan.v3');
-    assert.equal(plan.policy.id, 'delegate-budget-policy-v3');
-    assert.equal(plan.child_prompt_utf8_bytes, 1000);
-    assert.equal(plan.system_prompt_utf8_bytes, 500);
-    assert.equal(plan.launch_utf8_bytes, 1500);
-    assert.equal(plan.launch_input_tokens_upper_bound, plan.estimate.tokens);
-    assert.equal(plan.route.family, 'openai-codex');
-    assert.equal(plan.route.rate_source.source, 'delegate_conservative');
-    assert.doesNotThrow(() => {
-      assertDelegateAdmission(plan);
-    });
-  });
-
-  void it('accepts exactly at the affine boundary and rejects one byte past it', () => {
-    const pinnedRoute = route(200_000);
-    const exactBytes = maxDelegateKnownBytes(pinnedRoute);
-    const atLimit = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt: normalText(exactBytes),
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    assert.equal(atLimit.fits, true);
-    assert.equal(atLimit.signed_headroom_tokens, 0);
-    assert.doesNotThrow(() => {
-      assertDelegateAdmission(atLimit);
-    });
-
-    const overLimit = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt: normalText(exactBytes + 1),
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    assert.equal(overLimit.fits, false);
-    assert.throws(
-      () => {
-        assertDelegateAdmission(overLimit);
-      },
-      (error: unknown) =>
-        error instanceof DelegateError &&
-        error.code === 'seed_budget_exceeded' &&
-        error.childCreated === false,
-    );
-  });
-
-  void it('counts the child system prompt as input, not as free framing', () => {
-    const pinnedRoute = route(200_000);
-    const seedBytes = maxDelegateKnownBytes(pinnedRoute);
-    const plan = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt: normalText(seedBytes),
-      childSystemPrompt: 'y ',
-      limits: LIMITS,
-    });
-    assert.equal(plan.fits, false, 'the system prompt must consume the same allowance');
-  });
-
-  void it('measures multi-byte UTF-8 by bytes and persists the provable advisory ceiling', () => {
-    const emoji = '👩‍👩‍👧‍👦';
-    const plan = planDelegateAdmission({
-      route: route(200_000),
-      childPrompt: emoji.repeat(100),
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    assert.equal(plan.child_prompt_utf8_bytes, Buffer.byteLength(emoji.repeat(100), 'utf8'));
-    assert.ok(plan.child_prompt_utf8_bytes > emoji.repeat(100).length);
-    assert.equal(plan.estimate.byte_class_breakdown.multibyte_bytes, plan.child_prompt_utf8_bytes);
-    assert.equal(plan.estimate.perSegment[0]?.multibyte_provable_tokens, plan.child_prompt_utf8_bytes);
-    assert.ok((plan.estimate.perSegment[0]?.multibyte_tokens ?? 0) < plan.child_prompt_utf8_bytes);
-    assert.equal(plan.dominant_byte_class, 'multibyte');
-  });
-
-  void it('scope guard refuses the codex 23,674-byte delegate floor case', () => {
-    const pinnedRoute = route(DELEGATE_MIN_CONTEXT_WINDOW_TOKENS, 'openai-codex', 'gpt-5.5');
-    assert.equal(delegateAllowedInputTokens(pinnedRoute), 8192);
-    const plan = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt: 'x'.repeat(23_674),
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    assert.equal(plan.route.rate_source.source, 'delegate_conservative');
-    assert.equal(plan.fits, false);
-    assert.throws(
-      () => assertDelegateAdmission(plan),
-      (error: unknown) => error instanceof DelegateError && error.code === 'seed_budget_exceeded',
-    );
-  });
-
-  void it('scope guard refuses the anthropic 13,286-byte delegate zero-headroom case', () => {
-    const pinnedRoute = route(DELEGATE_MIN_CONTEXT_WINDOW_TOKENS, 'anthropic', 'claude-opus-5');
-    assert.equal(delegateAllowedInputTokens(pinnedRoute), 8192);
-    const beforeAnthropicForecast = estimateInputTokens({
-      family: 'anthropic',
-      allowedInputTokens: 8192,
-      scope: 'fusion',
-      segments: [{ kind: 'known_text', bytes: 13_286, multibyteBytes: 0, denseBytes: 0 }],
-    }).tokens;
-    assert.equal(beforeAnthropicForecast, 8_192);
-    const plan = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt: 'x'.repeat(13_286),
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    assert.equal(plan.route.rate_source.source, 'delegate_conservative');
-    assert.equal(plan.route.rate_source.backed, true);
-    assert.equal(plan.route.rate_source.effective_rate_bytes_per_token_x100, TOKEN_BUDGET_PROVABLE_RATE_X100);
-    assert.equal(plan.launch_input_tokens_upper_bound, 13_798);
-    assert.equal(plan.fits, false);
-    assert.throws(
-      () => assertDelegateAdmission(plan),
-      (error: unknown) => {
-        assert.ok(error instanceof DelegateError);
-        assert.equal(error.code, 'seed_budget_exceeded');
-        assert.equal(error.budget?.rate_source.source, 'delegate_conservative');
-        assert.equal(error.budget?.backed, true);
-        assert.equal(error.budget?.dominant_byte_class, 'normal');
-        assert.match(error.message, /backed=true/);
-        assert.match(error.message, /dominant_byte_class=normal/);
-        return true;
-      },
-    );
-  });
-
-  void it('replays the incident byte-class contract with calibrated runway and a provable counter-forecast', () => {
-    const promptBytes = incidentNumber('child_prompt_utf8_bytes');
-    const multibyteBytes = incidentNumber('launch_multibyte_bytes');
-    assert.equal(multibyteBytes % Buffer.byteLength('€', 'utf8'), 0);
-    const childPrompt = `${normalText(promptBytes - multibyteBytes)}${'€'.repeat(multibyteBytes / 3)}`;
-    const childSystemPrompt = normalText(incidentNumber('system_prompt_utf8_bytes'));
-    assert.equal(Buffer.byteLength(childPrompt, 'utf8'), promptBytes);
-    const pinnedRoute = route(
-      incidentNumber('context_window_tokens'),
-      'openai-codex',
-      'gpt-5.6-sol',
-    );
-    const plan = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt,
-      childSystemPrompt,
-      limits: LIMITS,
-    });
-    assert.equal(plan.launch_utf8_bytes, incidentNumber('launch_utf8_bytes'));
-    assert.equal(plan.byte_class_breakdown.normal_bytes, incidentNumber('launch_normal_bytes'));
-    assert.equal(plan.byte_class_breakdown.multibyte_bytes, multibyteBytes);
-    assert.equal(plan.route.allowed_input_tokens, incidentNumber('allowed_input_tokens'));
-    assert.equal(plan.route.rate_source.source, 'calibrated_large_window');
-    assert.equal(
-      plan.route.rate_source.effective_rate_bytes_per_token_x100,
-      incidentNumber('calibrated_rate_bytes_per_token_x100'),
-    );
-    assert.equal(plan.fits, true);
-    assert.ok(plan.launch_input_tokens_upper_bound < 70_000);
-    assert.equal(
-      plan.conservative_estimate.tokens,
-      incidentNumber('old_one_byte_normal_two_byte_multibyte_forecast'),
-    );
-    assert.equal(
-      plan.conservative_launch_input_tokens,
-      incidentNumber('provable_one_byte_per_token_forecast'),
-    );
-    assert.equal(plan.conservative_launch_fits, true);
-    assert.ok(plan.retained_growth_budget_tokens > 240_000);
-    assert.equal(
-      plan.finalization_input_reserve_tokens,
-      incidentNumber('finalization_reserve_tokens'),
-    );
-  });
-
-  void it('publishes the provable ceiling for multibyte-heavy conservative forecasts', () => {
-    const pinnedRoute = route(128_672, 'openai-codex', 'gpt-5.6-sol');
-    const plan = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt: '€'.repeat(40_000),
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    assert.equal(plan.route.allowed_input_tokens, 100_000);
-    assert.ok(
-      plan.conservative_estimate.tokens < plan.route.allowed_input_tokens,
-      'the legacy 2 B/token multibyte diagnostic demonstrates the original false fit',
-    );
-    assert.equal(plan.conservative_launch_input_tokens, 120_512);
-    assert.equal(plan.conservative_launch_fits, false);
-  });
-
-  void it('unknown family is loud and no looser than calibrated families', () => {
-    const unknownRoute = route(200_000, 'mystery-provider', 'mystery-model');
-    const plan = planDelegateAdmission({
-      route: unknownRoute,
-      childPrompt: 'x',
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    const calibratedRates = Object.values(TOKEN_BUDGET_FAMILY_CALIBRATIONS)
-      .filter((entry) => entry.provenance.backed)
-      .map((entry) => entry.rate_bytes_per_token_x100);
-    assert.equal(plan.route.family, 'unknown');
-    assert.equal(plan.route.rate_source.backed, false);
-    assert.ok(plan.route.rate_source.warning?.includes('unknown provider'));
-    assert.ok(
-      plan.route.rate_source.effective_rate_bytes_per_token_x100 <= Math.min(...calibratedRates),
-    );
-    const oversized = planDelegateAdmission({
-      route: unknownRoute,
-      childPrompt: 'x'.repeat(delegateAllowedInputTokens(unknownRoute)),
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    assert.throws(
-      () => assertDelegateAdmission(oversized),
-      (error: unknown) => {
-        assert.ok(error instanceof DelegateError);
-        assert.match(error.message, /unknown provider/);
-        return true;
-      },
-    );
-  });
-
-  void it('states what was preserved and how to remediate, and never clamps', () => {
-    const pinnedRoute = route(200_000);
-    const plan = planDelegateAdmission({
-      route: pinnedRoute,
-      childPrompt: 'x'.repeat(maxDelegateKnownBytes(pinnedRoute) * 2),
-      childSystemPrompt: '',
-      limits: LIMITS,
-    });
-    try {
-      assertDelegateAdmission(plan);
-      assert.fail('oversized admission must throw');
-    } catch (error) {
-      assert.ok(error instanceof DelegateError);
-      assert.equal(error.code, 'seed_budget_exceeded');
-      assert.equal(error.childCreated, false);
-      assert.match(error.message, /No child process, child session, or artifact was created/);
-      assert.match(error.message, /Nothing was clipped, dropped, or substituted/);
-      assert.ok(error.remediation.length > 0);
-      assert.match(error.describe(), /Child process created: no/);
-    }
-  });
-});
-
-void describe('delegate runtime governor', () => {
-  void it('is total and never throws from inside a hook', () => {
-    const pinnedRoute = route(200_000);
-    for (const bytes of [0, 1, 10_000, 10_000_000]) {
-      const verdict = evaluateDelegateRuntimeBudget(
-        { retainedInputBytes: bytes, retainedInputMultibyteBytes: 0, retainedInputDenseBytes: 0 },
-        5_000,
-        pinnedRoute,
-      );
-      assert.equal(verdict.measuredTokens, delegateTokenForecast(bytes, 5_000));
-      assert.equal(verdict.allowedTokens, 5_000);
-    }
-  });
-
-  void it('permits exactly the allowance and refuses one token past it', () => {
-    const allowed = 1_000;
-    const pinnedRoute = route(allowed + 28_672);
-    const exactBytes = maxKnownTextBytesForTokens({
-      family: 'openai-codex',
-      allowedInputTokens: allowed,
-      scope: 'delegate_launch',
-    });
-    const exact = evaluateDelegateRuntimeBudget(
-      {
-        retainedInputBytes: exactBytes,
-        retainedInputMultibyteBytes: 0,
-        retainedInputDenseBytes: 0,
-      },
-      allowed,
-      pinnedRoute,
-    );
-    assert.equal(exact.withinBudget, true);
-    assert.equal(exact.overageTokens, 0);
-
-    const over = evaluateDelegateRuntimeBudget(
-      {
-        retainedInputBytes: exactBytes + 1,
-        retainedInputMultibyteBytes: 0,
-        retainedInputDenseBytes: 0,
-      },
-      allowed,
-      pinnedRoute,
-    );
-    assert.equal(over.withinBudget, false);
-    assert.equal(over.overageTokens, 1);
-  });
-});
-
-void describe('delegate limits', () => {
-  void it('applies documented defaults and derives the route allowance', () => {
-    const limits = resolveDelegateLimits(route(200_000));
-    assert.equal(limits.max_turns, 24);
-    assert.equal(limits.max_tool_calls, 120);
-    assert.equal(limits.timeout_seconds, 1200);
-    assert.equal(limits.allowed_input_tokens, delegateAllowedInputTokens(route(200_000)));
-  });
-
-  void it('rejects non-positive and non-integer overrides loudly', () => {
-    for (const value of [0, -1, 1.5, Number.NaN]) {
-      assert.throws(
-        () => resolveDelegateLimits(route(200_000), { maxTurns: value }),
-        (error: unknown) => error instanceof DelegateError && error.code === 'invalid_arguments',
-      );
-    }
-  });
-
-  void it('keeps the inline answer cap strictly below the answer capture cap', () => {
-    const limits = resolveDelegateLimits(route(200_000));
-    assert.ok(
-      DELEGATE_INLINE_ANSWER_BYTES < limits.max_answer_bytes,
-      'an answer must be capturable even when it is too large to inline',
-    );
-    assert.equal(TOKEN_BUDGET_AFFINE_F_TOKENS, 512);
-  });
-});