deepline 0.1.182 → 0.1.183

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -0,0 +1,343 @@
+export const PLAY_RUNTIME_TEST_FAULT_HEADER = 'x-deepline-test-fault';
+
+export type RuntimeTestFaultName =
+  | 'receipt_complete_write_fail'
+  | 'receipt_fail_write_fail'
+  | 'worker_receipt_complete_write_fail';
+
+export type RuntimeTestFaultRegistry = {
+  consume(name: RuntimeTestFaultName): boolean;
+  remaining(): Record<string, number>;
+};
+
+type ParsedRuntimeTestFaults =
+  | { ok: true; registry: RuntimeTestFaultRegistry | null }
+  | { ok: false; status: 400 | 403; error: string };
+
+type RuntimeTestSeamContext = {
+  internalTokenHeader?: string | null;
+  syntheticRunHeader?: string | null;
+};
+
+type ParsedRuntimeTestFaultCounts =
+  | {
+      ok: true;
+      counts: Record<string, number> | null;
+      headerValue: string | null;
+    }
+  | { ok: false; status: 400 | 403; error: string };
+
+type ValidatedRuntimeTestFaultHeader =
+  | { ok: true; headerValue: string | null }
+  | { ok: false; status: 400 | 403; error: string };
+
+export type RuntimeTestPolicyOverrides = {
+  receiptLeaseTtlMs?: number;
+  sheetAttemptLeaseMs?: number;
+  heartbeatIntervalMs?: number;
+  workBudgetYieldLimits?: Partial<
+    Record<
+      | 'elapsed'
+      | 'subrequest'
+      | 'provider'
+      | 'tool'
+      | 'receipt'
+      | 'sheet'
+      | 'log'
+      | 'egress',
+      number
+    >
+  >;
+  batchGraceMs?: number;
+};
+
+type ValidatedRuntimeTestPolicyOverrides =
+  | { ok: true; overrides: RuntimeTestPolicyOverrides | null }
+  | { ok: false; status: 400 | 403; error: string };
+
+const SUPPORTED_RUNTIME_TEST_FAULTS = new Set<RuntimeTestFaultName>([
+  'receipt_complete_write_fail',
+  'receipt_fail_write_fail',
+  'worker_receipt_complete_write_fail',
+]);
+
+const RUNTIME_TEST_POLICY_MS_FIELDS = new Set([
+  'receiptLeaseTtlMs',
+  'sheetAttemptLeaseMs',
+  'heartbeatIntervalMs',
+  'batchGraceMs',
+]);
+
+const RUNTIME_TEST_POLICY_WORK_BUDGET_KEYS = new Set([
+  'elapsed',
+  'subrequest',
+  'provider',
+  'tool',
+  'receipt',
+  'sheet',
+  'log',
+  'egress',
+]);
+
+const MAX_RUNTIME_TEST_POLICY_MS = 10 * 60_000;
+const MAX_RUNTIME_TEST_POLICY_YIELD_LIMIT = 1_000_000;
+
+function isProductionRuntime(): boolean {
+  const deeplineEnv = process.env.DEEPLINE_ENV?.trim().toLowerCase();
+  if (deeplineEnv === 'production' || deeplineEnv === 'prod') return true;
+  const vercelEnv = process.env.VERCEL_ENV?.trim().toLowerCase();
+  if (vercelEnv === 'production') return true;
+  if (vercelEnv === 'preview' || vercelEnv === 'development') return false;
+  return process.env.NODE_ENV === 'production';
+}
+
+function testRuntimeSeamsEnabled(): boolean {
+  return process.env.DEEPLINE_TEST_RUNTIME_SEAMS === '1';
+}
+
+function internalTokenAllowsRuntimeSeams(
+  context: RuntimeTestSeamContext | undefined,
+): boolean {
+  const deeplineEnv = process.env.DEEPLINE_ENV?.trim().toLowerCase();
+  if (deeplineEnv === 'prod') return false;
+  if (deeplineEnv === 'production') return false;
+  const expected = process.env.DEEPLINE_INTERNAL_TOKEN?.trim();
+  const actual = context?.internalTokenHeader?.trim();
+  return Boolean(expected && actual && expected === actual);
+}
+
+function runtimeTestSeamsAuthorized(
+  context: RuntimeTestSeamContext | undefined,
+): boolean {
+  return (
+    testRuntimeSeamsEnabled() ||
+    internalTokenAllowsRuntimeSeams(context) ||
+    Boolean(
+      context?.syntheticRunHeader?.trim() &&
+      context.syntheticRunHeader.trim() !== '0',
+    )
+  );
+}
+
+function parseRuntimeTestFaultHeader(
+  rawHeader: string,
+): Map<RuntimeTestFaultName, number> | { error: string } {
+  const faults = new Map<RuntimeTestFaultName, number>();
+  for (const rawPart of rawHeader.split(',')) {
+    const part = rawPart.trim();
+    if (!part) continue;
+    const [rawName, rawCount] = part.split(':');
+    const name = rawName?.trim() as RuntimeTestFaultName | undefined;
+    if (!name || !SUPPORTED_RUNTIME_TEST_FAULTS.has(name)) {
+      return { error: `Unsupported runtime test fault "${rawName ?? part}".` };
+    }
+    const count =
+      rawCount === undefined || rawCount.trim() === ''
+        ? 1
+        : Number.parseInt(rawCount.trim(), 10);
+    if (!Number.isInteger(count) || count < 1) {
+      return {
+        error: `Runtime test fault "${name}" count must be a positive integer.`,
+      };
+    }
+    faults.set(name, (faults.get(name) ?? 0) + count);
+  }
+  return faults;
+}
+
+function readPositiveIntegerField(input: {
+  value: unknown;
+  path: string;
+  max: number;
+}): number | { error: string } {
+  if (
+    typeof input.value !== 'number' ||
+    !Number.isFinite(input.value) ||
+    !Number.isInteger(input.value) ||
+    input.value < 1 ||
+    input.value > input.max
+  ) {
+    return {
+      error: `${input.path} must be an integer between 1 and ${input.max}.`,
+    };
+  }
+  return input.value;
+}
+
+function parseRuntimeTestPolicyOverrides(
+  value: unknown,
+): RuntimeTestPolicyOverrides | { error: string } | null {
+  if (value === undefined || value === null) {
+    return null;
+  }
+  if (typeof value !== 'object' || Array.isArray(value)) {
+    return { error: 'testPolicyOverrides must be a JSON object.' };
+  }
+  const record = value as Record<string, unknown>;
+  const unknownKeys = Object.keys(record).filter(
+    (key) =>
+      !RUNTIME_TEST_POLICY_MS_FIELDS.has(key) &&
+      key !== 'workBudgetYieldLimits',
+  );
+  if (unknownKeys.length > 0) {
+    return {
+      error: `Unsupported testPolicyOverrides field "${unknownKeys[0]}".`,
+    };
+  }
+
+  const overrides: RuntimeTestPolicyOverrides = {};
+  for (const field of RUNTIME_TEST_POLICY_MS_FIELDS) {
+    if (!(field in record)) continue;
+    const parsed = readPositiveIntegerField({
+      value: record[field],
+      path: `testPolicyOverrides.${field}`,
+      max: MAX_RUNTIME_TEST_POLICY_MS,
+    });
+    if (typeof parsed !== 'number') return parsed;
+    (overrides as Record<string, number>)[field] = parsed;
+  }
+
+  if ('workBudgetYieldLimits' in record) {
+    const rawLimits = record.workBudgetYieldLimits;
+    if (
+      rawLimits === null ||
+      typeof rawLimits !== 'object' ||
+      Array.isArray(rawLimits)
+    ) {
+      return {
+        error: 'testPolicyOverrides.workBudgetYieldLimits must be an object.',
+      };
+    }
+    const limits: NonNullable<
+      RuntimeTestPolicyOverrides['workBudgetYieldLimits']
+    > = {};
+    for (const [key, rawLimit] of Object.entries(
+      rawLimits as Record<string, unknown>,
+    )) {
+      if (!RUNTIME_TEST_POLICY_WORK_BUDGET_KEYS.has(key)) {
+        return {
+          error: `Unsupported testPolicyOverrides.workBudgetYieldLimits field "${key}".`,
+        };
+      }
+      const parsed = readPositiveIntegerField({
+        value: rawLimit,
+        path: `testPolicyOverrides.workBudgetYieldLimits.${key}`,
+        max: MAX_RUNTIME_TEST_POLICY_YIELD_LIMIT,
+      });
+      if (typeof parsed !== 'number') return parsed;
+      limits[key as keyof typeof limits] = parsed;
+    }
+    if (Object.keys(limits).length > 0) {
+      overrides.workBudgetYieldLimits = limits;
+    }
+  }
+
+  return Object.keys(overrides).length > 0 ? overrides : null;
+}
+
+export function validateRuntimeTestPolicyOverrides(input: {
+  value: unknown;
+  internalTokenHeader?: string | null;
+  syntheticRunHeader?: string | null;
+}): ValidatedRuntimeTestPolicyOverrides {
+  if (input.value === undefined || input.value === null) {
+    return { ok: true, overrides: null };
+  }
+  if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
+    return {
+      ok: false,
+      status: 403,
+      error:
+        'Runtime test policy overrides are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.',
+    };
+  }
+  const parsed = parseRuntimeTestPolicyOverrides(input.value);
+  if (parsed === null) {
+    return { ok: true, overrides: null };
+  }
+  if ('error' in parsed) {
+    return { ok: false, status: 400, error: parsed.error };
+  }
+  return { ok: true, overrides: parsed };
+}
+
+export function readRuntimeTestFaultRegistry(input: {
+  headerValue: string | null;
+  internalTokenHeader?: string | null;
+  syntheticRunHeader?: string | null;
+}): ParsedRuntimeTestFaults {
+  const headerValue = input.headerValue?.trim();
+  if (!headerValue) {
+    return { ok: true, registry: null };
+  }
+  if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
+    return {
+      ok: false,
+      status: 403,
+      error:
+        'Runtime test seams are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.',
+    };
+  }
+  const parsed = parseRuntimeTestFaultHeader(headerValue);
+  if ('error' in parsed) {
+    return { ok: false, status: 400, error: parsed.error };
+  }
+  return {
+    ok: true,
+    registry: {
+      consume(name) {
+        const count = parsed.get(name) ?? 0;
+        if (count <= 0) return false;
+        if (count === 1) {
+          parsed.delete(name);
+        } else {
+          parsed.set(name, count - 1);
+        }
+        return true;
+      },
+      remaining() {
+        return Object.fromEntries(parsed.entries());
+      },
+    },
+  };
+}
+
+export function validateRuntimeTestFaultHeader(input: {
+  headerValue: string | null;
+  internalTokenHeader?: string | null;
+  syntheticRunHeader?: string | null;
+}): ValidatedRuntimeTestFaultHeader {
+  const parsed = parseRuntimeTestFaultCounts(input);
+  if (parsed.ok === false) {
+    return { ok: false, status: parsed.status, error: parsed.error };
+  }
+  return { ok: true, headerValue: parsed.headerValue };
+}
+
+export function parseRuntimeTestFaultCounts(input: {
+  headerValue: string | null;
+  internalTokenHeader?: string | null;
+  syntheticRunHeader?: string | null;
+}): ParsedRuntimeTestFaultCounts {
+  const headerValue = input.headerValue?.trim();
+  if (!headerValue) {
+    return { ok: true, counts: null, headerValue: null };
+  }
+  if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
+    return {
+      ok: false,
+      status: 403,
+      error:
+        'Runtime test seams are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.',
+    };
+  }
+  const parsed = parseRuntimeTestFaultHeader(headerValue);
+  if ('error' in parsed) {
+    return { ok: false, status: 400, error: parsed.error };
+  }
+  return {
+    ok: true,
+    counts: Object.fromEntries(parsed.entries()),
+    headerValue,
+  };
+}

package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts

@@ -12,6 +12,17 @@ export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000;
 export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
 export const TOOL_EXECUTE_RETRY_DELAY_MAX_MS = 5_000;
 export const TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS = 60_000;
+export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED';
+
+export class ToolExecuteAuthScopeChangedError extends Error {
+  readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE;
+  readonly status = 409;
+
+  constructor(message = 'Tool execution auth scope changed.') {
+    super(message);
+    this.name = 'ToolExecuteAuthScopeChangedError';
+  }
+}
 
 export type ToolExecuteHttpRetryDecision = {
   retryable: boolean;
@@ -42,6 +53,35 @@ export type ToolExecuteHttpFailureAttemptTracker = {
   }): number;
 };
 
+function parseJsonObject(text: string): Record<string, unknown> | null {
+  if (!text.trim()) return null;
+  try {
+    const parsed = JSON.parse(text) as unknown;
+    return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
+      ? (parsed as Record<string, unknown>)
+      : null;
+  } catch {
+    return null;
+  }
+}
+
+export function parseToolExecuteAuthScopeChangedError(input: {
+  status: number;
+  bodyText: string;
+}): ToolExecuteAuthScopeChangedError | null {
+  if (input.status !== 409) return null;
+  const body = parseJsonObject(input.bodyText);
+  const code = typeof body?.code === 'string' ? body.code.trim() : '';
+  if (code !== TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE) return null;
+  const message =
+    typeof body?.error === 'string' && body.error.trim()
+      ? body.error.trim()
+      : typeof body?.message === 'string' && body.message.trim()
+        ? body.message.trim()
+        : undefined;
+  return new ToolExecuteAuthScopeChangedError(message);
+}
+
 function decideToolExecuteHttpRetry(input: {
   status: number;
   hardBillingFailure?: boolean;
@@ -164,6 +204,9 @@ export function classifyToolExecuteHttpFailure(input: {
   });
   const shouldRetry =
     retryDecision.retryable && input.attempt < retryDecision.attemptCap;
+  error.receiptFailureKind = retryDecision.retryable
+    ? 'repairable'
+    : 'terminal';
   const retryAfterMs = parseToolExecuteRetryAfterMs(
     input.retryAfterHeader,
     input.nowMs,

package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts

@@ -1,17 +1,22 @@
+import type { WorkReceiptFailureKind } from './work-receipts';
+
 export class ToolHttpError extends Error {
   readonly billing: Record<string, unknown> | null;
   /** HTTP status of the failed tool-execute response (e.g. 429, 502). */
   readonly status: number;
+  receiptFailureKind: WorkReceiptFailureKind | null;
 
   constructor(
     message: string,
     billing: Record<string, unknown> | null,
     status: number,
+    receiptFailureKind: WorkReceiptFailureKind | null = null,
   ) {
     super(message);
     this.name = 'ToolHttpError';
     this.billing = billing;
     this.status = status;
+    this.receiptFailureKind = receiptFailureKind;
   }
 }
 
@@ -247,3 +252,9 @@ export function isHardBillingToolHttpError(error: unknown): boolean {
 export function isRateLimitToolHttpError(error: unknown): boolean {
   return error instanceof ToolHttpError && error.status === 429;
 }
+
+export function getToolHttpErrorReceiptFailureKind(
+  error: unknown,
+): WorkReceiptFailureKind | null {
+  return error instanceof ToolHttpError ? error.receiptFailureKind : null;
+}