@martian-engineering/lossless-claw 0.8.0 → 0.8.1

package/src/estimate-tokens.ts

@@ -1,80 +0,0 @@
-/**
- * Shared token estimation utility.
- *
- * Uses code-point-aware weighting instead of `text.length / 4`:
- *   - CJK (Chinese/Japanese/Korean) characters: ~1.5 tokens/char
- *   - Emoji / Supplementary Plane: ~2 tokens/char
- *   - ASCII / Latin: ~0.25 tokens/char (≈ 4 chars/token)
- *
- * Why not `text.length / 4`?
- * JavaScript `String.length` counts UTF-16 code units, not Unicode code points.
- * CJK characters are 1 UTF-16 unit but ~1.5 tokens; emoji are 2 UTF-16 units
- * (surrogate pairs) but ~2-4 tokens. The naive formula underestimates CJK by
- * ~6× and emoji by ~2-4×, causing compaction to trigger far too late for
- * non-English conversations.
- */
-
-/** Detect CJK code points across all relevant Unicode ranges. */
-function isCjkCodePoint(cp: number): boolean {
-  return (
-    (cp >= 0x4e00 && cp <= 0x9fff) ||   // CJK Unified Ideographs
-    (cp >= 0x3400 && cp <= 0x4dbf) ||    // CJK Extension A
-    (cp >= 0x20000 && cp <= 0x2a6df) ||  // CJK Extension B
-    (cp >= 0x2a700 && cp <= 0x2b73f) ||  // CJK Extension C
-    (cp >= 0x2b740 && cp <= 0x2b81f) ||  // CJK Extension D
-    (cp >= 0x2b820 && cp <= 0x2ceaf) ||  // CJK Extension E
-    (cp >= 0x2ceb0 && cp <= 0x2ebef) ||  // CJK Extension F
-    (cp >= 0x3000 && cp <= 0x303f) ||    // CJK Symbols and Punctuation
-    (cp >= 0x3040 && cp <= 0x30ff) ||    // Hiragana + Katakana
-    (cp >= 0xac00 && cp <= 0xd7af) ||    // Hangul Syllables
-    (cp >= 0xff00 && cp <= 0xffef)       // Fullwidth Forms
-  );
-}
-
-/** Estimate token cost for a single Unicode code point. */
-function estimateCodePointTokens(cp: number): number {
-  if (isCjkCodePoint(cp)) {
-    return 1.5;
-  }
-  if (cp > 0xffff) {
-    return 2;
-  }
-  return 0.25;
-}
-
-/** Estimate text tokens using Unicode-aware character weighting. */
-export function estimateTokens(text: string): number {
-  let tokens = 0;
-  for (const char of text) {
-    const cp = char.codePointAt(0) ?? 0;
-    tokens += estimateCodePointTokens(cp);
-  }
-  return Math.ceil(tokens);
-}
-
-/**
- * Truncate text so the estimated token count stays within `maxTokens`.
- *
- * Iterates by Unicode code point to avoid splitting surrogate pairs while
- * preserving the same weighting model as `estimateTokens()`.
- */
-export function truncateTextToEstimatedTokens(text: string, maxTokens: number): string {
-  if (maxTokens <= 0 || !text) {
-    return "";
-  }
-
-  let tokens = 0;
-  let end = 0;
-
-  for (const char of text) {
-    const cp = char.codePointAt(0) ?? 0;
-    const nextTokens = tokens + estimateCodePointTokens(cp);
-    if (Math.ceil(nextTokens) > maxTokens) {
-      break;
-    }
-    tokens = nextTokens;
-    end += char.length;
-  }
-
-  return text.slice(0, end);
-}

package/src/expansion-auth.ts

@@ -1,365 +0,0 @@
-import type { ExpansionOrchestrator, ExpansionRequest, ExpansionResult } from "./expansion.js";
-
-// ── Types ────────────────────────────────────────────────────────────────────
-
-export type ExpansionGrant = {
-  /** Unique grant ID */
-  grantId: string;
-  /** Session ID that issued the grant */
-  issuerSessionId: string;
-  /** Conversation IDs the grantee is allowed to traverse */
-  allowedConversationIds: number[];
-  /** Specific summary IDs the grantee is allowed to expand (if empty, all within conversation are allowed) */
-  allowedSummaryIds: string[];
-  /** Maximum traversal depth */
-  maxDepth: number;
-  /** Maximum tokens the grantee can retrieve */
-  tokenCap: number;
-  /** When the grant expires */
-  expiresAt: Date;
-  /** Whether this grant has been revoked */
-  revoked: boolean;
-  /** Creation timestamp */
-  createdAt: Date;
-};
-
-export type CreateGrantInput = {
-  issuerSessionId: string;
-  allowedConversationIds: number[];
-  allowedSummaryIds?: string[];
-  maxDepth?: number;
-  tokenCap?: number;
-  /** TTL in milliseconds (default: 5 minutes) */
-  ttlMs?: number;
-};
-
-export type CreateDelegatedExpansionGrantInput = CreateGrantInput & {
-  delegatedSessionKey: string;
-};
-
-export type ValidationResult = {
-  valid: boolean;
-  reason?: string;
-};
-
-export type AuthorizedExpansionOrchestrator = {
-  expand(grantId: string, request: ExpansionRequest): Promise<ExpansionResult>;
-};
-
-// ── Defaults ─────────────────────────────────────────────────────────────────
-
-const DEFAULT_MAX_DEPTH = 3;
-const DEFAULT_TOKEN_CAP = 4000;
-const DEFAULT_TTL_MS = 5 * 60 * 1000; // 5 minutes
-
-// ── ExpansionAuthManager ─────────────────────────────────────────────────────
-
-export class ExpansionAuthManager {
-  private grants: Map<string, ExpansionGrant> = new Map();
-  private consumedTokensByGrantId: Map<string, number> = new Map();
-
-  /**
-   * Create a new expansion grant with the given parameters.
-   * Generates a unique grant ID and applies defaults for optional fields.
-   */
-  createGrant(input: CreateGrantInput): ExpansionGrant {
-    const grantId = "grant_" + crypto.randomUUID().slice(0, 12);
-    const now = new Date();
-    const ttlMs = input.ttlMs ?? DEFAULT_TTL_MS;
-
-    const grant: ExpansionGrant = {
-      grantId,
-      issuerSessionId: input.issuerSessionId,
-      allowedConversationIds: input.allowedConversationIds,
-      allowedSummaryIds: input.allowedSummaryIds ?? [],
-      maxDepth: input.maxDepth ?? DEFAULT_MAX_DEPTH,
-      tokenCap: input.tokenCap ?? DEFAULT_TOKEN_CAP,
-      expiresAt: new Date(now.getTime() + ttlMs),
-      revoked: false,
-      createdAt: now,
-    };
-
-    this.grants.set(grantId, grant);
-    this.consumedTokensByGrantId.set(grantId, 0);
-    return grant;
-  }
-
-  /**
-   * Retrieve a grant by ID. Returns null if the grant does not exist,
-   * has been revoked, or has expired.
-   */
-  getGrant(grantId: string): ExpansionGrant | null {
-    const grant = this.grants.get(grantId);
-    if (!grant) {
-      return null;
-    }
-    if (grant.revoked) {
-      return null;
-    }
-    if (grant.expiresAt.getTime() <= Date.now()) {
-      return null;
-    }
-    return grant;
-  }
-
-  /**
-   * Revoke a grant, preventing any further use.
-   * Returns true if the grant was found and revoked, false if not found.
-   */
-  revokeGrant(grantId: string): boolean {
-    const grant = this.grants.get(grantId);
-    if (!grant) {
-      return false;
-    }
-    grant.revoked = true;
-    return true;
-  }
-
-  /**
-   * Resolve remaining token budget for an active grant.
-   */
-  getRemainingTokenBudget(grantId: string): number | null {
-    const grant = this.getGrant(grantId);
-    if (!grant) {
-      return null;
-    }
-    const consumed = Math.max(0, this.consumedTokensByGrantId.get(grantId) ?? 0);
-    return Math.max(0, Math.floor(grant.tokenCap) - consumed);
-  }
-
-  /**
-   * Consume token budget for a grant, clamped to the grant token cap.
-   */
-  consumeTokenBudget(grantId: string, consumedTokens: number): number | null {
-    const grant = this.getGrant(grantId);
-    if (!grant) {
-      return null;
-    }
-    const safeConsumed =
-      typeof consumedTokens === "number" && Number.isFinite(consumedTokens)
-        ? Math.max(0, Math.floor(consumedTokens))
-        : 0;
-    const previous = Math.max(0, this.consumedTokensByGrantId.get(grantId) ?? 0);
-    const next = Math.min(Math.max(1, Math.floor(grant.tokenCap)), previous + safeConsumed);
-    this.consumedTokensByGrantId.set(grantId, next);
-    return Math.max(0, Math.floor(grant.tokenCap) - next);
-  }
-
-  /**
-   * Validate an expansion request against a grant.
-   * Checks existence, expiry, revocation, conversation scope, and summary scope.
-   */
-  validateExpansion(
-    grantId: string,
-    request: {
-      conversationId: number;
-      summaryIds: string[];
-      depth: number;
-      tokenCap: number;
-    },
-  ): ValidationResult {
-    const grant = this.grants.get(grantId);
-
-    // 1. Grant must exist
-    if (!grant) {
-      return { valid: false, reason: "Grant not found" };
-    }
-
-    // 2. Grant must not be revoked
-    if (grant.revoked) {
-      return { valid: false, reason: "Grant has been revoked" };
-    }
-
-    // 3. Grant must not be expired
-    if (grant.expiresAt.getTime() <= Date.now()) {
-      return { valid: false, reason: "Grant has expired" };
-    }
-
-    // 4. Conversation ID must be in the allowed set
-    if (!grant.allowedConversationIds.includes(request.conversationId)) {
-      return {
-        valid: false,
-        reason: `Conversation ${request.conversationId} is not in the allowed set`,
-      };
-    }
-
-    // 5. If allowedSummaryIds is non-empty, all requested summaryIds must be allowed
-    if (grant.allowedSummaryIds.length > 0) {
-      const allowedSet = new Set(grant.allowedSummaryIds);
-      const unauthorized = request.summaryIds.filter((id) => !allowedSet.has(id));
-      if (unauthorized.length > 0) {
-        return {
-          valid: false,
-          reason: `Summary IDs not authorized: ${unauthorized.join(", ")}`,
-        };
-      }
-    }
-
-    // 6. Depth and tokenCap are enforced via clamping in wrapWithAuth, not
-    // rejected here. This allows callers to request more than the grant
-    // permits — the values will be clamped to the grant limits at execution time.
-
-    return { valid: true };
-  }
-
-  /**
-   * Remove all expired and revoked grants from the store.
-   * Returns the number of grants removed.
-   */
-  cleanup(): number {
-    const now = Date.now();
-    let removed = 0;
-
-    for (const [grantId, grant] of this.grants) {
-      if (grant.revoked || grant.expiresAt.getTime() <= now) {
-        this.grants.delete(grantId);
-        this.consumedTokensByGrantId.delete(grantId);
-        removed++;
-      }
-    }
-
-    return removed;
-  }
-}
-
-const runtimeExpansionAuthManager = new ExpansionAuthManager();
-const delegatedSessionGrantIds = new Map<string, string>();
-
-/**
- * Return the singleton auth manager used by runtime delegated expansion flows.
- */
-export function getRuntimeExpansionAuthManager(): ExpansionAuthManager {
-  return runtimeExpansionAuthManager;
-}
-
-/**
- * Create a delegated expansion grant and bind it to the child session key.
- */
-export function createDelegatedExpansionGrant(
-  input: CreateDelegatedExpansionGrantInput,
-): ExpansionGrant {
-  const delegatedSessionKey = input.delegatedSessionKey.trim();
-  if (!delegatedSessionKey) {
-    throw new Error("delegatedSessionKey is required for delegated expansion grants");
-  }
-
-  const grant = runtimeExpansionAuthManager.createGrant({
-    issuerSessionId: input.issuerSessionId,
-    allowedConversationIds: input.allowedConversationIds,
-    allowedSummaryIds: input.allowedSummaryIds,
-    maxDepth: input.maxDepth,
-    tokenCap: input.tokenCap,
-    ttlMs: input.ttlMs,
-  });
-  delegatedSessionGrantIds.set(delegatedSessionKey, grant.grantId);
-  return grant;
-}
-
-/**
- * Resolve the delegated expansion grant id bound to a session key.
- */
-export function resolveDelegatedExpansionGrantId(sessionKey: string): string | null {
-  const key = sessionKey.trim();
-  if (!key) {
-    return null;
-  }
-  return delegatedSessionGrantIds.get(key) ?? null;
-}
-
-/**
- * Revoke the delegated grant bound to a session key.
- * Optionally remove the binding after revocation.
- */
-export function revokeDelegatedExpansionGrantForSession(
-  sessionKey: string,
-  opts?: { removeBinding?: boolean },
-): boolean {
-  const key = sessionKey.trim();
-  if (!key) {
-    return false;
-  }
-  const grantId = delegatedSessionGrantIds.get(key);
-  if (!grantId) {
-    return false;
-  }
-  const didRevoke = runtimeExpansionAuthManager.revokeGrant(grantId);
-  if (opts?.removeBinding) {
-    delegatedSessionGrantIds.delete(key);
-  }
-  return didRevoke;
-}
-
-/**
- * Remove delegated grant binding for a session key without revoking.
- */
-export function removeDelegatedExpansionGrantForSession(sessionKey: string): boolean {
-  const key = sessionKey.trim();
-  if (!key) {
-    return false;
-  }
-  return delegatedSessionGrantIds.delete(key);
-}
-
-/**
- * Test-only reset helper for delegated runtime grants.
- */
-export function resetDelegatedExpansionGrantsForTests(): void {
-  delegatedSessionGrantIds.clear();
-}
-
-// ── Authorized wrapper ───────────────────────────────────────────────────────
-
-/**
- * Create a thin authorization wrapper around an ExpansionOrchestrator.
- * The wrapper validates the grant before delegating to the underlying
- * orchestrator.
- */
-export function wrapWithAuth(
-  orchestrator: ExpansionOrchestrator,
-  authManager: ExpansionAuthManager,
-): AuthorizedExpansionOrchestrator {
-  return {
-    async expand(grantId: string, request: ExpansionRequest): Promise<ExpansionResult> {
-      const validation = authManager.validateExpansion(grantId, {
-        conversationId: request.conversationId,
-        summaryIds: request.summaryIds,
-        depth: request.maxDepth ?? DEFAULT_MAX_DEPTH,
-        tokenCap: request.tokenCap ?? DEFAULT_TOKEN_CAP,
-      });
-
-      if (!validation.valid) {
-        throw new Error(`Expansion authorization failed: ${validation.reason}`);
-      }
-
-      const remainingBudget = authManager.getRemainingTokenBudget(grantId);
-      if (remainingBudget == null) {
-        throw new Error("Expansion authorization failed: Grant not found");
-      }
-      if (remainingBudget <= 0) {
-        throw new Error("Expansion authorization failed: Grant token budget exhausted");
-      }
-
-      // Clamp depth to grant maxDepth
-      const grant = authManager.getGrant(grantId);
-      const grantMaxDepth = grant?.maxDepth ?? DEFAULT_MAX_DEPTH;
-      const requestedDepth =
-        typeof request.maxDepth === "number" && Number.isFinite(request.maxDepth)
-          ? Math.max(1, Math.trunc(request.maxDepth))
-          : grantMaxDepth;
-      const effectiveDepth = Math.min(requestedDepth, grantMaxDepth);
-
-      const requestedTokenCap =
-        typeof request.tokenCap === "number" && Number.isFinite(request.tokenCap)
-          ? Math.max(1, Math.trunc(request.tokenCap))
-          : remainingBudget;
-      const effectiveTokenCap = Math.max(1, Math.min(requestedTokenCap, remainingBudget));
-      const result = await orchestrator.expand({
-        ...request,
-        maxDepth: effectiveDepth,
-        tokenCap: effectiveTokenCap,
-      });
-      authManager.consumeTokenBudget(grantId, result.totalTokens);
-      return result;
-    },
-  };
-}