@martian-engineering/lossless-claw 0.8.0 → 0.8.1

package/src/expansion-policy.ts

@@ -1,303 +0,0 @@
-export type LcmExpansionRoutingIntent = "query_probe" | "explicit_expand";
-
-export type LcmExpansionRoutingAction = "answer_directly" | "expand_shallow" | "delegate_traversal";
-
-export type LcmExpansionTokenRiskLevel = "low" | "moderate" | "high";
-
-export type LcmExpansionRoutingInput = {
-  intent: LcmExpansionRoutingIntent;
-  query?: string;
-  requestedMaxDepth?: number;
-  candidateSummaryCount: number;
-  tokenCap: number;
-  includeMessages?: boolean;
-};
-
-export type LcmExpansionRoutingDecision = {
-  action: LcmExpansionRoutingAction;
-  normalizedMaxDepth: number;
-  candidateSummaryCount: number;
-  estimatedTokens: number;
-  tokenCap: number;
-  tokenRiskRatio: number;
-  tokenRiskLevel: LcmExpansionTokenRiskLevel;
-  indicators: {
-    broadTimeRange: boolean;
-    multiHopRetrieval: boolean;
-  };
-  triggers: {
-    directByNoCandidates: boolean;
-    directByLowComplexityProbe: boolean;
-    delegateByDepth: boolean;
-    delegateByCandidateCount: boolean;
-    delegateByTokenRisk: boolean;
-    delegateByBroadTimeRangeAndMultiHop: boolean;
-  };
-  reasons: string[];
-};
-
-export const EXPANSION_ROUTING_THRESHOLDS = {
-  defaultDepth: 3,
-  minDepth: 1,
-  maxDepth: 10,
-  directMaxDepth: 2,
-  directMaxCandidates: 1,
-  moderateTokenRiskRatio: 0.35,
-  highTokenRiskRatio: 0.7,
-  baseTokensPerSummary: 220,
-  includeMessagesTokenMultiplier: 1.9,
-  perDepthTokenGrowth: 0.65,
-  broadTimeRangeTokenMultiplier: 1.35,
-  multiHopTokenMultiplier: 1.25,
-  multiHopDepthThreshold: 3,
-  multiHopCandidateThreshold: 5,
-} as const;
-
-const BROAD_TIME_RANGE_PATTERNS = [
-  /\b(last|past)\s+(month|months|quarter|quarters|year|years)\b/i,
-  /\b(over|across|throughout)\s+(time|months|quarters|years)\b/i,
-  /\b(timeline|chronology|history|long[-\s]?term)\b/i,
-  /\bbetween\s+[^.]{0,40}\s+and\s+[^.]{0,40}\b/i,
-];
-
-const MULTI_HOP_QUERY_PATTERNS = [
-  /\b(root\s+cause|causal\s+chain|chain\s+of\s+events)\b/i,
-  /\b(multi[-\s]?hop|multi[-\s]?step|cross[-\s]?summary)\b/i,
-  /\bhow\s+did\b.+\blead\s+to\b/i,
-];
-
-/** Normalize a requested depth to a deterministic bounded value. */
-function normalizeDepth(requestedMaxDepth?: number): number {
-  if (typeof requestedMaxDepth !== "number" || !Number.isFinite(requestedMaxDepth)) {
-    return EXPANSION_ROUTING_THRESHOLDS.defaultDepth;
-  }
-  const rounded = Math.trunc(requestedMaxDepth);
-  return Math.max(
-    EXPANSION_ROUTING_THRESHOLDS.minDepth,
-    Math.min(EXPANSION_ROUTING_THRESHOLDS.maxDepth, rounded),
-  );
-}
-
-/** Normalize token cap to a positive bounded value for risk computation. */
-function normalizeTokenCap(tokenCap: number): number {
-  if (!Number.isFinite(tokenCap)) {
-    return Number.MAX_SAFE_INTEGER;
-  }
-  return Math.max(1, Math.trunc(tokenCap));
-}
-
-/**
- * Detect broad time-range intent from the user query.
- *
- * This is a deterministic text heuristic used by orchestration policy only;
- * it does not perform retrieval.
- */
-export function detectBroadTimeRangeIndicator(query?: string): boolean {
-  if (!query) {
-    return false;
-  }
-  const trimmed = query.trim();
-  if (!trimmed) {
-    return false;
-  }
-
-  if (BROAD_TIME_RANGE_PATTERNS.some((pattern) => pattern.test(trimmed))) {
-    return true;
-  }
-
-  const years = Array.from(trimmed.matchAll(/\b(?:19|20)\d{2}\b/g), (match) => Number(match[0]));
-  if (years.length < 2) {
-    return false;
-  }
-
-  const earliest = Math.min(...years);
-  const latest = Math.max(...years);
-  return latest - earliest >= 2;
-}
-
-/**
- * Detect whether traversal likely requires multi-hop expansion.
- *
- * Multi-hop is inferred from depth, breadth, and explicit language in the query.
- */
-export function detectMultiHopIndicator(input: {
-  query?: string;
-  requestedMaxDepth?: number;
-  candidateSummaryCount: number;
-}): boolean {
-  const normalizedMaxDepth = normalizeDepth(input.requestedMaxDepth);
-  const candidateSummaryCount = Math.max(0, Math.trunc(input.candidateSummaryCount));
-
-  if (normalizedMaxDepth >= EXPANSION_ROUTING_THRESHOLDS.multiHopDepthThreshold) {
-    return true;
-  }
-  if (candidateSummaryCount >= EXPANSION_ROUTING_THRESHOLDS.multiHopCandidateThreshold) {
-    return true;
-  }
-  if (!input.query) {
-    return false;
-  }
-
-  const trimmed = input.query.trim();
-  if (!trimmed) {
-    return false;
-  }
-  return MULTI_HOP_QUERY_PATTERNS.some((pattern) => pattern.test(trimmed));
-}
-
-/**
- * Estimate expansion token volume from traversal characteristics.
- *
- * This deterministic estimate intentionally over-approximates near deep/broad
- * traversals so delegation triggers before hitting hard runtime caps.
- */
-export function estimateExpansionTokens(input: {
-  requestedMaxDepth?: number;
-  candidateSummaryCount: number;
-  includeMessages?: boolean;
-  broadTimeRangeIndicator?: boolean;
-  multiHopIndicator?: boolean;
-}): number {
-  const normalizedMaxDepth = normalizeDepth(input.requestedMaxDepth);
-  const candidateSummaryCount = Math.max(0, Math.trunc(input.candidateSummaryCount));
-  if (candidateSummaryCount === 0) {
-    return 0;
-  }
-
-  const includeMessagesMultiplier = input.includeMessages
-    ? EXPANSION_ROUTING_THRESHOLDS.includeMessagesTokenMultiplier
-    : 1;
-  const depthMultiplier =
-    1 + (normalizedMaxDepth - 1) * EXPANSION_ROUTING_THRESHOLDS.perDepthTokenGrowth;
-  const timeRangeMultiplier = input.broadTimeRangeIndicator
-    ? EXPANSION_ROUTING_THRESHOLDS.broadTimeRangeTokenMultiplier
-    : 1;
-  const multiHopMultiplier = input.multiHopIndicator
-    ? EXPANSION_ROUTING_THRESHOLDS.multiHopTokenMultiplier
-    : 1;
-
-  const perSummaryEstimate =
-    EXPANSION_ROUTING_THRESHOLDS.baseTokensPerSummary *
-    includeMessagesMultiplier *
-    depthMultiplier *
-    timeRangeMultiplier *
-    multiHopMultiplier;
-
-  return Math.max(0, Math.ceil(perSummaryEstimate * candidateSummaryCount));
-}
-
-/** Classify token risk level relative to a provided cap. */
-export function classifyExpansionTokenRisk(input: { estimatedTokens: number; tokenCap: number }): {
-  ratio: number;
-  level: LcmExpansionTokenRiskLevel;
-} {
-  const estimatedTokens = Math.max(0, Math.trunc(input.estimatedTokens));
-  const tokenCap = normalizeTokenCap(input.tokenCap);
-  const ratio = estimatedTokens / tokenCap;
-
-  if (ratio >= EXPANSION_ROUTING_THRESHOLDS.highTokenRiskRatio) {
-    return { ratio, level: "high" };
-  }
-  if (ratio >= EXPANSION_ROUTING_THRESHOLDS.moderateTokenRiskRatio) {
-    return { ratio, level: "moderate" };
-  }
-  return { ratio, level: "low" };
-}
-
-/**
- * Decide deterministic route-vs-delegate policy for LCM expansion orchestration.
- *
- * The decision matrix supports three outcomes:
- * - answer directly (skip expansion)
- * - do shallow/direct expansion
- * - delegate deep traversal to a sub-agent
- */
-export function decideLcmExpansionRouting(
-  input: LcmExpansionRoutingInput,
-): LcmExpansionRoutingDecision {
-  const normalizedMaxDepth = normalizeDepth(input.requestedMaxDepth);
-  const candidateSummaryCount = Math.max(0, Math.trunc(input.candidateSummaryCount));
-  const tokenCap = normalizeTokenCap(input.tokenCap);
-  const broadTimeRange = detectBroadTimeRangeIndicator(input.query);
-  const multiHopRetrieval = detectMultiHopIndicator({
-    query: input.query,
-    requestedMaxDepth: normalizedMaxDepth,
-    candidateSummaryCount,
-  });
-  const estimatedTokens = estimateExpansionTokens({
-    requestedMaxDepth: normalizedMaxDepth,
-    candidateSummaryCount,
-    includeMessages: input.includeMessages,
-    broadTimeRangeIndicator: broadTimeRange,
-    multiHopIndicator: multiHopRetrieval,
-  });
-  const tokenRisk = classifyExpansionTokenRisk({ estimatedTokens, tokenCap });
-
-  const directByNoCandidates = candidateSummaryCount === 0;
-  const directByLowComplexityProbe =
-    input.intent === "query_probe" &&
-    !directByNoCandidates &&
-    normalizedMaxDepth <= EXPANSION_ROUTING_THRESHOLDS.directMaxDepth &&
-    candidateSummaryCount <= EXPANSION_ROUTING_THRESHOLDS.directMaxCandidates &&
-    tokenRisk.level === "low" &&
-    !broadTimeRange &&
-    !multiHopRetrieval;
-
-  const delegateByDepth = false;
-  const delegateByCandidateCount = false;
-  const delegateByTokenRisk = tokenRisk.level === "high";
-  const delegateByBroadTimeRangeAndMultiHop = broadTimeRange && multiHopRetrieval;
-
-  const shouldDirect = directByNoCandidates || directByLowComplexityProbe;
-  const shouldDelegate =
-    !shouldDirect && (delegateByTokenRisk || delegateByBroadTimeRangeAndMultiHop);
-
-  const action: LcmExpansionRoutingAction = shouldDirect
-    ? "answer_directly"
-    : shouldDelegate
-      ? "delegate_traversal"
-      : "expand_shallow";
-
-  const reasons: string[] = [];
-  if (directByNoCandidates) {
-    reasons.push("No candidate summary IDs are available.");
-  }
-  if (directByLowComplexityProbe) {
-    reasons.push("Query probe is low complexity and below retrieval-risk thresholds.");
-  }
-  if (delegateByTokenRisk) {
-    reasons.push(
-      `Estimated token risk ratio ${tokenRisk.ratio.toFixed(2)} meets delegate threshold ` +
-        `${EXPANSION_ROUTING_THRESHOLDS.highTokenRiskRatio.toFixed(2)}.`,
-    );
-  }
-  if (delegateByBroadTimeRangeAndMultiHop) {
-    reasons.push("Broad time-range request combined with multi-hop retrieval indicators.");
-  }
-  if (action === "expand_shallow") {
-    reasons.push("Complexity is bounded; use direct/shallow expansion.");
-  }
-
-  return {
-    action,
-    normalizedMaxDepth,
-    candidateSummaryCount,
-    estimatedTokens,
-    tokenCap,
-    tokenRiskRatio: tokenRisk.ratio,
-    tokenRiskLevel: tokenRisk.level,
-    indicators: {
-      broadTimeRange,
-      multiHopRetrieval,
-    },
-    triggers: {
-      directByNoCandidates,
-      directByLowComplexityProbe,
-      delegateByDepth,
-      delegateByCandidateCount,
-      delegateByTokenRisk,
-      delegateByBroadTimeRangeAndMultiHop,
-    },
-    reasons,
-  };
-}

package/src/expansion.ts

@@ -1,383 +0,0 @@
-import { Type } from "@sinclair/typebox";
-import type { LcmConfig } from "./db/config.js";
-import type { RetrievalEngine, ExpandResult, GrepResult } from "./retrieval.js";
-
-// ── Types ────────────────────────────────────────────────────────────────────
-
-export type ExpansionRequest = {
-  /** Summary IDs to expand */
-  summaryIds: string[];
-  /** Max traversal depth per summary (default: 3) */
-  maxDepth?: number;
-  /** Max tokens across the entire expansion (default: config.maxExpandTokens) */
-  tokenCap?: number;
-  /** Whether to include raw source messages at leaf level */
-  includeMessages?: boolean;
-  /** Conversation ID scope */
-  conversationId: number;
-};
-
-export type ExpansionResult = {
-  /** Expanded summaries with their children/messages */
-  expansions: Array<{
-    summaryId: string;
-    children: Array<{
-      summaryId: string;
-      kind: string;
-      snippet: string;
-      tokenCount: number;
-    }>;
-    messages: Array<{
-      messageId: number;
-      role: string;
-      snippet: string;
-      tokenCount: number;
-    }>;
-  }>;
-  /** Cited IDs for follow-up traversal */
-  citedIds: string[];
-  /** Total tokens in the result */
-  totalTokens: number;
-  /** Whether any expansion was truncated */
-  truncated: boolean;
-};
-
-// ── Helpers ──────────────────────────────────────────────────────────────────
-
-const SNIPPET_MAX_CHARS = 200;
-
-/** Truncate content to a short snippet for display. */
-function truncateSnippet(content: string, maxChars: number = SNIPPET_MAX_CHARS): string {
-  if (content.length <= maxChars) {
-    return content;
-  }
-  return content.slice(0, maxChars) + "...";
-}
-
-/**
- * Resolve the effective expansion token cap by applying a configured default
- * and an explicit upper bound.
- */
-export function resolveExpansionTokenCap(input: {
-  requestedTokenCap?: number;
-  maxExpandTokens: number;
-}): number {
-  const maxExpandTokens = Math.max(1, Math.trunc(input.maxExpandTokens));
-  const requestedTokenCap = input.requestedTokenCap;
-  if (typeof requestedTokenCap !== "number" || !Number.isFinite(requestedTokenCap)) {
-    return maxExpandTokens;
-  }
-  return Math.min(Math.max(1, Math.trunc(requestedTokenCap)), maxExpandTokens);
-}
-
-/**
- * Convert a single RetrievalEngine.expand() result into the ExpansionResult
- * entry format, truncating content to short snippets.
- */
-function toExpansionEntry(
-  summaryId: string,
-  raw: ExpandResult,
-): ExpansionResult["expansions"][number] {
-  return {
-    summaryId,
-    children: raw.children.map((c) => ({
-      summaryId: c.summaryId,
-      kind: c.kind,
-      snippet: truncateSnippet(c.content),
-      tokenCount: c.tokenCount,
-    })),
-    messages: raw.messages.map((m) => ({
-      messageId: m.messageId,
-      role: m.role,
-      snippet: truncateSnippet(m.content),
-      tokenCount: m.tokenCount,
-    })),
-  };
-}
-
-/** Collect all referenced summary IDs from an expansion entry. */
-function collectCitedIds(entry: ExpansionResult["expansions"][number]): string[] {
-  const ids: string[] = [entry.summaryId];
-  for (const child of entry.children) {
-    ids.push(child.summaryId);
-  }
-  return ids;
-}
-
-// ── ExpansionOrchestrator ────────────────────────────────────────────────────
-
-export class ExpansionOrchestrator {
-  constructor(private retrieval: RetrievalEngine) {}
-
-  /**
-   * Expand each summary ID using the RetrievalEngine, collecting results and
-   * enforcing a global token cap across all expansions.
-   */
-  async expand(request: ExpansionRequest): Promise<ExpansionResult> {
-    const maxDepth = request.maxDepth ?? 3;
-    const tokenCap = request.tokenCap ?? Infinity;
-    const includeMessages = request.includeMessages ?? false;
-
-    const result: ExpansionResult = {
-      expansions: [],
-      citedIds: [],
-      totalTokens: 0,
-      truncated: false,
-    };
-
-    const citedSet = new Set<string>();
-
-    for (const summaryId of request.summaryIds) {
-      if (result.truncated) {
-        break;
-      }
-
-      // Calculate remaining budget for this expansion
-      const remainingBudget = tokenCap - result.totalTokens;
-      if (remainingBudget <= 0) {
-        result.truncated = true;
-        break;
-      }
-
-      const raw = await this.retrieval.expand({
-        summaryId,
-        depth: maxDepth,
-        includeMessages,
-        tokenCap: remainingBudget,
-      });
-
-      const entry = toExpansionEntry(summaryId, raw);
-      result.expansions.push(entry);
-      result.totalTokens += raw.estimatedTokens;
-
-      // Track cited IDs
-      for (const id of collectCitedIds(entry)) {
-        citedSet.add(id);
-      }
-
-      if (raw.truncated) {
-        result.truncated = true;
-      }
-    }
-
-    result.citedIds = [...citedSet];
-    return result;
-  }
-
-  /**
-   * Convenience method: grep for matching summaries, then expand the top results.
-   * Combines the routing pass (grep) with the deep expansion pass.
-   */
-  async describeAndExpand(input: {
-    query: string;
-    mode: "regex" | "full_text";
-    conversationId?: number;
-    maxDepth?: number;
-    tokenCap?: number;
-  }): Promise<ExpansionResult> {
-    const grepResult: GrepResult = await this.retrieval.grep({
-      query: input.query,
-      mode: input.mode,
-      scope: "summaries",
-      conversationId: input.conversationId,
-    });
-
-    const summaryIds = [...grepResult.summaries]
-      .sort((a, b) => {
-        const recencyDelta = b.createdAt.getTime() - a.createdAt.getTime();
-        if (recencyDelta !== 0) {
-          return recencyDelta;
-        }
-        const aRank = a.rank ?? Number.POSITIVE_INFINITY;
-        const bRank = b.rank ?? Number.POSITIVE_INFINITY;
-        return aRank - bRank;
-      })
-      .map((s) => s.summaryId);
-    if (summaryIds.length === 0) {
-      return {
-        expansions: [],
-        citedIds: [],
-        totalTokens: 0,
-        truncated: false,
-      };
-    }
-
-    return this.expand({
-      summaryIds,
-      maxDepth: input.maxDepth,
-      tokenCap: input.tokenCap,
-      includeMessages: false,
-      conversationId: input.conversationId ?? 0,
-    });
-  }
-}
-
-// ── Distill for subagent ─────────────────────────────────────────────────────
-
-/**
- * Format an ExpansionResult into a compact text payload suitable for passing
- * to a subagent or returning to the main agent.
- */
-export function distillForSubagent(result: ExpansionResult): string {
-  const lines: string[] = [];
-
-  lines.push(
-    `## Expansion Results (${result.expansions.length} summaries, ${result.totalTokens} total tokens)`,
-  );
-  lines.push("");
-
-  for (const entry of result.expansions) {
-    // Determine kind from children presence: if it has children it was a condensed node
-    const kind = entry.children.length > 0 ? "condensed" : "leaf";
-    const tokenSum =
-      entry.children.reduce((sum, c) => sum + c.tokenCount, 0) +
-      entry.messages.reduce((sum, m) => sum + m.tokenCount, 0);
-
-    lines.push(`### ${entry.summaryId} (${kind}, ${tokenSum} tokens)`);
-
-    if (entry.children.length > 0) {
-      lines.push(`Children: ${entry.children.map((c) => c.summaryId).join(", ")}`);
-    }
-
-    if (entry.messages.length > 0) {
-      const msgParts = entry.messages.map(
-        (m) => `msg#${m.messageId} (${m.role}, ${m.tokenCount} tokens)`,
-      );
-      lines.push(`Messages: ${msgParts.join(", ")}`);
-    }
-
-    // Show a snippet for children that have content
-    for (const child of entry.children) {
-      if (child.snippet) {
-        lines.push(`[Snippet: ${truncateSnippet(child.snippet)}]`);
-        break; // Only show one snippet per entry to keep it compact
-      }
-    }
-
-    lines.push("");
-  }
-
-  if (result.citedIds.length > 0) {
-    lines.push(`Cited IDs for follow-up: ${result.citedIds.join(", ")}`);
-  }
-
-  lines.push(`[Truncated: ${result.truncated ? "yes" : "no"}]`);
-
-  return lines.join("\n");
-}
-
-// ── Tool definition ──────────────────────────────────────────────────────────
-
-const LcmExpansionSchema = Type.Object({
-  summaryIds: Type.Optional(
-    Type.Array(Type.String(), {
-      description: "Summary IDs to expand (e.g. sum_abc123). Required if query is not provided.",
-    }),
-  ),
-  query: Type.Optional(
-    Type.String({
-      description:
-        "Text query to grep for matching summaries before expanding. " +
-        "If provided, summaryIds is ignored and the top grep results are expanded instead.",
-    }),
-  ),
-  maxDepth: Type.Optional(
-    Type.Number({
-      description: "Max traversal depth per summary (default: 3).",
-      minimum: 1,
-      maximum: 10,
-    }),
-  ),
-  tokenCap: Type.Optional(
-    Type.Number({
-      description: "Max tokens across the entire expansion result.",
-      minimum: 1,
-    }),
-  ),
-  includeMessages: Type.Optional(
-    Type.Boolean({
-      description: "Whether to include raw source messages at leaf level (default: false).",
-    }),
-  ),
-});
-
-/**
- * Build a tool definition object for LCM expansion that can be registered as
- * an agent tool. Follows the pattern used in `src/agents/tools/`.
- *
- * Requires an already-initialised ExpansionOrchestrator and an LcmConfig
- * (for the default tokenCap).
- */
-export function buildExpansionToolDefinition(options: {
-  orchestrator: ExpansionOrchestrator;
-  config: LcmConfig;
-  conversationId: number;
-}) {
-  const { orchestrator, config, conversationId } = options;
-
-  return {
-    name: "lcm_expand",
-    description:
-      "Expand compacted conversation summaries from LCM (Lossless Context Management). " +
-      "Traverses the summary DAG to retrieve children and source messages. " +
-      "Use this to drill into previously-compacted context when you need detail " +
-      "that was summarised away. Returns a compact text payload with cited IDs for follow-up.",
-    parameters: LcmExpansionSchema,
-    execute: async (
-      _toolCallId: string,
-      params: Record<string, unknown>,
-    ): Promise<{ content: Array<{ type: "text"; text: string }>; details: unknown }> => {
-      const summaryIds = params.summaryIds as string[] | undefined;
-      const query = typeof params.query === "string" ? params.query.trim() : undefined;
-      const maxDepth =
-        typeof params.maxDepth === "number" ? Math.trunc(params.maxDepth) : undefined;
-      const requestedTokenCap =
-        typeof params.tokenCap === "number" ? Math.trunc(params.tokenCap) : undefined;
-      const tokenCap = resolveExpansionTokenCap({
-        requestedTokenCap,
-        maxExpandTokens: config.maxExpandTokens,
-      });
-      const includeMessages =
-        typeof params.includeMessages === "boolean" ? params.includeMessages : false;
-
-      let result: ExpansionResult;
-
-      if (query) {
-        // Grep-first path: find summaries matching the query, then expand
-        result = await orchestrator.describeAndExpand({
-          query,
-          mode: "full_text",
-          conversationId,
-          maxDepth,
-          tokenCap,
-        });
-      } else if (summaryIds && summaryIds.length > 0) {
-        // Direct expansion of specific summary IDs
-        result = await orchestrator.expand({
-          summaryIds,
-          maxDepth,
-          tokenCap,
-          includeMessages,
-          conversationId,
-        });
-      } else {
-        const text = "Error: either summaryIds or query must be provided.";
-        return {
-          content: [{ type: "text", text }],
-          details: { error: text },
-        };
-      }
-
-      const distilled = distillForSubagent(result);
-      return {
-        content: [{ type: "text", text: distilled }],
-        details: {
-          expansionCount: result.expansions.length,
-          citedIds: result.citedIds,
-          totalTokens: result.totalTokens,
-          truncated: result.truncated,
-        },
-      };
-    },
-  };
-}