@diyor28/context 1.0.0

package/src/adapters/anthropic-estimator.ts

@@ -0,0 +1,125 @@
+/**
+ * AnthropicTokenEstimator: Exact token counting via Anthropic API.
+ *
+ * Uses client.messages.countTokens() for exact counts.
+ */
+
+import Anthropic from '@anthropic-ai/sdk';
+import type { ContextBlock } from '../types/block.js';
+import type { TokenEstimator, TokenEstimate } from './token-estimator.js';
+import {
+  heuristicTokenCount,
+  serializeBlockForEstimation,
+} from './token-estimator.js';
+
+/**
+ * AnthropicTokenEstimator using API-based exact counting.
+ */
+export class AnthropicTokenEstimator implements TokenEstimator {
+  private readonly client: Anthropic;
+  private readonly model: string;
+
+  /**
+   * Create an AnthropicTokenEstimator.
+   *
+   * @param client - Anthropic SDK client
+   * @param model - Model name (e.g., 'claude-sonnet-4-5')
+   */
+  constructor(client: Anthropic, model: string = 'claude-sonnet-4-5') {
+    this.client = client;
+    this.model = model;
+  }
+
+  /**
+   * Estimate tokens for a single block using Anthropic API.
+   *
+   * @param block - Block to estimate
+   * @returns Token estimate (exact confidence)
+   */
+  async estimateBlock(block: ContextBlock<unknown>): Promise<TokenEstimate> {
+    try {
+      // Serialize block to text
+      const text = serializeBlockForEstimation(block);
+
+      // Use Anthropic API for exact count
+      const result = await this.client.messages.countTokens({
+        model: this.model,
+        messages: [
+          {
+            role: 'user',
+            content: text,
+          },
+        ],
+      });
+
+      return {
+        tokens: result.input_tokens,
+        confidence: 'exact',
+      };
+    } catch (error) {
+      // Fallback to heuristic on API error
+      console.warn(
+        '[AnthropicTokenEstimator] API error, falling back to heuristic:',
+        error
+      );
+      const text = serializeBlockForEstimation(block);
+      return {
+        tokens: heuristicTokenCount(text),
+        confidence: 'low',
+      };
+    }
+  }
+
+  /**
+   * Estimate tokens for multiple blocks.
+   * Batches API calls for efficiency.
+   *
+   * @param blocks - Blocks to estimate
+   * @returns Token estimate (exact confidence if all succeed)
+   */
+  async estimate(blocks: ContextBlock<unknown>[]): Promise<TokenEstimate> {
+    if (blocks.length === 0) {
+      return { tokens: 0, confidence: 'exact' };
+    }
+
+    try {
+      // Concatenate all block texts
+      const combinedText = blocks
+        .map((block) => serializeBlockForEstimation(block))
+        .join('\n\n');
+
+      // Single API call for all blocks
+      const result = await this.client.messages.countTokens({
+        model: this.model,
+        messages: [
+          {
+            role: 'user',
+            content: combinedText,
+          },
+        ],
+      });
+
+      return {
+        tokens: result.input_tokens,
+        confidence: 'exact',
+      };
+    } catch (error) {
+      // Fallback: sum individual heuristic estimates
+      console.warn(
+        '[AnthropicTokenEstimator] API error, falling back to heuristic:',
+        error
+      );
+
+      let totalTokens = 0;
+      for (const block of blocks) {
+        const text = serializeBlockForEstimation(block);
+        totalTokens += heuristicTokenCount(text);
+      }
+
+      return {
+        tokens: totalTokens,
+        confidence: 'low',
+      };
+    }
+  }
+}

package/src/adapters/attachment-resolver.ts

@@ -0,0 +1,295 @@
+/**
+ * AttachmentResolver: Versioned resolution of attachments with provenance.
+ *
+ * Responsibilities:
+ * - Resolve attachment references to actual content
+ * - Support different resolution levels (metadata_only, extract, full)
+ * - Generate derived blocks with provenance tracking
+ * - Compute snapshot hashes for reproducibility
+ */
+
+import type { ContextBlock, BlockKind } from '../types/block.js';
+import type { AttachmentRef, AttachmentMeta, AttachmentMimeType } from '../types/attachment.js';
+import { computeBlockHash } from '../types/hash.js';
+
+/**
+ * Resolution level for attachments.
+ */
+export type AttachmentResolutionLevel =
+  | 'metadata_only'  // Only metadata (filename, size, type)
+  | 'extract'        // Extract text/structured data (PDFs -> text, images -> OCR)
+  | 'full';          // Full content (base64-encoded for images)
+
+/**
+ * Resolved attachment part (for multi-part content).
+ */
+export interface AttachmentPart {
+  /** Part type */
+  type: 'text' | 'image' | 'json';
+
+  /** Text content (for text parts) */
+  text?: string;
+
+  /** Image content (base64-encoded, for image parts) */
+  image?: {
+    data: string;
+    mimeType: AttachmentMimeType;
+  };
+
+  /** JSON content (for json parts) */
+  json?: unknown;
+
+  /** Optional description */
+  description?: string;
+}
+
+/**
+ * Resolved attachment with content and derived blocks.
+ * Extended version with provenance tracking.
+ */
+export interface ResolvedAttachmentWithProvenance extends AttachmentMeta {
+  /** Resolution level used */
+  level: AttachmentResolutionLevel;
+
+  /** Resolved content parts */
+  parts: AttachmentPart[];
+
+  /** Derived blocks (generated from attachment content) */
+  derivedBlocks: ContextBlock[];
+
+  /** Snapshot hash (for reproducibility) */
+  snapshotHash: string;
+
+  /** Resolver version (for debugging) */
+  resolverVersion: string;
+}
+
+/**
+ * Attachment resolver interface.
+ */
+export interface AttachmentResolver {
+  /** Resolver identifier */
+  resolverId: string;
+
+  /** Resolver version */
+  version: string;
+
+  /**
+   * Resolve an attachment reference to actual content.
+   *
+   * @param ref - Attachment reference
+   * @param level - Resolution level
+   * @returns Resolved attachment with derived blocks
+   */
+  resolve(
+    ref: AttachmentRef,
+    level: AttachmentResolutionLevel
+  ): Promise<ResolvedAttachmentWithProvenance>;
+}
+
+/**
+ * Create a snapshot hash for a resolved attachment.
+ * Used for reproducibility and change detection.
+ *
+ * @param attachmentId - Attachment ID
+ * @param level - Resolution level
+ * @param parts - Resolved parts
+ * @param resolverVersion - Resolver version
+ * @returns Snapshot hash
+ */
+export function createSnapshotHash(
+  attachmentId: string,
+  level: AttachmentResolutionLevel,
+  parts: AttachmentPart[],
+  resolverVersion: string
+): string {
+  const snapshot = {
+    attachmentId,
+    level,
+    parts: parts.map((part) => ({
+      type: part.type,
+      // Hash content without including full data
+      textHash: part.text ? computeBlockHash({ kind: 'reference' as BlockKind, sensitivity: 'public', codecId: 'text', codecVersion: '1.0.0' }, { text: part.text }) : undefined,
+      imageHash: part.image ? computeBlockHash({ kind: 'reference' as BlockKind, sensitivity: 'public', codecId: 'image', codecVersion: '1.0.0' }, { data: part.image.data, mimeType: part.image.mimeType }) : undefined,
+      jsonHash: part.json ? computeBlockHash({ kind: 'reference' as BlockKind, sensitivity: 'public', codecId: 'json', codecVersion: '1.0.0' }, part.json) : undefined,
+    })),
+    resolverVersion,
+  };
+
+  return computeBlockHash(
+    { kind: 'reference' as BlockKind, sensitivity: 'public', codecId: 'attachment-snapshot', codecVersion: '1.0.0' },
+    snapshot
+  );
+}
+
+/**
+ * Create derived blocks from resolved attachment parts.
+ *
+ * @param attachmentId - Attachment ID
+ * @param parts - Resolved parts
+ * @param parentHash - Parent block hash (for provenance)
+ * @returns Array of derived blocks
+ */
+export function createDerivedBlocks(
+  attachmentId: string,
+  parts: AttachmentPart[],
+  parentHash: string
+): ContextBlock[] {
+  const blocks: ContextBlock[] = [];
+
+  for (let i = 0; i < parts.length; i++) {
+    const part = parts[i];
+
+    if (part.type === 'text' && part.text) {
+      const payload = {
+        text: part.text,
+        source: attachmentId,
+        partIndex: i,
+        description: part.description,
+      };
+
+      const blockHash = computeBlockHash(
+        { kind: 'reference', sensitivity: 'public', codecId: 'attachment-text', codecVersion: '1.0.0' },
+        payload
+      );
+
+      blocks.push({
+        blockHash,
+        meta: {
+          kind: 'reference',
+          sensitivity: 'public',
+          codecId: 'attachment-text',
+          codecVersion: '1.0.0',
+          createdAt: Math.floor(Date.now() / 1000),
+          source: attachmentId,
+        },
+        payload,
+      });
+    } else if (part.type === 'image' && part.image) {
+      const payload = {
+        data: part.image.data,
+        mimeType: part.image.mimeType,
+        source: attachmentId,
+        partIndex: i,
+        description: part.description,
+      };
+
+      const blockHash = computeBlockHash(
+        { kind: 'reference', sensitivity: 'public', codecId: 'attachment-image', codecVersion: '1.0.0' },
+        payload
+      );
+
+      blocks.push({
+        blockHash,
+        meta: {
+          kind: 'reference',
+          sensitivity: 'public',
+          codecId: 'attachment-image',
+          codecVersion: '1.0.0',
+          createdAt: Math.floor(Date.now() / 1000),
+          source: attachmentId,
+        },
+        payload,
+      });
+    } else if (part.type === 'json' && part.json) {
+      const payload = {
+        data: part.json,
+        source: attachmentId,
+        partIndex: i,
+        description: part.description,
+      };
+
+      const blockHash = computeBlockHash(
+        { kind: 'reference', sensitivity: 'public', codecId: 'attachment-json', codecVersion: '1.0.0' },
+        payload
+      );
+
+      blocks.push({
+        blockHash,
+        meta: {
+          kind: 'reference',
+          sensitivity: 'public',
+          codecId: 'attachment-json',
+          codecVersion: '1.0.0',
+          createdAt: Math.floor(Date.now() / 1000),
+          source: attachmentId,
+        },
+        payload,
+      });
+    }
+  }
+
+  return blocks;
+}
+
+/**
+ * Default attachment resolver (stub implementation).
+ * In production, this would integrate with actual storage backends (S3, GCS, etc.).
+ */
+export class DefaultAttachmentResolver implements AttachmentResolver {
+  resolverId = 'default-resolver';
+  version = '1.0.0';
+
+  async resolve(
+    ref: AttachmentRef,
+    level: AttachmentResolutionLevel
+  ): Promise<ResolvedAttachmentWithProvenance> {
+    // Stub implementation - in production, this would:
+    // 1. Fetch attachment metadata from storage
+    // 2. Fetch content based on resolution level
+    // 3. Process content (extract text, OCR images, etc.)
+    // 4. Generate derived blocks
+
+    // For now, return a minimal resolved attachment
+    const parts: AttachmentPart[] = [];
+
+    if (level === 'metadata_only') {
+      // No content parts for metadata_only
+    } else if (level === 'extract') {
+      // Extract text/structured data
+      parts.push({
+        type: 'text',
+        text: `Extracted content from ${ref.attachmentId}`,
+        description: ref.description,
+      });
+    } else if (level === 'full') {
+      // Full content (would be base64-encoded image data in production)
+      parts.push({
+        type: 'text',
+        text: `Full content of ${ref.attachmentId}`,
+        description: ref.description,
+      });
+    }
+
+    const snapshotHash = createSnapshotHash(
+      ref.attachmentId,
+      level,
+      parts,
+      this.version
+    );
+
+    const derivedBlocks = createDerivedBlocks(
+      ref.attachmentId,
+      parts,
+      ref.attachmentId // Parent hash (would be actual block hash in production)
+    );
+
+    return {
+      // Metadata (would be fetched from storage in production)
+      attachmentId: ref.attachmentId,
+      mimeType: 'text/plain',
+      sizeBytes: 0,
+      filename: ref.attachmentId,
+      storage: 'local',
+      storagePath: '',
+      createdAt: Math.floor(Date.now() / 1000),
+
+      // Resolution results
+      level,
+      parts,
+      derivedBlocks,
+      snapshotHash,
+      resolverVersion: this.version,
+    };
+  }
+}

package/src/adapters/attachment-selector.ts

@@ -0,0 +1,218 @@
+/**
+ * AttachmentSelector: Token budget-aware attachment selection.
+ *
+ * Selects attachments based on policy-driven ranking with deterministic ordering.
+ */
+
+import type { ResolvedAttachment, AttachmentRef } from '../types/attachment.js';
+import type { AttachmentPolicy, AttachmentPurpose, RankingCriterion } from '../types/policy.js';
+import type { TokenEstimator } from './token-estimator.js';
+
+/**
+ * Attachment with selection metadata.
+ */
+export interface RankedAttachment extends ResolvedAttachment {
+  /** Attachment purpose (for ranking) */
+  purpose: AttachmentPurpose;
+
+  /** User explicitly mentioned this attachment */
+  userMention: boolean;
+
+  /** Ranking score (lower = higher priority) */
+  rankScore: number;
+}
+
+/**
+ * Selected attachments result.
+ */
+export interface SelectedAttachments {
+  /** Selected attachments (within budget) */
+  selected: ResolvedAttachment[];
+
+  /** Excluded attachments (over budget) */
+  excluded: AttachmentRef[];
+
+  /** Total tokens used by selected attachments */
+  tokensUsed: number;
+}
+
+/**
+ * Compute rank score based on selection strategy.
+ *
+ * @param attachment - Ranked attachment
+ * @param rankBy - Ranking criteria in priority order
+ * @param purposePriority - Purpose priority mapping
+ * @returns Rank score (lower = higher priority)
+ */
+function computeRankScore(
+  attachment: RankedAttachment,
+  rankBy: RankingCriterion[],
+  purposePriority: Record<AttachmentPurpose, number>
+): number {
+  let score = 0;
+  let multiplier = 1000; // High multiplier for primary criterion
+
+  for (const criterion of rankBy) {
+    switch (criterion) {
+      case 'purpose':
+        score += purposePriority[attachment.purpose] * multiplier;
+        break;
+      case 'user_mention':
+        // User mention: 0 if mentioned, 1 if not
+        score += (attachment.userMention ? 0 : 1) * multiplier;
+        break;
+      case 'recency':
+        // More recent = lower score (inverted createdAt)
+        score += (Date.now() / 1000 - attachment.createdAt) * multiplier * 0.01;
+        break;
+    }
+
+    multiplier /= 100; // Reduce multiplier for next criterion
+  }
+
+  return score;
+}
+
+/**
+ * Estimate tokens for an attachment.
+ *
+ * @param attachment - Resolved attachment
+ * @returns Estimated tokens
+ */
+function estimateAttachmentTokens(attachment: ResolvedAttachment): number {
+  // For text attachments, estimate from text content
+  if (attachment.text) {
+    // Rough estimate: 1 token per 4 characters
+    return Math.ceil(attachment.text.length / 4);
+  }
+
+  // For images, use a heuristic based on size
+  // (actual token count depends on image dimensions and provider)
+  if (attachment.mimeType.startsWith('image/')) {
+    // Rough estimate: ~85 tokens per 512x512 tile
+    // Assume average image is ~4 tiles
+    return 340;
+  }
+
+  // For PDFs and JSON, use a conservative estimate
+  if (attachment.mimeType === 'application/pdf' || attachment.mimeType === 'application/json') {
+    // Estimate based on content length if available
+    if (attachment.content) {
+      // Base64 content: approximate text length
+      const textLength = Math.floor(attachment.content.length * 0.75);
+      return Math.ceil(textLength / 4); // Rough token estimate
+    }
+    // Fallback: 500 tokens per attachment
+    return 500;
+  }
+
+  // Default fallback
+  return 100;
+}
+
+/**
+ * AttachmentSelector: Select attachments with policy enforcement.
+ */
+export class AttachmentSelector {
+  constructor(
+    private readonly policy: AttachmentPolicy,
+    private readonly estimator: TokenEstimator
+  ) {}
+
+  /**
+   * Select attachments within token budget.
+   * Deterministic: same inputs → same selection order.
+   *
+   * @param attachments - Ranked attachments to select from
+   * @returns Selected attachments result
+   */
+  async selectAttachments(
+    attachments: RankedAttachment[]
+  ): Promise<SelectedAttachments> {
+    const { maxTokensTotal, selectionStrategy } = this.policy;
+    const { rankBy, purposePriority } = selectionStrategy;
+
+    // Use default purpose priority if not provided
+    const effectivePurposePriority = purposePriority ?? {
+      evidence: 1,
+      input: 2,
+      context: 3,
+      artifact: 4,
+    };
+
+    // Compute rank scores for all attachments
+    const scoredAttachments = attachments.map((attachment) => ({
+      ...attachment,
+      rankScore: computeRankScore(attachment, rankBy, effectivePurposePriority),
+    }));
+
+    // Sort by rank score (ascending - lower score = higher priority)
+    // Use attachmentId as tiebreaker for deterministic ordering
+    scoredAttachments.sort((a, b) => {
+      if (a.rankScore !== b.rankScore) {
+        return a.rankScore - b.rankScore;
+      }
+      return a.attachmentId.localeCompare(b.attachmentId);
+    });
+
+    // Select attachments within budget
+    const selected: ResolvedAttachment[] = [];
+    const excluded: AttachmentRef[] = [];
+    let tokensUsed = 0;
+
+    for (const attachment of scoredAttachments) {
+      // Estimate tokens for this attachment
+      const attachmentTokens = estimateAttachmentTokens(attachment);
+
+      // Check if adding this attachment would exceed budget
+      if (tokensUsed + attachmentTokens > maxTokensTotal) {
+        // Budget exceeded - exclude this attachment
+        excluded.push({
+          attachmentId: attachment.attachmentId,
+          description: attachment.filename,
+        });
+        continue;
+      }
+
+      // Include attachment
+      selected.push(attachment);
+      tokensUsed += attachmentTokens;
+    }
+
+    return {
+      selected,
+      excluded,
+      tokensUsed,
+    };
+  }
+
+  /**
+   * Select attachments from raw list with purpose/mention metadata.
+   * Convenience method that wraps selectAttachments.
+   *
+   * @param attachments - Resolved attachments
+   * @param metadata - Per-attachment metadata
+   * @returns Selected attachments result
+   */
+  async selectFromList(
+    attachments: ResolvedAttachment[],
+    metadata: Map<string, { purpose: AttachmentPurpose; userMention: boolean }>
+  ): Promise<SelectedAttachments> {
+    // Build ranked attachments
+    const ranked: RankedAttachment[] = attachments.map((attachment) => {
+      const meta = metadata.get(attachment.attachmentId) ?? {
+        purpose: 'context' as const,
+        userMention: false,
+      };
+
+      return {
+        ...attachment,
+        purpose: meta.purpose,
+        userMention: meta.userMention,
+        rankScore: 0, // Will be computed in selectAttachments
+      };
+    });
+
+    return this.selectAttachments(ranked);
+  }
+}