mdcontext 0.1.0 → 0.2.0

package/src/summarization/types.ts

@@ -0,0 +1,184 @@
+/**
+ * Summarization Module Types
+ *
+ * Core interfaces for AI-powered summarization of search results.
+ * Supports both CLI-based providers (free with subscriptions) and
+ * API-based providers (pay-per-use via Vercel AI SDK).
+ */
+
+/**
+ * Summarization mode - CLI providers are free, API providers cost money
+ */
+export type SummarizationMode = 'cli' | 'api'
+
+/**
+ * Known CLI tools that can be used for summarization
+ */
+export type CLIProviderName =
+  | 'claude'
+  | 'copilot'
+  | 'cline'
+  | 'aider'
+  | 'opencode'
+  | 'amp'
+
+/**
+ * Known API providers for summarization
+ */
+export type APIProviderName =
+  | 'deepseek'
+  | 'anthropic'
+  | 'openai'
+  | 'gemini'
+  | 'qwen'
+
+/**
+ * Information about a detected CLI tool
+ */
+export interface CLIInfo {
+  /** Internal name identifier */
+  readonly name: CLIProviderName
+  /** Command to execute */
+  readonly command: string
+  /** Display name for UI */
+  readonly displayName: string
+  /** Arguments to pass to the CLI for summarization */
+  readonly args: readonly string[]
+  /** Whether to use stdin for input */
+  readonly useStdin: boolean
+}
+
+/**
+ * API provider pricing information
+ */
+export interface APIProviderPricing {
+  /** Provider name */
+  readonly provider: APIProviderName
+  /** Display name */
+  readonly displayName: string
+  /** Cost per million input tokens */
+  readonly inputCostPer1M: number
+  /** Cost per million output tokens */
+  readonly outputCostPer1M: number
+  /** Default model to use */
+  readonly defaultModel: string
+  /** Base URL for the API */
+  readonly baseURL: string
+  /** Environment variable for API key */
+  readonly apiKeyEnvVar: string
+}
+
+/**
+ * Result from a summarization operation
+ */
+export interface SummaryResult {
+  /** The generated summary text */
+  readonly summary: string
+  /** Provider that generated the summary */
+  readonly provider: CLIProviderName | APIProviderName
+  /** Mode used (cli or api) */
+  readonly mode: SummarizationMode
+  /** Estimated cost in USD (0 for CLI providers) */
+  readonly estimatedCost: number
+  /** Time taken in milliseconds */
+  readonly durationMs: number
+  /** Token count estimates (if available) */
+  readonly tokens?: {
+    readonly input: number
+    readonly output: number
+  }
+}
+
+/**
+ * Core summarizer interface - simple and focused
+ *
+ * Each provider just needs to implement this interface.
+ * CLI providers return 0 for cost, API providers calculate actual costs.
+ */
+export interface Summarizer {
+  /** Generate a summary from input text */
+  summarize(input: string, prompt: string): Promise<SummaryResult>
+
+  /** Estimate cost before running (optional, defaults to 0 for CLI) */
+  estimateCost?(inputTokens: number): number
+
+  /** Check if the provider is available/configured */
+  isAvailable(): Promise<boolean>
+}
+
+/**
+ * Options for streaming summarization
+ */
+export interface StreamOptions {
+  /** Callback for each chunk of text */
+  onChunk: (chunk: string) => void
+  /** Callback when streaming completes */
+  onComplete?: (result: SummaryResult) => void
+  /** Callback on error */
+  onError?: (error: Error) => void
+}
+
+/**
+ * Extended summarizer interface with streaming support
+ */
+export interface StreamingSummarizer extends Summarizer {
+  /** Generate a summary with streaming output */
+  summarizeStream(
+    input: string,
+    prompt: string,
+    options: StreamOptions,
+  ): Promise<void>
+}
+
+/**
+ * Configuration for AI summarization (distinct from existing SummarizationConfig)
+ *
+ * This configures the AI provider for generating summaries, not the
+ * token budget settings in the existing SummarizationConfig.
+ */
+export interface AISummarizationConfig {
+  /** Mode: 'cli' (free) or 'api' (pay-per-use) */
+  readonly mode: SummarizationMode
+  /** Provider name */
+  readonly provider: CLIProviderName | APIProviderName
+  /** Model name (for API providers) */
+  readonly model?: string
+  /** Enable streaming output */
+  readonly stream?: boolean
+  /** Custom API base URL */
+  readonly baseURL?: string
+  /** API key (for API providers, usually from env) */
+  readonly apiKey?: string
+}
+
+/**
+ * Factory function type for creating summarizers
+ */
+export type SummarizerFactory = (
+  config: AISummarizationConfig,
+) => Promise<Summarizer>
+
+/**
+ * Error types specific to summarization
+ */
+export class SummarizationError extends Error {
+  constructor(
+    message: string,
+    public readonly code: SummarizationErrorCode,
+    public readonly provider?: string,
+    public readonly cause?: Error,
+  ) {
+    super(message)
+    this.name = 'SummarizationError'
+  }
+}
+
+export type SummarizationErrorCode =
+  | 'PROVIDER_NOT_FOUND'
+  | 'PROVIDER_NOT_AVAILABLE'
+  | 'CLI_EXECUTION_FAILED'
+  | 'API_REQUEST_FAILED'
+  | 'RATE_LIMITED'
+  | 'INVALID_RESPONSE'
+  | 'TIMEOUT'
+  | 'NO_API_KEY'

package/src/summarize/summarizer.ts

@@ -7,11 +7,19 @@
 import * as fs from 'node:fs/promises'
 import * as path from 'node:path'
 import { Effect } from 'effect'
-import type { MdDocument, MdSection } from '../core/types.js'
+import type { MdDocument, MdSection, ParseError } from '../core/types.js'
+import type { FileReadError } from '../errors/index.js'
 import { parseFile } from '../parser/parser.js'
+import { filterDocumentSections } from '../parser/section-filter.js'
 import { countTokensApprox } from '../utils/tokens.js'
 import { formatSummary as formatSummaryImpl } from './formatters.js'
 
+/**
+ * Error type from parseFile function
+ * Uses centralized errors from src/errors/index.ts
+ */
+type ParseFileError = ParseError | FileReadError
+
 // ============================================================================
 // Types
 // ============================================================================
@@ -23,6 +31,8 @@ export interface SummarizeOptions {
   readonly level?: CompressionLevel | undefined
   /** Maximum tokens for output */
   readonly maxTokens?: number | undefined
+  /** Section patterns to exclude from output */
+  readonly exclude?: readonly string[] | undefined
 }
 
 export interface SectionSummary {
@@ -56,6 +66,8 @@ export interface AssembleContextOptions {
   readonly budget: number
   /** Compression level for each source */
   readonly level?: CompressionLevel | undefined
+  /** Section patterns to exclude from output */
+  readonly exclude?: readonly string[] | undefined
 }
 
 export interface AssembledContext {
@@ -375,14 +387,27 @@ export const summarizeDocument = (
   return result
 }
 
+/**
+ * Summarize a markdown file
+ *
+ * @throws ParseError - File content cannot be parsed
+ * @throws FileReadError - File cannot be read from filesystem
+ */
 export const summarizeFile = (
   filePath: string,
   options: SummarizeOptions = {},
-): Effect.Effect<DocumentSummary, Error> =>
+): Effect.Effect<DocumentSummary, ParseFileError> =>
   Effect.gen(function* () {
-    const document = yield* parseFile(filePath).pipe(
-      Effect.mapError((e) => new Error(`${e._tag}: ${e.message}`)),
-    )
+    let document = yield* parseFile(filePath)
+
+    // Apply exclusion filter if patterns provided
+    if (options.exclude && options.exclude.length > 0) {
+      const { document: filteredDoc } = filterDocumentSections(
+        document,
+        options.exclude,
+      )
+      document = filteredDoc
+    }
 
     return summarizeDocument(document, options)
   })
@@ -397,14 +422,21 @@ export { type FormatSummaryOptions, formatSummary } from './formatters.js'
 // Multi-Document Context Assembly
 // ============================================================================
 
+/**
+ * Assemble context from multiple markdown files within a token budget
+ *
+ * @throws ParseError - File content cannot be parsed
+ * @throws FileReadError - File cannot be read from filesystem
+ */
 export const assembleContext = (
   rootPath: string,
   sourcePaths: readonly string[],
   options: AssembleContextOptions,
-): Effect.Effect<AssembledContext, Error> =>
+): Effect.Effect<AssembledContext, ParseFileError> =>
   Effect.gen(function* () {
     const budget = options.budget
     const level = options.level ?? 'summary'
+    const excludePatterns = options.exclude ?? []
 
     const sources: SourceContext[] = []
     const overflow: string[] = []
@@ -418,33 +450,65 @@ export const assembleContext = (
         ? sourcePath
         : path.join(rootPath, sourcePath)
 
-      try {
-        const summary = yield* summarizeFile(resolvedPath, {
-          level,
-          maxTokens: perSourceBudget,
-        })
-
-        const content = formatSummaryImpl(summary)
-        // Count actual formatted output tokens, not pre-format summary tokens
-        const tokens = countTokensApprox(content)
+      // Use catchAll for graceful degradation - individual file failures
+      // shouldn't stop the entire context assembly operation
+      const summaryResult = yield* summarizeFile(resolvedPath, {
+        level,
+        maxTokens: perSourceBudget,
+        exclude: excludePatterns,
+      }).pipe(
+        Effect.map((s): DocumentSummary | null => s),
+        // Log error for observability before gracefully degrading
+        Effect.tapError((error) =>
+          Effect.logError(`Failed to summarize ${sourcePath}`, error),
+        ),
+        // Note: catchAll intentional for batch processing - individual file
+        // failures add to overflow instead of stopping assembly
+        Effect.catchAll(() => Effect.succeed(null as DocumentSummary | null)),
+      )
+
+      if (!summaryResult) {
+        overflow.push(sourcePath)
+        continue
+      }
 
-        if (totalTokens + tokens <= budget) {
-          sources.push({
-            path: path.relative(rootPath, resolvedPath),
-            title: summary.title,
-            tokens,
-            content,
-          })
-          totalTokens += tokens
-        } else {
-          // Over budget
-          const remaining = budget - totalTokens
-          if (remaining > MIN_PARTIAL_BUDGET) {
-            // Include partial if we have some room
-            const briefSummary = yield* summarizeFile(resolvedPath, {
-              level: 'brief',
-              maxTokens: remaining,
-            })
+      const summary = summaryResult
+      const content = formatSummaryImpl(summary)
+      // Count actual formatted output tokens, not pre-format summary tokens
+      const tokens = countTokensApprox(content)
+
+      if (totalTokens + tokens <= budget) {
+        sources.push({
+          path: path.relative(rootPath, resolvedPath),
+          title: summary.title,
+          tokens,
+          content,
+        })
+        totalTokens += tokens
+      } else {
+        // Over budget
+        const remaining = budget - totalTokens
+        if (remaining > MIN_PARTIAL_BUDGET) {
+          // Include partial if we have some room
+          const briefSummary = yield* summarizeFile(resolvedPath, {
+            level: 'brief',
+            maxTokens: remaining,
+            exclude: excludePatterns,
+          }).pipe(
+            Effect.map((s): DocumentSummary | null => s),
+            // Log error for observability before gracefully degrading
+            Effect.tapError((error) =>
+              Effect.logError(
+                `Failed to create brief summary for ${sourcePath}`,
+                error,
+              ),
+            ),
+            Effect.catchAll(() =>
+              Effect.succeed(null as DocumentSummary | null),
+            ),
+          )
+
+          if (briefSummary) {
             const briefContent = formatSummaryImpl(briefSummary)
             // Count actual formatted output tokens, not pre-format summary tokens
             const briefTokens = countTokensApprox(briefContent)
@@ -459,10 +523,9 @@ export const assembleContext = (
           } else {
             overflow.push(path.relative(rootPath, resolvedPath))
           }
+        } else {
+          overflow.push(path.relative(rootPath, resolvedPath))
         }
-      } catch (_e) {
-        // Skip files that can't be processed
-        overflow.push(sourcePath)
       }
     }
 
@@ -500,8 +563,14 @@ export const measureReduction = async (
   const originalTokens = countTokensApprox(originalContent)
 
   // Get summary
+  // Note: catchAll is intentional - measureReduction is a utility function
+  // where failures should return default values (no reduction) rather than throw
   const result = await Effect.runPromise(
     summarizeFile(filePath, { level }).pipe(
+      // Log error for observability before gracefully degrading
+      Effect.tapError((error) =>
+        Effect.logError(`Failed to measure reduction for ${filePath}`, error),
+      ),
       Effect.catchAll(() => Effect.succeed(null)),
     ),
   )

package/src/types/huggingface-transformers.d.ts

@@ -0,0 +1,66 @@
+/**
+ * Type declarations for @huggingface/transformers (optional dependency)
+ *
+ * This package is an optional peer dependency used for cross-encoder re-ranking.
+ * Users who want re-ranking can install it with: npm install @huggingface/transformers
+ */
+
+declare module '@huggingface/transformers' {
+  export interface ProgressCallbackData {
+    file?: string
+    progress?: number
+  }
+
+  export type ProgressCallback = (data: ProgressCallbackData) => void
+
+  export interface AutoModelOptions {
+    progress_callback?: ProgressCallback | undefined
+  }
+
+  export interface AutoTokenizerOptions {
+    progress_callback?: ProgressCallback | undefined
+  }
+
+  export interface TokenizerOutput {
+    input_ids: unknown
+    attention_mask: unknown
+  }
+
+  export interface ModelOutput {
+    logits: {
+      data: Float32Array
+    }
+  }
+
+  export const env: {
+    cacheDir: string
+  }
+
+  export type AutoTokenizerInstance = (
+    texts: string[],
+    options: {
+      text_pair?: string[]
+      padding?: boolean
+      truncation?: boolean
+      max_length?: number
+    },
+  ) => TokenizerOutput
+
+  export type AutoModelInstance = (
+    input: TokenizerOutput,
+  ) => Promise<ModelOutput>
+
+  export const AutoTokenizer: {
+    from_pretrained(
+      model: string,
+      options?: AutoTokenizerOptions,
+    ): Promise<AutoTokenizerInstance>
+  }
+
+  export const AutoModelForSequenceClassification: {
+    from_pretrained(
+      model: string,
+      options?: AutoModelOptions,
+    ): Promise<AutoModelInstance>
+  }
+}

package/tests/fixtures/cli/.mdcontext/active-provider.json

@@ -0,0 +1,7 @@
+{
+  "namespace": "openai_text-embedding-3-small_512",
+  "provider": "openai",
+  "model": "text-embedding-3-small",
+  "dimensions": 512,
+  "activatedAt": "2026-01-27T07:43:50.361Z"
+}

package/tests/fixtures/cli/.mdcontext/indexes/documents.json

@@ -1,12 +1,12 @@
 {
   "version": 1,
-  "rootPath": "/Users/alphab/Dev/LLM/DEV/mdcontext/tests/fixtures/cli",
+  "rootPath": "tests/fixtures/cli",
   "documents": {
     "README.md": {
       "id": "04c6e90faac2",
       "path": "README.md",
       "title": "Test Project",
-      "mtime": 1768999918845,
+      "mtime": 1769492431399,
       "hash": "54872b0fdbf6858a",
       "tokenCount": 76,
       "sectionCount": 2
@@ -15,7 +15,7 @@
       "id": "392e93c0f22d",
       "path": "api-reference.md",
       "title": "API Reference",
-      "mtime": 1768999919391,
+      "mtime": 1769492431399,
       "hash": "4a879da54a831235",
       "tokenCount": 109,
       "sectionCount": 2
@@ -24,7 +24,7 @@
       "id": "b6885e1f8555",
       "path": "getting-started.md",
       "title": "Getting Started",
-      "mtime": 1769013198631,
+      "mtime": 1769492431400,
       "hash": "2d44a41d5d2579f2",
       "tokenCount": 66,
       "sectionCount": 3

package/tests/fixtures/cli/.mdcontext/indexes/sections.json

@@ -110,6 +110,8 @@
       "04c6e90faac2-test-project",
       "04c6e90faac2-test-project",
       "04c6e90faac2-test-project",
+      "04c6e90faac2-test-project",
+      "04c6e90faac2-test-project",
       "04c6e90faac2-test-project"
     ],
     "overview": [
@@ -128,6 +130,8 @@
       "04c6e90faac2-overview",
       "04c6e90faac2-overview",
       "04c6e90faac2-overview",
+      "04c6e90faac2-overview",
+      "04c6e90faac2-overview",
       "04c6e90faac2-overview"
     ],
     "api reference": [
@@ -146,6 +150,8 @@
       "392e93c0f22d-api-reference",
       "392e93c0f22d-api-reference",
       "392e93c0f22d-api-reference",
+      "392e93c0f22d-api-reference",
+      "392e93c0f22d-api-reference",
       "392e93c0f22d-api-reference"
     ],
     "endpoints": [
@@ -164,6 +170,8 @@
       "392e93c0f22d-endpoints",
       "392e93c0f22d-endpoints",
       "392e93c0f22d-endpoints",
+      "392e93c0f22d-endpoints",
+      "392e93c0f22d-endpoints",
       "392e93c0f22d-endpoints"
     ],
     "getting started": [
@@ -182,6 +190,8 @@
       "b6885e1f8555-getting-started",
       "b6885e1f8555-getting-started",
       "b6885e1f8555-getting-started",
+      "b6885e1f8555-getting-started",
+      "b6885e1f8555-getting-started",
       "b6885e1f8555-getting-started"
     ],
     "installation": [
@@ -200,6 +210,8 @@
       "b6885e1f8555-installation",
       "b6885e1f8555-installation",
       "b6885e1f8555-installation",
+      "b6885e1f8555-installation",
+      "b6885e1f8555-installation",
       "b6885e1f8555-installation"
     ],
     "usage": [
@@ -218,6 +230,8 @@
       "b6885e1f8555-usage",
       "b6885e1f8555-usage",
       "b6885e1f8555-usage",
+      "b6885e1f8555-usage",
+      "b6885e1f8555-usage",
       "b6885e1f8555-usage"
     ]
   },