@framers/agentos 0.1.112 → 0.1.114

package/dist/core/vision/providers/PipelineVisionProvider.d.ts

@@ -0,0 +1,154 @@
+/**
+ * @module core/vision/providers/PipelineVisionProvider
+ *
+ * Wraps the full {@link VisionPipeline} as an {@link IVisionProvider}
+ * for seamless integration with the {@link MultimodalIndexer}.
+ *
+ * Unlike the simpler {@link LLMVisionProvider} which only calls a cloud
+ * LLM, this provider runs the complete multi-tier pipeline — local OCR,
+ * handwriting recognition, document understanding, and cloud fallback —
+ * before returning the best text description.
+ *
+ * This is the recommended provider when you want the multimodal indexer
+ * to benefit from the full progressive enhancement pipeline.
+ *
+ * @see {@link VisionPipeline} for the underlying pipeline engine.
+ * @see {@link LLMVisionProvider} for the cloud-only alternative.
+ * @see {@link IVisionProvider} for the interface contract.
+ *
+ * @example
+ * ```typescript
+ * import { VisionPipeline, PipelineVisionProvider } from '../../../core/vision';
+ * import { MultimodalIndexer } from '../../../rag/multimodal';
+ *
+ * const pipeline = new VisionPipeline({
+ *   strategy: 'progressive',
+ *   ocr: 'paddle',
+ *   cloudProvider: 'openai',
+ * });
+ *
+ * const indexer = new MultimodalIndexer({
+ *   embeddingManager,
+ *   vectorStore,
+ *   visionProvider: new PipelineVisionProvider(pipeline),
+ * });
+ * ```
+ */
+import type { IVisionProvider } from '../../../rag/multimodal/types.js';
+import type { VisionPipeline } from '../VisionPipeline.js';
+import type { VisionResult } from '../types.js';
+/**
+ * Adapts the full {@link VisionPipeline} to the narrow
+ * {@link IVisionProvider} interface used by the multimodal indexer.
+ *
+ * The pipeline's `process()` method runs all configured tiers and returns
+ * a rich {@link VisionResult}. This adapter extracts just the text field
+ * that the indexer needs for embedding generation.
+ *
+ * For callers that need the full pipeline result (embeddings, layout,
+ * confidence, regions), use `processWithFullResult()` instead.
+ *
+ * @example
+ * ```typescript
+ * const provider = new PipelineVisionProvider(pipeline);
+ *
+ * // Simple: just the description text
+ * const text = await provider.describeImage(imageUrl);
+ *
+ * // Advanced: full pipeline result
+ * const result = await provider.processWithFullResult(imageBuffer);
+ * console.log(result.embedding);  // CLIP vector
+ * console.log(result.layout);     // Florence-2 layout
+ * ```
+ */
+export declare class PipelineVisionProvider implements IVisionProvider {
+    /**
+     * Reference to the underlying vision pipeline.
+     * Held as a readonly reference — the caller is responsible for
+     * disposing the pipeline when they're done with it.
+     */
+    private readonly _pipeline;
+    /**
+     * Create a new pipeline vision provider.
+     *
+     * @param pipeline - An initialized {@link VisionPipeline} instance.
+     *   The caller retains ownership and is responsible for calling
+     *   `pipeline.dispose()` when done.
+     *
+     * @throws {Error} If pipeline is null or undefined.
+     *
+     * @example
+     * ```typescript
+     * const pipeline = new VisionPipeline({ strategy: 'progressive' });
+     * const provider = new PipelineVisionProvider(pipeline);
+     * ```
+     */
+    constructor(pipeline: VisionPipeline);
+    /**
+     * Generate a text description of the provided image by running it
+     * through the full vision pipeline.
+     *
+     * This satisfies the {@link IVisionProvider} contract. The image passes
+     * through all configured tiers (OCR, handwriting, document-ai, cloud)
+     * and the best extracted text is returned.
+     *
+     * @param image - Image as a URL string (https://... or data:image/...).
+     * @returns Text description or extracted content from the image.
+     *
+     * @throws {Error} If all pipeline tiers fail to produce output.
+     * @throws {Error} If the pipeline has been disposed.
+     *
+     * @example
+     * ```typescript
+     * const description = await provider.describeImage(imageUrl);
+     * console.log(description);
+     * ```
+     */
+    describeImage(image: string): Promise<string>;
+    /**
+     * Process an image through the full pipeline and return the complete
+     * {@link VisionResult} — including embeddings, layout, confidence
+     * scores, and per-tier breakdowns.
+     *
+     * Use this when you need more than just the text description (e.g.
+     * to store the CLIP embedding alongside the text embedding in the
+     * vector store).
+     *
+     * @param image - Image data as a Buffer or URL string.
+     * @returns Full vision pipeline result.
+     *
+     * @throws {Error} If all pipeline tiers fail.
+     * @throws {Error} If the pipeline has been disposed.
+     *
+     * @example
+     * ```typescript
+     * const result = await provider.processWithFullResult(imageBuffer);
+     *
+     * // Use both text embedding (via indexer) and image embedding (via CLIP)
+     * if (result.embedding) {
+     *   await imageVectorStore.upsert('images', [{
+     *     id: docId,
+     *     embedding: result.embedding,
+     *     metadata: { text: result.text },
+     *   }]);
+     * }
+     * ```
+     */
+    processWithFullResult(image: Buffer | string): Promise<VisionResult>;
+    /**
+     * Get a reference to the underlying pipeline for direct access.
+     *
+     * Useful when the caller needs to invoke pipeline-specific methods
+     * like `extractText()`, `embed()`, or `analyzeLayout()` that aren't
+     * exposed through the {@link IVisionProvider} interface.
+     *
+     * @returns The underlying VisionPipeline instance.
+     *
+     * @example
+     * ```typescript
+     * const layout = await provider.getPipeline().analyzeLayout(image);
+     * ```
+     */
+    getPipeline(): VisionPipeline;
+}
+//# sourceMappingURL=PipelineVisionProvider.d.ts.map

package/dist/core/vision/providers/PipelineVisionProvider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"PipelineVisionProvider.d.ts","sourceRoot":"","sources":["../../../../src/core/vision/providers/PipelineVisionProvider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kCAAkC,CAAC;AACxE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,sBAAuB,YAAW,eAAe;IAC5D;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAiB;IAE3C;;;;;;;;;;;;;;OAcG;gBACS,QAAQ,EAAE,cAAc;IASpC;;;;;;;;;;;;;;;;;;;OAmBG;IACG,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKnD;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACG,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAI1E;;;;;;;;;;;;;OAaG;IACH,WAAW,IAAI,cAAc;CAG9B"}

package/dist/core/vision/providers/PipelineVisionProvider.js

@@ -0,0 +1,160 @@
+/**
+ * @module core/vision/providers/PipelineVisionProvider
+ *
+ * Wraps the full {@link VisionPipeline} as an {@link IVisionProvider}
+ * for seamless integration with the {@link MultimodalIndexer}.
+ *
+ * Unlike the simpler {@link LLMVisionProvider} which only calls a cloud
+ * LLM, this provider runs the complete multi-tier pipeline — local OCR,
+ * handwriting recognition, document understanding, and cloud fallback —
+ * before returning the best text description.
+ *
+ * This is the recommended provider when you want the multimodal indexer
+ * to benefit from the full progressive enhancement pipeline.
+ *
+ * @see {@link VisionPipeline} for the underlying pipeline engine.
+ * @see {@link LLMVisionProvider} for the cloud-only alternative.
+ * @see {@link IVisionProvider} for the interface contract.
+ *
+ * @example
+ * ```typescript
+ * import { VisionPipeline, PipelineVisionProvider } from '../../../core/vision/index.js';
+ * import { MultimodalIndexer } from '../../../rag/multimodal/index.js';
+ *
+ * const pipeline = new VisionPipeline({
+ *   strategy: 'progressive',
+ *   ocr: 'paddle',
+ *   cloudProvider: 'openai',
+ * });
+ *
+ * const indexer = new MultimodalIndexer({
+ *   embeddingManager,
+ *   vectorStore,
+ *   visionProvider: new PipelineVisionProvider(pipeline),
+ * });
+ * ```
+ */
+// ---------------------------------------------------------------------------
+// Implementation
+// ---------------------------------------------------------------------------
+/**
+ * Adapts the full {@link VisionPipeline} to the narrow
+ * {@link IVisionProvider} interface used by the multimodal indexer.
+ *
+ * The pipeline's `process()` method runs all configured tiers and returns
+ * a rich {@link VisionResult}. This adapter extracts just the text field
+ * that the indexer needs for embedding generation.
+ *
+ * For callers that need the full pipeline result (embeddings, layout,
+ * confidence, regions), use `processWithFullResult()` instead.
+ *
+ * @example
+ * ```typescript
+ * const provider = new PipelineVisionProvider(pipeline);
+ *
+ * // Simple: just the description text
+ * const text = await provider.describeImage(imageUrl);
+ *
+ * // Advanced: full pipeline result
+ * const result = await provider.processWithFullResult(imageBuffer);
+ * console.log(result.embedding);  // CLIP vector
+ * console.log(result.layout);     // Florence-2 layout
+ * ```
+ */
+export class PipelineVisionProvider {
+    /**
+     * Create a new pipeline vision provider.
+     *
+     * @param pipeline - An initialized {@link VisionPipeline} instance.
+     *   The caller retains ownership and is responsible for calling
+     *   `pipeline.dispose()` when done.
+     *
+     * @throws {Error} If pipeline is null or undefined.
+     *
+     * @example
+     * ```typescript
+     * const pipeline = new VisionPipeline({ strategy: 'progressive' });
+     * const provider = new PipelineVisionProvider(pipeline);
+     * ```
+     */
+    constructor(pipeline) {
+        if (!pipeline) {
+            throw new Error('PipelineVisionProvider: pipeline instance is required.');
+        }
+        this._pipeline = pipeline;
+    }
+    /**
+     * Generate a text description of the provided image by running it
+     * through the full vision pipeline.
+     *
+     * This satisfies the {@link IVisionProvider} contract. The image passes
+     * through all configured tiers (OCR, handwriting, document-ai, cloud)
+     * and the best extracted text is returned.
+     *
+     * @param image - Image as a URL string (https://... or data:image/...).
+     * @returns Text description or extracted content from the image.
+     *
+     * @throws {Error} If all pipeline tiers fail to produce output.
+     * @throws {Error} If the pipeline has been disposed.
+     *
+     * @example
+     * ```typescript
+     * const description = await provider.describeImage(imageUrl);
+     * console.log(description);
+     * ```
+     */
+    async describeImage(image) {
+        const result = await this._pipeline.process(image);
+        return result.text;
+    }
+    /**
+     * Process an image through the full pipeline and return the complete
+     * {@link VisionResult} — including embeddings, layout, confidence
+     * scores, and per-tier breakdowns.
+     *
+     * Use this when you need more than just the text description (e.g.
+     * to store the CLIP embedding alongside the text embedding in the
+     * vector store).
+     *
+     * @param image - Image data as a Buffer or URL string.
+     * @returns Full vision pipeline result.
+     *
+     * @throws {Error} If all pipeline tiers fail.
+     * @throws {Error} If the pipeline has been disposed.
+     *
+     * @example
+     * ```typescript
+     * const result = await provider.processWithFullResult(imageBuffer);
+     *
+     * // Use both text embedding (via indexer) and image embedding (via CLIP)
+     * if (result.embedding) {
+     *   await imageVectorStore.upsert('images', [{
+     *     id: docId,
+     *     embedding: result.embedding,
+     *     metadata: { text: result.text },
+     *   }]);
+     * }
+     * ```
+     */
+    async processWithFullResult(image) {
+        return this._pipeline.process(image);
+    }
+    /**
+     * Get a reference to the underlying pipeline for direct access.
+     *
+     * Useful when the caller needs to invoke pipeline-specific methods
+     * like `extractText()`, `embed()`, or `analyzeLayout()` that aren't
+     * exposed through the {@link IVisionProvider} interface.
+     *
+     * @returns The underlying VisionPipeline instance.
+     *
+     * @example
+     * ```typescript
+     * const layout = await provider.getPipeline().analyzeLayout(image);
+     * ```
+     */
+    getPipeline() {
+        return this._pipeline;
+    }
+}
+//# sourceMappingURL=PipelineVisionProvider.js.map

package/dist/core/vision/providers/PipelineVisionProvider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"PipelineVisionProvider.js","sourceRoot":"","sources":["../../../../src/core/vision/providers/PipelineVisionProvider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAMH,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,OAAO,sBAAsB;IAQjC;;;;;;;;;;;;;;OAcG;IACH,YAAY,QAAwB;QAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CACb,wDAAwD,CACzD,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED;;;;;;;;;;;;;;;;;;;OAmBG;IACH,KAAK,CAAC,aAAa,CAAC,KAAa;QAC/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACnD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA4BG;IACH,KAAK,CAAC,qBAAqB,CAAC,KAAsB;QAChD,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACvC,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,WAAW;QACT,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;CACF"}

package/dist/core/vision/types.d.ts

@@ -0,0 +1,286 @@
+/**
+ * @module core/vision/types
+ *
+ * Type definitions for the unified vision pipeline.
+ *
+ * The vision pipeline processes images through configurable tiers — from fast,
+ * free, offline OCR (PaddleOCR / Tesseract.js) through local vision models
+ * (TrOCR, Florence-2, CLIP via HuggingFace Transformers) to cloud vision LLMs
+ * (GPT-4o, Claude, Gemini). Each tier adds progressively richer understanding
+ * at increasing cost and latency.
+ *
+ * ## Tier overview
+ *
+ * | Tier | Provider | Capability | Cost |
+ * |------|----------|-----------|------|
+ * | 1 — OCR | PaddleOCR / Tesseract.js | Printed text extraction | Free/offline |
+ * | 2 — Local Vision | TrOCR / Florence-2 / CLIP | Handwriting, layout, embeddings | Free/offline |
+ * | 3 — Cloud Vision | GPT-4o / Claude / Gemini | Scene understanding, complex docs | API cost |
+ *
+ * @see {@link VisionPipeline} for the orchestration engine.
+ * @see {@link VisionStrategy} for how tiers are combined.
+ */
+/**
+ * Strategy for how vision tiers are combined during processing.
+ *
+ * - `'progressive'` — Try local OCR first, enhance with local vision models,
+ *   upgrade with cloud only when confidence is below threshold. Best balance
+ *   of cost and quality.
+ * - `'local-only'` — Tiers 1 + 2 only, never call cloud APIs. For air-gapped
+ *   or privacy-sensitive environments.
+ * - `'cloud-only'` — Skip local processing entirely, send directly to cloud
+ *   vision LLM. Highest quality but highest cost.
+ * - `'parallel'` — Run local and cloud simultaneously, merge the best results.
+ *   Lowest latency for high-quality output when cost is not a concern.
+ */
+export type VisionStrategy = 'progressive' | 'local-only' | 'cloud-only' | 'parallel';
+/**
+ * What kind of visual content the pipeline detected or was told to expect.
+ *
+ * Used to route images to the most appropriate processing tier — e.g.
+ * handwritten content triggers TrOCR, complex layouts trigger Florence-2.
+ */
+export type ContentCategory = 'printed-text' | 'handwritten' | 'document-layout' | 'photograph' | 'diagram' | 'screenshot' | 'mixed';
+/**
+ * Identifies which processing tier produced a result.
+ *
+ * - `'ocr'` — Tier 1: PaddleOCR or Tesseract.js text extraction
+ * - `'handwriting'` — Tier 2a: TrOCR handwriting recognition
+ * - `'document-ai'` — Tier 2b: Florence-2 document understanding
+ * - `'embedding'` — Tier 2c: CLIP image embedding generation
+ * - `'cloud-vision'` — Tier 3: Cloud vision LLM (GPT-4o, Claude, etc.)
+ */
+export type VisionTier = 'ocr' | 'handwriting' | 'document-ai' | 'embedding' | 'cloud-vision';
+/**
+ * Result from a single processing tier.
+ *
+ * Each tier that runs produces one of these, regardless of whether the
+ * pipeline ultimately uses its output or prefers a higher-confidence
+ * alternative from another tier.
+ */
+export interface TierResult {
+    /** Which tier produced this result. */
+    tier: VisionTier;
+    /** Provider name within the tier (e.g. 'paddle', 'tesseract', 'openai'). */
+    provider: string;
+    /** Extracted or generated text content. */
+    text: string;
+    /** Confidence score from 0 (no confidence) to 1 (certain). */
+    confidence: number;
+    /** Wall-clock processing time in milliseconds. */
+    durationMs: number;
+    /** Bounding boxes for detected text regions, when available. */
+    regions?: TextRegion[];
+}
+/**
+ * A detected text region within an image, with spatial coordinates
+ * and per-region confidence.
+ */
+export interface TextRegion {
+    /** The text content within this region. */
+    text: string;
+    /** Confidence score for this specific region (0–1). */
+    confidence: number;
+    /**
+     * Bounding box in image coordinates (pixels).
+     * Origin is top-left corner of the image.
+     */
+    bbox: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+}
+/**
+ * Structured document layout produced by Florence-2 or similar
+ * document understanding models.
+ *
+ * Captures the spatial arrangement of text, tables, figures, headings,
+ * and other semantic blocks within a multi-page document.
+ */
+export interface DocumentLayout {
+    /** Pages in document order. */
+    pages: DocumentPage[];
+}
+/**
+ * A single page within a structured document layout.
+ */
+export interface DocumentPage {
+    /** 1-based page number. */
+    pageNumber: number;
+    /** Page width in pixels. */
+    width: number;
+    /** Page height in pixels. */
+    height: number;
+    /** Semantic blocks detected on this page. */
+    blocks: LayoutBlock[];
+}
+/**
+ * A semantic block within a document page — a paragraph, table, figure,
+ * heading, list, or code snippet.
+ */
+export interface LayoutBlock {
+    /** Semantic type of the block. */
+    type: 'text' | 'table' | 'figure' | 'heading' | 'list' | 'code';
+    /** Text content extracted from the block. */
+    content: string;
+    /**
+     * Bounding box in page coordinates (pixels).
+     * Origin is top-left corner of the page.
+     */
+    bbox: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    /** Confidence score for this block detection (0–1). */
+    confidence: number;
+}
+/**
+ * Aggregated result from the vision pipeline after all configured tiers
+ * have run. Contains the best extracted text, content classification,
+ * optional embeddings, and detailed per-tier breakdowns.
+ *
+ * @example
+ * ```typescript
+ * const result = await pipeline.process(imageBuffer);
+ *
+ * // Best extracted text across all tiers
+ * console.log(result.text);
+ *
+ * // What kind of content was detected
+ * console.log(result.category); // 'printed-text' | 'handwritten' | ...
+ *
+ * // CLIP embedding for similarity search
+ * if (result.embedding) {
+ *   await vectorStore.upsert('images', [{ embedding: result.embedding }]);
+ * }
+ *
+ * // Inspect individual tier contributions
+ * for (const tr of result.tierResults) {
+ *   console.log(`${tr.tier} (${tr.provider}): ${tr.confidence}`);
+ * }
+ * ```
+ */
+export interface VisionResult {
+    /** Best extracted text (from OCR, handwriting, or vision description). */
+    text: string;
+    /** Overall confidence score 0–1, taken from the winning tier. */
+    confidence: number;
+    /** What kind of content was detected. */
+    category: ContentCategory;
+    /** Which tier(s) contributed to the final result. */
+    tiers: VisionTier[];
+    /** Detailed results from each tier that ran, ordered by execution. */
+    tierResults: TierResult[];
+    /** CLIP image embedding vector, when embedding tier is enabled. */
+    embedding?: number[];
+    /** Structured document layout, when Florence-2 ran. */
+    layout?: DocumentLayout;
+    /** Bounding boxes for detected text regions from the winning tier. */
+    regions?: TextRegion[];
+    /** Total wall-clock processing time in milliseconds. */
+    durationMs: number;
+}
+/**
+ * Preprocessing options applied to images before they enter the
+ * vision pipeline tiers. Uses `sharp` under the hood.
+ */
+export interface VisionPreprocessingConfig {
+    /** Convert to grayscale before OCR (improves contrast for printed text). */
+    grayscale?: boolean;
+    /**
+     * Resize constraints. The image is scaled down proportionally
+     * so that neither dimension exceeds the specified maximum.
+     * No upscaling is performed.
+     */
+    resize?: {
+        maxWidth?: number;
+        maxHeight?: number;
+    };
+    /** Apply unsharp-mask sharpening (helps blurry scans). */
+    sharpen?: boolean;
+    /** Normalize brightness/contrast (histogram stretching). */
+    normalize?: boolean;
+}
+/**
+ * Configuration for the {@link VisionPipeline}.
+ *
+ * All fields are optional — the factory function {@link createVisionPipeline}
+ * auto-detects available providers and fills in sensible defaults.
+ *
+ * @example
+ * ```typescript
+ * const config: VisionPipelineConfig = {
+ *   strategy: 'progressive',
+ *   ocr: 'paddle',
+ *   handwriting: true,
+ *   documentAI: true,
+ *   embedding: true,
+ *   cloudProvider: 'openai',
+ *   cloudModel: 'gpt-4o',
+ *   confidenceThreshold: 0.8,
+ *   preprocessing: { grayscale: true, sharpen: true },
+ * };
+ * ```
+ */
+export interface VisionPipelineConfig {
+    /**
+     * How to combine tiers.
+     * @default 'progressive'
+     */
+    strategy: VisionStrategy;
+    /**
+     * OCR engine for Tier 1 text extraction.
+     * - `'paddle'` — PaddleOCR (via `ppu-paddle-ocr`). Best accuracy for most scripts.
+     * - `'tesseract'` — Tesseract.js. Wider language support, slightly lower accuracy.
+     * - `'none'` — Skip OCR entirely (useful for photo-only pipelines).
+     * @default 'paddle' (if installed), else 'tesseract' (if installed), else 'none'
+     */
+    ocr?: 'paddle' | 'tesseract' | 'none';
+    /**
+     * Enable handwriting recognition via TrOCR (`@huggingface/transformers`).
+     * Only triggered when OCR confidence is low and content appears handwritten.
+     * @default false
+     */
+    handwriting?: boolean;
+    /**
+     * Enable document understanding via Florence-2 (`@huggingface/transformers`).
+     * Produces structured {@link DocumentLayout} with semantic block detection.
+     * @default false
+     */
+    documentAI?: boolean;
+    /**
+     * Enable CLIP image embeddings (`@huggingface/transformers`).
+     * Runs in parallel with other tiers — does not affect text extraction.
+     * @default false
+     */
+    embedding?: boolean;
+    /**
+     * Cloud vision LLM provider name for Tier 3 fallback.
+     * Must match a provider known to `generateText()` (e.g. 'openai', 'anthropic', 'google').
+     * When unset, cloud vision is disabled.
+     */
+    cloudProvider?: string;
+    /**
+     * Cloud model override. When unset, the provider's default vision model is used.
+     * @example 'gpt-4o', 'claude-sonnet-4-20250514', 'gemini-2.0-flash'
+     */
+    cloudModel?: string;
+    /**
+     * Minimum confidence to accept an OCR result without escalating to cloud.
+     * Only applies to `'progressive'` strategy — if OCR confidence is below
+     * this threshold, the pipeline escalates to the next tier.
+     * @default 0.7
+     */
+    confidenceThreshold?: number;
+    /**
+     * Image preprocessing options applied before any tier runs.
+     * Uses `sharp` for resizing, grayscale conversion, sharpening,
+     * and normalization.
+     */
+    preprocessing?: VisionPreprocessingConfig;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/core/vision/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/core/vision/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAMH;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,cAAc,GACtB,aAAa,GACb,YAAY,GACZ,YAAY,GACZ,UAAU,CAAC;AAMf;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GACvB,cAAc,GACd,aAAa,GACb,iBAAiB,GACjB,YAAY,GACZ,SAAS,GACT,YAAY,GACZ,OAAO,CAAC;AAMZ;;;;;;;;GAQG;AACH,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,aAAa,GAAG,aAAa,GAAG,WAAW,GAAG,cAAc,CAAC;AAM9F;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IACzB,uCAAuC;IACvC,IAAI,EAAE,UAAU,CAAC;IAEjB,4EAA4E;IAC5E,QAAQ,EAAE,MAAM,CAAC;IAEjB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IAEb,8DAA8D;IAC9D,UAAU,EAAE,MAAM,CAAC;IAEnB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IAEnB,gEAAgE;IAChE,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IAEb,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAMD;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,+BAA+B;IAC/B,KAAK,EAAE,YAAY,EAAE,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IAEnB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IAEd,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IAEf,6CAA6C;IAC7C,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,kCAAkC;IAClC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IAEhE,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IAEF,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,YAAY;IAC3B,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAC;IAEb,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IAEnB,yCAAyC;IACzC,QAAQ,EAAE,eAAe,CAAC;IAE1B,qDAAqD;IACrD,KAAK,EAAE,UAAU,EAAE,CAAC;IAEpB,sEAAsE;IACtE,WAAW,EAAE,UAAU,EAAE,CAAC;IAE1B,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB,uDAAuD;IACvD,MAAM,CAAC,EAAE,cAAc,CAAC;IAExB,sEAAsE;IACtE,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IAEvB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,4EAA4E;IAC5E,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;OAIG;IACH,MAAM,CAAC,EAAE;QACP,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,0DAA0D;IAC1D,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,4DAA4D;IAC5D,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,QAAQ,EAAE,cAAc,CAAC;IAEzB;;;;;;OAMG;IACH,GAAG,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,MAAM,CAAC;IAEtC;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;OAIG;IACH,aAAa,CAAC,EAAE,yBAAyB,CAAC;CAC3C"}

package/dist/core/vision/types.js

@@ -0,0 +1,24 @@
+/**
+ * @module core/vision/types
+ *
+ * Type definitions for the unified vision pipeline.
+ *
+ * The vision pipeline processes images through configurable tiers — from fast,
+ * free, offline OCR (PaddleOCR / Tesseract.js) through local vision models
+ * (TrOCR, Florence-2, CLIP via HuggingFace Transformers) to cloud vision LLMs
+ * (GPT-4o, Claude, Gemini). Each tier adds progressively richer understanding
+ * at increasing cost and latency.
+ *
+ * ## Tier overview
+ *
+ * | Tier | Provider | Capability | Cost |
+ * |------|----------|-----------|------|
+ * | 1 — OCR | PaddleOCR / Tesseract.js | Printed text extraction | Free/offline |
+ * | 2 — Local Vision | TrOCR / Florence-2 / CLIP | Handwriting, layout, embeddings | Free/offline |
+ * | 3 — Cloud Vision | GPT-4o / Claude / Gemini | Scene understanding, complex docs | API cost |
+ *
+ * @see {@link VisionPipeline} for the orchestration engine.
+ * @see {@link VisionStrategy} for how tiers are combined.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/core/vision/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/core/vision/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG"}

package/dist/discovery/CapabilityDiscoveryEngine.d.ts

@@ -56,7 +56,7 @@ export declare class CapabilityDiscoveryEngine implements ICapabilityDiscoveryEn
     /**
      * Index emergent tools into the capability discovery system.
      *
-     * Converts {@link EmergentTool} objects to {@link CapabilityDescriptor}s and
+     * Converts `EmergentTool` objects to `CapabilityDescriptor` records and
      * upserts them into the vector index and relationship graph. Session-tier tools
      * are skipped because they are too ephemeral to warrant indexing overhead.
      *

package/dist/discovery/CapabilityDiscoveryEngine.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"CapabilityDiscoveryEngine.d.ts","sourceRoot":"","sources":["../../src/discovery/CapabilityDiscoveryEngine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EACV,yBAAyB,EACzB,yBAAyB,EACzB,oBAAoB,EACpB,sBAAsB,EACtB,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EACnB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAUzD,qBAAa,yBAA0B,YAAW,0BAA0B;IAC1E,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkB;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkB;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA4B;IACnD,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,WAAW,CAAS;gBAG1B,gBAAgB,EAAE,iBAAiB,EACnC,WAAW,EAAE,YAAY,EACzB,MAAM,CAAC,EAAE,OAAO,CAAC,yBAAyB,CAAC;IAiB7C;;;;;OAKG;IACG,UAAU,CACd,OAAO,EAAE,sBAAsB,EAC/B,mBAAmB,CAAC,EAAE,kBAAkB,EAAE,GACzC,OAAO,CAAC,IAAI,CAAC;IAgBhB;;;;;;;;;OASG;IACG,QAAQ,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,yBAAyB,CAAC;IAgErC;;OAEG;IACH,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,oBAAoB,GAAG,SAAS;IAQjE;;;OAGG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B5E;;;;;;;;;;;;;;;;OAgBG;IACG,kBAAkB,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAwChE;;;;;OAKG;IACG,mBAAmB,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAwBjE,aAAa,IAAI,OAAO;IAIxB,iBAAiB,IAAI,MAAM,EAAE;IAI7B;;OAEG;IACH,SAAS,IAAI,QAAQ,CAAC,yBAAyB,CAAC;IAIhD;;OAEG;IACH,QAAQ,IAAI;QACV,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB;IASD;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,yBAAyB,GAAG,MAAM;IAQ1D,OAAO,CAAC,WAAW;CAoBpB"}
+{"version":3,"file":"CapabilityDiscoveryEngine.d.ts","sourceRoot":"","sources":["../../src/discovery/CapabilityDiscoveryEngine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AAC3D,OAAO,KAAK,EACV,yBAAyB,EACzB,yBAAyB,EACzB,oBAAoB,EACpB,sBAAsB,EACtB,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EACnB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAUzD,qBAAa,yBAA0B,YAAW,0BAA0B;IAC1E,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkB;IACxC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkB;IACxC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA6B;IACvD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAA4B;IACnD,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,WAAW,CAAS;gBAG1B,gBAAgB,EAAE,iBAAiB,EACnC,WAAW,EAAE,YAAY,EACzB,MAAM,CAAC,EAAE,OAAO,CAAC,yBAAyB,CAAC;IAiB7C;;;;;OAKG;IACG,UAAU,CACd,OAAO,EAAE,sBAAsB,EAC/B,mBAAmB,CAAC,EAAE,kBAAkB,EAAE,GACzC,OAAO,CAAC,IAAI,CAAC;IAgBhB;;;;;;;;;OASG;IACG,QAAQ,CACZ,WAAW,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,yBAAyB,CAAC;IAyDrC;;OAEG;IACH,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,oBAAoB,GAAG,SAAS;IAQjE;;;OAGG;IACG,YAAY,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,sBAAsB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IA8B5E;;;;;;;;;;;;;;;;OAgBG;IACG,kBAAkB,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAwChE;;;;;OAKG;IACG,mBAAmB,CAAC,KAAK,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IAwBjE,aAAa,IAAI,OAAO;IAIxB,iBAAiB,IAAI,MAAM,EAAE;IAI7B;;OAEG;IACH,SAAS,IAAI,QAAQ,CAAC,yBAAyB,CAAC;IAIhD;;OAEG;IACH,QAAQ,IAAI;QACV,eAAe,EAAE,MAAM,CAAC;QACxB,UAAU,EAAE,MAAM,CAAC;QACnB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB;IASD;;OAEG;IACH,eAAe,CAAC,MAAM,EAAE,yBAAyB,GAAG,MAAM;IAQ1D,OAAO,CAAC,WAAW;CAoBpB"}

package/dist/discovery/CapabilityDiscoveryEngine.js

@@ -144,7 +144,7 @@ export class CapabilityDiscoveryEngine {
     /**
      * Index emergent tools into the capability discovery system.
      *
-     * Converts {@link EmergentTool} objects to {@link CapabilityDescriptor}s and
+     * Converts `EmergentTool` objects to `CapabilityDescriptor` records and
      * upserts them into the vector index and relationship graph. Session-tier tools
      * are skipped because they are too ephemeral to warrant indexing overhead.
      *