@framers/agentos 0.1.125 → 0.1.127

package/dist/vision/types.d.ts

@@ -0,0 +1,411 @@
+/**
+ * @module vision/types
+ *
+ * Type definitions for the unified vision pipeline.
+ *
+ * The vision pipeline processes images through configurable tiers — from fast,
+ * free, offline OCR (PaddleOCR / Tesseract.js) through local vision models
+ * (TrOCR, Florence-2, CLIP via HuggingFace Transformers) to cloud vision LLMs
+ * (GPT-4o, Claude, Gemini). Each tier adds progressively richer understanding
+ * at increasing cost and latency.
+ *
+ * ## Tier overview
+ *
+ * | Tier | Provider | Capability | Cost |
+ * |------|----------|-----------|------|
+ * | 1 — OCR | PaddleOCR / Tesseract.js | Printed text extraction | Free/offline |
+ * | 2 — Local Vision | TrOCR / Florence-2 / CLIP | Handwriting, layout, embeddings | Free/offline |
+ * | 3 — Cloud Vision | GPT-4o / Claude / Gemini | Scene understanding, complex docs | API cost |
+ *
+ * @see {@link VisionPipeline} for the orchestration engine.
+ * @see {@link VisionStrategy} for how tiers are combined.
+ */
+/**
+ * Strategy for how vision tiers are combined during processing.
+ *
+ * - `'progressive'` — Try local OCR first, enhance with local vision models,
+ *   upgrade with cloud only when confidence is below threshold. Best balance
+ *   of cost and quality.
+ * - `'local-only'` — Tiers 1 + 2 only, never call cloud APIs. For air-gapped
+ *   or privacy-sensitive environments.
+ * - `'cloud-only'` — Skip local processing entirely, send directly to cloud
+ *   vision LLM. Highest quality but highest cost.
+ * - `'parallel'` — Run local and cloud simultaneously, merge the best results.
+ *   Lowest latency for high-quality output when cost is not a concern.
+ */
+export type VisionStrategy = 'progressive' | 'local-only' | 'cloud-only' | 'parallel';
+/**
+ * What kind of visual content the pipeline detected or was told to expect.
+ *
+ * Used to route images to the most appropriate processing tier — e.g.
+ * handwritten content triggers TrOCR, complex layouts trigger Florence-2.
+ */
+export type ContentCategory = 'printed-text' | 'handwritten' | 'document-layout' | 'photograph' | 'diagram' | 'screenshot' | 'mixed';
+/**
+ * Identifies which processing tier produced a result.
+ *
+ * - `'ocr'` — Tier 1: PaddleOCR or Tesseract.js text extraction
+ * - `'handwriting'` — Tier 2a: TrOCR handwriting recognition
+ * - `'document-ai'` — Tier 2b: Florence-2 document understanding
+ * - `'embedding'` — Tier 2c: CLIP image embedding generation
+ * - `'cloud-vision'` — Tier 3: Cloud vision LLM (GPT-4o, Claude, etc.)
+ */
+export type VisionTier = 'ocr' | 'handwriting' | 'document-ai' | 'embedding' | 'cloud-vision';
+/**
+ * Result from a single processing tier.
+ *
+ * Each tier that runs produces one of these, regardless of whether the
+ * pipeline ultimately uses its output or prefers a higher-confidence
+ * alternative from another tier.
+ */
+export interface TierResult {
+    /** Which tier produced this result. */
+    tier: VisionTier;
+    /** Provider name within the tier (e.g. 'paddle', 'tesseract', 'openai'). */
+    provider: string;
+    /** Extracted or generated text content. */
+    text: string;
+    /** Confidence score from 0 (no confidence) to 1 (certain). */
+    confidence: number;
+    /** Wall-clock processing time in milliseconds. */
+    durationMs: number;
+    /** Bounding boxes for detected text regions, when available. */
+    regions?: TextRegion[];
+}
+/**
+ * A detected text region within an image, with spatial coordinates
+ * and per-region confidence.
+ */
+export interface TextRegion {
+    /** The text content within this region. */
+    text: string;
+    /** Confidence score for this specific region (0–1). */
+    confidence: number;
+    /**
+     * Bounding box in image coordinates (pixels).
+     * Origin is top-left corner of the image.
+     */
+    bbox: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+}
+/**
+ * Structured document layout produced by Florence-2 or similar
+ * document understanding models.
+ *
+ * Captures the spatial arrangement of text, tables, figures, headings,
+ * and other semantic blocks within a multi-page document.
+ */
+export interface DocumentLayout {
+    /** Pages in document order. */
+    pages: DocumentPage[];
+}
+/**
+ * A single page within a structured document layout.
+ */
+export interface DocumentPage {
+    /** 1-based page number. */
+    pageNumber: number;
+    /** Page width in pixels. */
+    width: number;
+    /** Page height in pixels. */
+    height: number;
+    /** Semantic blocks detected on this page. */
+    blocks: LayoutBlock[];
+}
+/**
+ * A semantic block within a document page — a paragraph, table, figure,
+ * heading, list, or code snippet.
+ */
+export interface LayoutBlock {
+    /** Semantic type of the block. */
+    type: 'text' | 'table' | 'figure' | 'heading' | 'list' | 'code';
+    /** Text content extracted from the block. */
+    content: string;
+    /**
+     * Bounding box in page coordinates (pixels).
+     * Origin is top-left corner of the page.
+     */
+    bbox: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    /** Confidence score for this block detection (0–1). */
+    confidence: number;
+}
+/**
+ * Aggregated result from the vision pipeline after all configured tiers
+ * have run. Contains the best extracted text, content classification,
+ * optional embeddings, and detailed per-tier breakdowns.
+ *
+ * @example
+ * ```typescript
+ * const result = await pipeline.process(imageBuffer);
+ *
+ * // Best extracted text across all tiers
+ * console.log(result.text);
+ *
+ * // What kind of content was detected
+ * console.log(result.category); // 'printed-text' | 'handwritten' | ...
+ *
+ * // CLIP embedding for similarity search
+ * if (result.embedding) {
+ *   await vectorStore.upsert('images', [{ embedding: result.embedding }]);
+ * }
+ *
+ * // Inspect individual tier contributions
+ * for (const tr of result.tierResults) {
+ *   console.log(`${tr.tier} (${tr.provider}): ${tr.confidence}`);
+ * }
+ * ```
+ */
+export interface VisionResult {
+    /** Best extracted text (from OCR, handwriting, or vision description). */
+    text: string;
+    /** Overall confidence score 0–1, taken from the winning tier. */
+    confidence: number;
+    /** What kind of content was detected. */
+    category: ContentCategory;
+    /** Which tier(s) contributed to the final result. */
+    tiers: VisionTier[];
+    /** Detailed results from each tier that ran, ordered by execution. */
+    tierResults: TierResult[];
+    /** CLIP image embedding vector, when embedding tier is enabled. */
+    embedding?: number[];
+    /** Structured document layout, when Florence-2 ran. */
+    layout?: DocumentLayout;
+    /** Bounding boxes for detected text regions from the winning tier. */
+    regions?: TextRegion[];
+    /** Total wall-clock processing time in milliseconds. */
+    durationMs: number;
+}
+/**
+ * Preprocessing options applied to images before they enter the
+ * vision pipeline tiers. Uses `sharp` under the hood.
+ */
+export interface VisionPreprocessingConfig {
+    /** Convert to grayscale before OCR (improves contrast for printed text). */
+    grayscale?: boolean;
+    /**
+     * Resize constraints. The image is scaled down proportionally
+     * so that neither dimension exceeds the specified maximum.
+     * No upscaling is performed.
+     */
+    resize?: {
+        maxWidth?: number;
+        maxHeight?: number;
+    };
+    /** Apply unsharp-mask sharpening (helps blurry scans). */
+    sharpen?: boolean;
+    /** Normalize brightness/contrast (histogram stretching). */
+    normalize?: boolean;
+}
+/**
+ * Configuration for the {@link VisionPipeline}.
+ *
+ * All fields are optional — the factory function {@link createVisionPipeline}
+ * auto-detects available providers and fills in sensible defaults.
+ *
+ * @example
+ * ```typescript
+ * const config: VisionPipelineConfig = {
+ *   strategy: 'progressive',
+ *   ocr: 'paddle',
+ *   handwriting: true,
+ *   documentAI: true,
+ *   embedding: true,
+ *   cloudProvider: 'openai',
+ *   cloudModel: 'gpt-4o',
+ *   confidenceThreshold: 0.8,
+ *   preprocessing: { grayscale: true, sharpen: true },
+ * };
+ * ```
+ */
+export interface VisionPipelineConfig {
+    /**
+     * How to combine tiers.
+     * @default 'progressive'
+     */
+    strategy: VisionStrategy;
+    /**
+     * OCR engine for Tier 1 text extraction.
+     * - `'paddle'` — PaddleOCR (via `ppu-paddle-ocr`). Best accuracy for most scripts.
+     * - `'tesseract'` — Tesseract.js. Wider language support, slightly lower accuracy.
+     * - `'none'` — Skip OCR entirely (useful for photo-only pipelines).
+     * @default 'paddle' (if installed), else 'tesseract' (if installed), else 'none'
+     */
+    ocr?: 'paddle' | 'tesseract' | 'none';
+    /**
+     * Enable handwriting recognition via TrOCR (`@huggingface/transformers`).
+     * Only triggered when OCR confidence is low and content appears handwritten.
+     * @default false
+     */
+    handwriting?: boolean;
+    /**
+     * Enable document understanding via Florence-2 (`@huggingface/transformers`).
+     * Produces structured {@link DocumentLayout} with semantic block detection.
+     * @default false
+     */
+    documentAI?: boolean;
+    /**
+     * Enable CLIP image embeddings (`@huggingface/transformers`).
+     * Runs in parallel with other tiers — does not affect text extraction.
+     * @default false
+     */
+    embedding?: boolean;
+    /**
+     * Cloud vision LLM provider name for Tier 3 fallback.
+     * Must match a provider known to `generateText()` (e.g. 'openai', 'anthropic', 'google').
+     * When unset, cloud vision is disabled.
+     */
+    cloudProvider?: string;
+    /**
+     * Cloud model override. When unset, the provider's default vision model is used.
+     * @example 'gpt-4o', 'claude-sonnet-4-20250514', 'gemini-2.0-flash'
+     */
+    cloudModel?: string;
+    /**
+     * Minimum confidence to accept an OCR result without escalating to cloud.
+     * Only applies to `'progressive'` strategy — if OCR confidence is below
+     * this threshold, the pipeline escalates to the next tier.
+     * @default 0.7
+     */
+    confidenceThreshold?: number;
+    /**
+     * Image preprocessing options applied before any tier runs.
+     * Uses `sharp` for resizing, grayscale conversion, sharpening,
+     * and normalization.
+     */
+    preprocessing?: VisionPreprocessingConfig;
+}
+/**
+ * A single decoded video frame with its timestamp and sequence index.
+ *
+ * Frames are produced by the frame extraction stage of the video
+ * analysis pipeline and consumed by {@link SceneDetector} for visual
+ * change detection.
+ */
+export interface Frame {
+    /** Raw pixel data as a Buffer (RGB, 3 bytes per pixel, row-major). */
+    buffer: Buffer;
+    /**
+     * Original encoded frame bytes (for example PNG/JPEG) when the frame was
+     * decoded into raw RGB for scene detection.
+     */
+    sourceBuffer?: Buffer;
+    /** Timestamp of this frame within the source video, in seconds. */
+    timestampSec: number;
+    /** 0-based sequential index of this frame in the extraction order. */
+    index: number;
+}
+/**
+ * A detected scene boundary within a video frame sequence.
+ *
+ * Produced by {@link SceneDetector.detectScenes} when visual
+ * discontinuity between consecutive frames exceeds the configured
+ * threshold. Each boundary marks the start of a new scene.
+ */
+export interface SceneBoundary {
+    /** 0-based index of this scene boundary in detection order. */
+    index: number;
+    /** Index of the first frame belonging to this scene. */
+    startFrame: number;
+    /** Index of the last frame belonging to this scene. */
+    endFrame: number;
+    /** Timestamp (seconds) of the first frame in this scene. */
+    startTimeSec: number;
+    /** Timestamp (seconds) of the last frame in this scene. */
+    endTimeSec: number;
+    /** Duration of this scene in seconds (`endTimeSec - startTimeSec`). */
+    durationSec: number;
+    /**
+     * Classification of the visual transition that starts this scene.
+     *
+     * - `'hard-cut'` — Abrupt frame-to-frame change (chi-squared > hardCutThreshold)
+     * - `'dissolve'` — Cross-dissolve / superimposition (diff > 0.25)
+     * - `'fade'`     — Fade from/to black or white (diff > 0.20)
+     * - `'wipe'`     — Directional wipe transition
+     * - `'gradual'`  — Other gradual transition below dissolve threshold
+     */
+    cutType: 'hard-cut' | 'dissolve' | 'fade' | 'wipe' | 'gradual';
+    /**
+     * Confidence score (0–1) for this scene boundary.
+     * Derived from the diff score relative to the threshold — higher
+     * values indicate a more definitive visual discontinuity.
+     */
+    confidence: number;
+    /**
+     * Raw difference score that triggered the scene boundary.
+     * Useful for debugging threshold tuning. The scale depends on the
+     * detection method (histogram chi-squared, 1 - SSIM, etc.).
+     */
+    diffScore?: number;
+}
+/**
+ * Detection method used by the {@link SceneDetector}.
+ *
+ * - `'histogram'` — RGB histogram chi-squared distance. Fast, works
+ *   well for hard cuts. No external dependencies.
+ * - `'ssim'`      — Structural Similarity Index. Better for gradual
+ *   transitions. Requires `sharp` (falls back to histogram if missing).
+ * - `'clip'`      — CLIP embedding cosine distance. Semantic scene
+ *   change detection. Requires a CLIP provider (local or OpenAI).
+ */
+export type SceneDetectionMethod = 'histogram' | 'ssim' | 'clip';
+/**
+ * Configuration for the {@link SceneDetector}.
+ *
+ * All fields are optional — the detector uses sensible defaults that
+ * work well for typical video content.
+ *
+ * @example
+ * ```typescript
+ * const config: SceneDetectorConfig = {
+ *   methods: ['histogram', 'ssim'],
+ *   hardCutThreshold: 0.3,
+ *   gradualThreshold: 0.15,
+ *   minSceneDurationSec: 1.0,
+ * };
+ * ```
+ */
+export interface SceneDetectorConfig {
+    /**
+     * Detection methods to use. Multiple methods are combined by
+     * taking the maximum diff score across all methods.
+     * @default ['histogram', 'ssim']
+     */
+    methods?: SceneDetectionMethod[];
+    /**
+     * Diff score threshold above which a frame transition is classified
+     * as a hard cut. Applied to histogram chi-squared distance (0–1).
+     * @default 0.3
+     */
+    hardCutThreshold?: number;
+    /**
+     * Diff score threshold for gradual transitions (dissolves, fades).
+     * Transitions with scores between this and {@link hardCutThreshold}
+     * are classified as gradual cuts.
+     * @default 0.15
+     */
+    gradualThreshold?: number;
+    /**
+     * Minimum scene duration in seconds. Scene boundaries that would
+     * create scenes shorter than this are suppressed. Prevents
+     * over-segmentation from flashes, strobe effects, or noise.
+     * @default 1.0
+     */
+    minSceneDurationSec?: number;
+    /**
+     * CLIP embedding provider to use when `'clip'` is in {@link methods}.
+     * - `'local'`  — Use local CLIP model via @huggingface/transformers
+     * - `'openai'` — Use OpenAI's CLIP-based embedding endpoint
+     * @default 'local'
+     */
+    clipProvider?: 'local' | 'openai';
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/vision/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/vision/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAMH;;;;;;;;;;;;GAYG;AACH,MAAM,MAAM,cAAc,GACtB,aAAa,GACb,YAAY,GACZ,YAAY,GACZ,UAAU,CAAC;AAMf;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GACvB,cAAc,GACd,aAAa,GACb,iBAAiB,GACjB,YAAY,GACZ,SAAS,GACT,YAAY,GACZ,OAAO,CAAC;AAMZ;;;;;;;;GAQG;AACH,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,aAAa,GAAG,aAAa,GAAG,WAAW,GAAG,cAAc,CAAC;AAM9F;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IACzB,uCAAuC;IACvC,IAAI,EAAE,UAAU,CAAC;IAEjB,4EAA4E;IAC5E,QAAQ,EAAE,MAAM,CAAC;IAEjB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IAEb,8DAA8D;IAC9D,UAAU,EAAE,MAAM,CAAC;IAEnB,kDAAkD;IAClD,UAAU,EAAE,MAAM,CAAC;IAEnB,gEAAgE;IAChE,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;CACxB;AAED;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB,2CAA2C;IAC3C,IAAI,EAAE,MAAM,CAAC;IAEb,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH;AAMD;;;;;;GAMG;AACH,MAAM,WAAW,cAAc;IAC7B,+BAA+B;IAC/B,KAAK,EAAE,YAAY,EAAE,CAAC;CACvB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC;IAEnB,4BAA4B;IAC5B,KAAK,EAAE,MAAM,CAAC;IAEd,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAC;IAEf,6CAA6C;IAC7C,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,kCAAkC;IAClC,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,CAAC;IAEhE,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,IAAI,EAAE;QACJ,CAAC,EAAE,MAAM,CAAC;QACV,CAAC,EAAE,MAAM,CAAC;QACV,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;IAEF,uDAAuD;IACvD,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,WAAW,YAAY;IAC3B,0EAA0E;IAC1E,IAAI,EAAE,MAAM,CAAC;IAEb,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;IAEnB,yCAAyC;IACzC,QAAQ,EAAE,eAAe,CAAC;IAE1B,qDAAqD;IACrD,KAAK,EAAE,UAAU,EAAE,CAAC;IAEpB,sEAAsE;IACtE,WAAW,EAAE,UAAU,EAAE,CAAC;IAE1B,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IAErB,uDAAuD;IACvD,MAAM,CAAC,EAAE,cAAc,CAAC;IAExB,sEAAsE;IACtE,OAAO,CAAC,EAAE,UAAU,EAAE,CAAC;IAEvB,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;CACpB;AAMD;;;GAGG;AACH,MAAM,WAAW,yBAAyB;IACxC,4EAA4E;IAC5E,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;OAIG;IACH,MAAM,CAAC,EAAE;QACP,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,0DAA0D;IAC1D,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB,4DAA4D;IAC5D,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;OAGG;IACH,QAAQ,EAAE,cAAc,CAAC;IAEzB;;;;;;OAMG;IACH,GAAG,CAAC,EAAE,QAAQ,GAAG,WAAW,GAAG,MAAM,CAAC;IAEtC;;;;OAIG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;;OAIG;IACH,UAAU,CAAC,EAAE,OAAO,CAAC;IAErB;;;;OAIG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IAEpB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IAEvB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;OAIG;IACH,aAAa,CAAC,EAAE,yBAAyB,CAAC;CAC3C;AAMD;;;;;;GAMG;AACH,MAAM,WAAW,KAAK;IACpB,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IAEf;;;OAGG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,mEAAmE;IACnE,YAAY,EAAE,MAAM,CAAC;IAErB,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAC;CACf;AAMD;;;;;;GAMG;AACH,MAAM,WAAW,aAAa;IAC5B,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IAEd,wDAAwD;IACxD,UAAU,EAAE,MAAM,CAAC;IAEnB,uDAAuD;IACvD,QAAQ,EAAE,MAAM,CAAC;IAEjB,4DAA4D;IAC5D,YAAY,EAAE,MAAM,CAAC;IAErB,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAC;IAEnB,uEAAuE;IACvE,WAAW,EAAE,MAAM,CAAC;IAEpB;;;;;;;;OAQG;IACH,OAAO,EAAE,UAAU,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IAE/D;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAC;IAEnB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAMD;;;;;;;;;GASG;AACH,MAAM,MAAM,oBAAoB,GAAG,WAAW,GAAG,MAAM,GAAG,MAAM,CAAC;AAEjE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,OAAO,CAAC,EAAE,oBAAoB,EAAE,CAAC;IAEjC;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;OAKG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;IACH,YAAY,CAAC,EAAE,OAAO,GAAG,QAAQ,CAAC;CACnC"}

package/dist/vision/types.js

@@ -0,0 +1,24 @@
+/**
+ * @module vision/types
+ *
+ * Type definitions for the unified vision pipeline.
+ *
+ * The vision pipeline processes images through configurable tiers — from fast,
+ * free, offline OCR (PaddleOCR / Tesseract.js) through local vision models
+ * (TrOCR, Florence-2, CLIP via HuggingFace Transformers) to cloud vision LLMs
+ * (GPT-4o, Claude, Gemini). Each tier adds progressively richer understanding
+ * at increasing cost and latency.
+ *
+ * ## Tier overview
+ *
+ * | Tier | Provider | Capability | Cost |
+ * |------|----------|-----------|------|
+ * | 1 — OCR | PaddleOCR / Tesseract.js | Printed text extraction | Free/offline |
+ * | 2 — Local Vision | TrOCR / Florence-2 / CLIP | Handwriting, layout, embeddings | Free/offline |
+ * | 3 — Cloud Vision | GPT-4o / Claude / Gemini | Scene understanding, complex docs | API cost |
+ *
+ * @see {@link VisionPipeline} for the orchestration engine.
+ * @see {@link VisionStrategy} for how tiers are combined.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/vision/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/vision/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG"}

package/dist/voice-pipeline/AcousticEndpointDetector.d.ts

@@ -9,7 +9,7 @@
  * ## How it works
  *
  * This detector delegates all silence timing to a {@link SilenceDetector} instance
- * (from `core/audio/`). The SilenceDetector maintains an internal timer that
+ * (from `media/audio/`). The SilenceDetector maintains an internal timer that
  * starts when `handleSpeechEnd()` is called and fires `'utterance_end_detected'`
  * when silence exceeds the configured `utteranceEndThresholdMs`. A
  * `handleSpeechStart()` call cancels the timer.
@@ -106,7 +106,7 @@ export declare class AcousticEndpointDetector extends EventEmitter implements IE
      */
     readonly mode: "acoustic";
     /**
-     * Underlying silence-duration tracker from `core/audio/`.
+     * Underlying silence-duration tracker from `media/audio/`.
      * Handles the actual timer management and threshold comparison.
      */
     private readonly silenceDetector;

package/dist/voice-pipeline/AcousticEndpointDetector.js

@@ -9,7 +9,7 @@
  * ## How it works
  *
  * This detector delegates all silence timing to a {@link SilenceDetector} instance
- * (from `core/audio/`). The SilenceDetector maintains an internal timer that
+ * (from `media/audio/`). The SilenceDetector maintains an internal timer that
  * starts when `handleSpeechEnd()` is called and fires `'utterance_end_detected'`
  * when silence exceeds the configured `utteranceEndThresholdMs`. A
  * `handleSpeechStart()` call cancels the timer.
@@ -45,7 +45,7 @@
  * | `'speech_start'`  | *(none)*                 | Re-emitted from incoming VAD event.         |
  */
 import { EventEmitter } from 'node:events';
-import { SilenceDetector } from '../core/audio/SilenceDetector.js';
+import { SilenceDetector } from '../hearing/SilenceDetector.js';
 // ---------------------------------------------------------------------------
 // Implementation
 // ---------------------------------------------------------------------------

package/dist/voice-pipeline/AcousticEndpointDetector.js.map

@@ -1 +1 @@
-{"version":3,"file":"AcousticEndpointDetector.js","sourceRoot":"","sources":["../../src/voice-pipeline/AcousticEndpointDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAA8B,MAAM,kCAAkC,CAAC;AAyC/F,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,OAAO,wBAAyB,SAAQ,YAAY;IA+BxD,8EAA8E;IAC9E,cAAc;IACd,8EAA8E;IAE9E;;;;OAIG;IACH,YAAY,SAAyC,EAAE;QACrD,KAAK,EAAE,CAAC;QAxCV;;;WAGG;QACa,SAAI,GAAG,UAAmB,CAAC;QAQ3C;;;;;;WAMG;QACK,sBAAiB,GAAkB,IAAI,CAAC;QAEhD;;;;;;WAMG;QACK,oBAAe,GAAkB,IAAI,CAAC;QAc5C,uEAAuE;QACvE,MAAM,QAAQ,GAA0B;YACtC,2BAA2B,EAAE,MAAM,CAAC,2BAA2B,IAAI,IAAI;YACvE,uBAAuB,EAAE,MAAM,CAAC,uBAAuB,IAAI,IAAI;SAChE,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,CAAC,QAAQ,CAAC,CAAC;QAErD,yEAAyE;QACzE,qEAAqE;QACrE,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,wBAAwB,EAAE,CAAC,kBAA0B,EAAE,EAAE;YAC/E,iEAAiE;YACjE,yDAAyD;YACzD,MAAM,UAAU,GACd,IAAI,CAAC,iBAAiB,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI;gBAC9D,CAAC,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,iBAAiB;gBAC/C,CAAC,CAAC,CAAC,CAAC;YAER,MAAM,KAAK,GAAsB;gBAC/B,kEAAkE;gBAClE,sEAAsE;gBACtE,UAAU,EAAE,EAAE;gBACd,wDAAwD;gBACxD,UAAU,EAAE,CAAC;gBACb,UAAU;gBACV,MAAM,EAAE,iBAAiB;aAC1B,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,8EAA8E;IAC9E,oCAAoC;IACpC,8EAA8E;IAE9E;;;;;;;;;;OAUG;IACI,YAAY,CAAC,KAAe;QACjC,wEAAwE;QACxE,sEAAsE;QACtE,mEAAmE;QACnE,uDAAuD;QACvD,MAAM,aAAa,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAW,CAAC;QAE9D,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,oDAAoD;gBACpD,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzC,mEAAmE;gBACnE,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;gBAC5B,6DAA6D;gBAC7D,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;gBACtD,2DAA2D;gBAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC1B,MAAM;YAER,KAAK,YAAY;gBACf,oDAAoD;gBACpD,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC,SAAS,CAAC;gBACvC,wDAAwD;gBACxD,yEAAyE;gBACzE,0EAA0E;gBAC1E,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;gBACvD,MAAM;YAER,KAAK,SAAS;gBACZ,mEAAmE;gBACnE,0DAA0D;gBAC1D,IAAI,CAAC,eAAe,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAC;gBAC1D,MAAM;QACV,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,sCAAsC;IACtC,8EAA8E;IAE9E;;;;;;;OAOG;IACI,cAAc,CAAC,MAAuB;QAC3C,mEAAmE;QACnE,mEAAmE;QACnE,iCAAiC;IACnC,CAAC;IAED,8EAA8E;IAC9E,6BAA6B;IAC7B,8EAA8E;IAE9E;;;;;;OAMG;IACI,KAAK;QACV,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF"}
+{"version":3,"file":"AcousticEndpointDetector.js","sourceRoot":"","sources":["../../src/voice-pipeline/AcousticEndpointDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6CG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAA8B,MAAM,+BAA+B,CAAC;AAyC5F,8EAA8E;AAC9E,iBAAiB;AACjB,8EAA8E;AAE9E;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,OAAO,wBAAyB,SAAQ,YAAY;IA+BxD,8EAA8E;IAC9E,cAAc;IACd,8EAA8E;IAE9E;;;;OAIG;IACH,YAAY,SAAyC,EAAE;QACrD,KAAK,EAAE,CAAC;QAxCV;;;WAGG;QACa,SAAI,GAAG,UAAmB,CAAC;QAQ3C;;;;;;WAMG;QACK,sBAAiB,GAAkB,IAAI,CAAC;QAEhD;;;;;;WAMG;QACK,oBAAe,GAAkB,IAAI,CAAC;QAc5C,uEAAuE;QACvE,MAAM,QAAQ,GAA0B;YACtC,2BAA2B,EAAE,MAAM,CAAC,2BAA2B,IAAI,IAAI;YACvE,uBAAuB,EAAE,MAAM,CAAC,uBAAuB,IAAI,IAAI;SAChE,CAAC;QAEF,IAAI,CAAC,eAAe,GAAG,IAAI,eAAe,CAAC,QAAQ,CAAC,CAAC;QAErD,yEAAyE;QACzE,qEAAqE;QACrE,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,wBAAwB,EAAE,CAAC,kBAA0B,EAAE,EAAE;YAC/E,iEAAiE;YACjE,yDAAyD;YACzD,MAAM,UAAU,GACd,IAAI,CAAC,iBAAiB,KAAK,IAAI,IAAI,IAAI,CAAC,eAAe,KAAK,IAAI;gBAC9D,CAAC,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,iBAAiB;gBAC/C,CAAC,CAAC,CAAC,CAAC;YAER,MAAM,KAAK,GAAsB;gBAC/B,kEAAkE;gBAClE,sEAAsE;gBACtE,UAAU,EAAE,EAAE;gBACd,wDAAwD;gBACxD,UAAU,EAAE,CAAC;gBACb,UAAU;gBACV,MAAM,EAAE,iBAAiB;aAC1B,CAAC;YAEF,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,KAAK,CAAC,CAAC;QACpC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,8EAA8E;IAC9E,oCAAoC;IACpC,8EAA8E;IAE9E;;;;;;;;;;OAUG;IACI,YAAY,CAAC,KAAe;QACjC,wEAAwE;QACxE,sEAAsE;QACtE,mEAAmE;QACnE,uDAAuD;QACvD,MAAM,aAAa,GAAG,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAW,CAAC;QAE9D,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;YACnB,KAAK,cAAc;gBACjB,oDAAoD;gBACpD,IAAI,CAAC,iBAAiB,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzC,mEAAmE;gBACnE,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;gBAC5B,6DAA6D;gBAC7D,IAAI,CAAC,eAAe,CAAC,iBAAiB,CAAC,aAAa,CAAC,CAAC;gBACtD,2DAA2D;gBAC3D,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAC1B,MAAM;YAER,KAAK,YAAY;gBACf,oDAAoD;gBACpD,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC,SAAS,CAAC;gBACvC,wDAAwD;gBACxD,yEAAyE;gBACzE,0EAA0E;gBAC1E,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;gBACvD,MAAM;YAER,KAAK,SAAS;gBACZ,mEAAmE;gBACnE,0DAA0D;gBAC1D,IAAI,CAAC,eAAe,CAAC,qBAAqB,CAAC,aAAa,CAAC,CAAC;gBAC1D,MAAM;QACV,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,sCAAsC;IACtC,8EAA8E;IAE9E;;;;;;;OAOG;IACI,cAAc,CAAC,MAAuB;QAC3C,mEAAmE;QACnE,mEAAmE;QACnE,iCAAiC;IACnC,CAAC;IAED,8EAA8E;IAC9E,6BAA6B;IAC7B,8EAA8E;IAE9E;;;;;;OAMG;IACI,KAAK;QACV,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAC9B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;CACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@framers/agentos",
-  "version": "0.1.125",
+  "version": "0.1.127",
   "description": "Modular AgentOS orchestration library",
   "type": "module",
   "main": "dist/index.js",
@@ -28,40 +28,60 @@
       "default": "./dist/speech/index.js",
       "types": "./dist/speech/index.d.ts"
     },
+    "./hearing": {
+      "import": "./dist/hearing/index.js",
+      "default": "./dist/hearing/index.js",
+      "types": "./dist/hearing/index.d.ts"
+    },
     "./images": {
-      "import": "./dist/core/images/index.js",
-      "default": "./dist/core/images/index.js",
-      "types": "./dist/core/images/index.d.ts"
+      "import": "./dist/media/images/index.js",
+      "default": "./dist/media/images/index.js",
+      "types": "./dist/media/images/index.d.ts"
+    },
+    "./safety/guardrails": {
+      "import": "./dist/safety/guardrails/index.js",
+      "default": "./dist/safety/guardrails/index.js",
+      "types": "./dist/safety/guardrails/index.d.ts"
+    },
+    "./safety/guardrails/index": {
+      "import": "./dist/safety/guardrails/index.js",
+      "default": "./dist/safety/guardrails/index.js",
+      "types": "./dist/safety/guardrails/index.d.ts"
     },
     "./core/guardrails": {
-      "import": "./dist/core/guardrails/index.js",
-      "default": "./dist/core/guardrails/index.js",
-      "types": "./dist/core/guardrails/index.d.ts"
+      "import": "./dist/safety/guardrails/index.js",
+      "default": "./dist/safety/guardrails/index.js",
+      "types": "./dist/safety/guardrails/index.d.ts"
     },
-    "./core/guardrails/index": {
-      "import": "./dist/core/guardrails/index.js",
-      "default": "./dist/core/guardrails/index.js",
-      "types": "./dist/core/guardrails/index.d.ts"
+    "./safety/runtime": {
+      "import": "./dist/safety/runtime/index.js",
+      "default": "./dist/safety/runtime/index.js",
+      "types": "./dist/safety/runtime/index.d.ts"
     },
-    "./core/safety": {
-      "import": "./dist/core/safety/index.js",
-      "default": "./dist/core/safety/index.js",
-      "types": "./dist/core/safety/index.d.ts"
+    "./safety/runtime/index": {
+      "import": "./dist/safety/runtime/index.js",
+      "default": "./dist/safety/runtime/index.js",
+      "types": "./dist/safety/runtime/index.d.ts"
+    },
+    "./safety/runtime/*": {
+      "import": "./dist/safety/runtime/*.js",
+      "default": "./dist/safety/runtime/*.js",
+      "types": "./dist/safety/runtime/*.d.ts"
     },
-    "./core/safety/index": {
-      "import": "./dist/core/safety/index.js",
-      "default": "./dist/core/safety/index.js",
-      "types": "./dist/core/safety/index.d.ts"
+    "./core/safety": {
+      "import": "./dist/safety/runtime/index.js",
+      "default": "./dist/safety/runtime/index.js",
+      "types": "./dist/safety/runtime/index.d.ts"
     },
-    "./core/safety/*": {
-      "import": "./dist/core/safety/*.js",
-      "default": "./dist/core/safety/*.js",
-      "types": "./dist/core/safety/*.d.ts"
+    "./orchestration/hitl": {
+      "import": "./dist/orchestration/hitl/index.js",
+      "default": "./dist/orchestration/hitl/index.js",
+      "types": "./dist/orchestration/hitl/index.d.ts"
     },
     "./core/hitl": {
-      "import": "./dist/core/hitl/index.js",
-      "default": "./dist/core/hitl/index.js",
-      "types": "./dist/core/hitl/index.d.ts"
+      "import": "./dist/orchestration/hitl/index.js",
+      "default": "./dist/orchestration/hitl/index.js",
+      "types": "./dist/orchestration/hitl/index.d.ts"
     },
     "./rag": {
       "import": "./dist/rag/index.js",
@@ -103,10 +123,10 @@
       "default": "./dist/core/tools/*.js",
       "types": "./dist/core/tools/*.d.ts"
     },
-    "./core/vision": {
-      "import": "./dist/core/vision/index.js",
-      "default": "./dist/core/vision/index.js",
-      "types": "./dist/core/vision/index.d.ts"
+    "./vision": {
+      "import": "./dist/vision/index.js",
+      "default": "./dist/vision/index.js",
+      "types": "./dist/vision/index.d.ts"
     },
     "./discovery": {
       "import": "./dist/discovery/index.js",
@@ -118,10 +138,10 @@
       "default": "./dist/orchestration/index.js",
       "types": "./dist/orchestration/index.d.ts"
     },
-    "./voice": {
-      "import": "./dist/voice/index.js",
-      "default": "./dist/voice/index.js",
-      "types": "./dist/voice/index.d.ts"
+    "./telephony": {
+      "import": "./dist/channels/telephony/index.js",
+      "default": "./dist/channels/telephony/index.js",
+      "types": "./dist/channels/telephony/index.d.ts"
     },
     "./voice-pipeline": {
       "import": "./dist/voice-pipeline/index.js",
@@ -179,6 +199,7 @@
   "scripts": {
     "clean": "rimraf dist tsconfig.build.tsbuildinfo",
     "build": "npm run clean && tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json && node scripts/fix-esm-imports.mjs",
+    "prepublishOnly": "npm run build",
     "typecheck": "tsc --noEmit",
     "lint": "node scripts/strip-nbsp.mjs && eslint \"src/**/*.ts\"",
     "test": "vitest run",
@@ -256,7 +277,7 @@
     "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@framers/sql-storage-adapter": ">=0.5.0",
+    "@framers/sql-storage-adapter": ">=0.6.0",
     "graphology": ">=0.25.0",
     "graphology-communities-louvain": ">=2.0.0",
     "graphology-types": ">=0.24.0",