@framers/agentos-ext-vision-pipeline 0.1.0 → 0.2.0

package/SKILL.md

@@ -8,13 +8,24 @@ tools_required: [vision-pipeline]
 
 # Vision & OCR
 
-Extract text from images, documents, and handwritten notes using a progressive 3-tier pipeline: local OCR (PaddleOCR) -> local vision models (TrOCR, Florence-2) -> cloud vision (GPT-4o, Claude).
+Extract text from images, documents, and handwritten notes with a progressive 3-tier pipeline: local OCR (PaddleOCR or Tesseract), then local vision models (TrOCR, Florence-2), then cloud vision.
 
-## Capabilities
-- **Printed text OCR**: Extract text from documents, receipts, screenshots
-- **Handwriting recognition**: Read handwritten notes and forms via TrOCR
-- **Document layout**: Understand tables, figures, headings via Florence-2
-- **Image embeddings**: Generate CLIP vectors for semantic image search
+## Tool
+**vision-pipeline** takes `imageUrl`, `mode` and `maxTier`. `imageUrl` is a `data:image/...` URL or an http(s) URL on a public host: local file paths and addresses on this machine or a private network are refused, so a model cannot send the machine's files, or what its network serves, to a cloud model. The check reads the URL's host as written: the address a public name resolves to, and a redirect's target, are not checked.
+
+| mode | what it returns | lowest maxTier |
+|------|-----------------|----------------|
+| `auto` (default) | the pipeline's best text | 1 |
+| `ocr` | printed text | 1 |
+| `handwriting` | handwritten text (TrOCR) | 2 |
+| `layout` | document text and structure (Florence-2) | 2 |
+| `embed` | a CLIP embedding vector | 2 |
+| `describe` | what the image shows (cloud vision) | 3 |
+
+`maxTier` 1 runs local OCR alone, 2 adds the local vision models, 3 (default) may use cloud vision.
+
+## Setup
+The local tiers use the optional packages AgentOS finds installed: `ppu-paddle-ocr` or `tesseract.js` for OCR, and `@huggingface/transformers` for handwriting, layout and embeddings. With an OpenAI key in the pack option `openaiApiKey` or the secret `openai.apiKey`, the cloud tier uses OpenAI with that key. Without one, it uses the first provider whose key is in the environment: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` (or `GEMINI_API_KEY`), `OPENROUTER_API_KEY`. `describe` and the cloud tier need `@framers/agentos` 0.12.14 or later: earlier releases send the image to the cloud model as text, and read the key from the environment only.
 
 ## Example
 "Read the text from this receipt"

package/dist/index.d.ts

@@ -0,0 +1,59 @@
+/**
+ * @fileoverview Vision & OCR Pipeline Extension Pack: the vision-pipeline tool
+ * on AgentOS's `createVisionPipeline`.
+ *
+ * Pipeline tiers:
+ *   Tier 1 (local, fast):   PaddleOCR or Tesseract for printed text
+ *   Tier 2 (local, medium): TrOCR for handwriting, Florence-2 for layout, CLIP embeddings
+ *   Tier 3 (cloud, slow):   a vision model of the provider whose key is set
+ *
+ * The local tiers need their optional packages (ppu-paddle-ocr or tesseract.js,
+ * and @huggingface/transformers); AgentOS uses the ones it finds installed.
+ * The cloud tier reads OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY (or
+ * GEMINI_API_KEY) or OPENROUTER_API_KEY from the environment.
+ *
+ * @module @framers/agentos-ext-vision-pipeline
+ */
+/** What the extension manager passes the pack factory. */
+export interface ExtensionContext {
+    /** `priority` of the tool descriptor (default 45); `openaiApiKey` for the cloud tier. */
+    options?: {
+        priority?: number;
+        openaiApiKey?: string;
+    } & Record<string, unknown>;
+    /** Reads a secret, such as `openai.apiKey`. */
+    getSecret?: (key: string) => string | undefined;
+    logger?: {
+        info: (msg: string) => void;
+        warn?: (msg: string) => void;
+    };
+}
+/** The pack the factory returns: one `vision-pipeline` tool descriptor. */
+export interface ExtensionPack {
+    name: string;
+    version: string;
+    descriptors: Array<{
+        id: string;
+        kind: string;
+        priority?: number;
+        payload: unknown;
+        requiredSecrets?: Array<{
+            id: string;
+            optional?: boolean;
+        }>;
+    }>;
+    onActivate?: () => Promise<void>;
+    onDeactivate?: () => Promise<void>;
+}
+/**
+ * Create the Vision & OCR Pipeline extension pack.
+ *
+ * The pack builds a pipeline per strategy the first time a call needs it:
+ * building one probes for the optional OCR and model packages, which loading
+ * the pack does not do. A build that fails is tried again on the next call.
+ */
+export declare function createExtensionPack(context?: ExtensionContext): ExtensionPack;
+export { VisionPipelineTool, imageInput } from './tools/visionPipeline.js';
+export type { VisionPipelineInput, VisionPipelineOutput, VisionMode } from './tools/visionPipeline.js';
+export default createExtensionPack;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA;;;;;;;;;;;;;;;GAeG;AAQH,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC/B,yFAAyF;IACzF,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjF,+CAA+C;IAC/C,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;IAChD,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;QAAC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;KAAE,CAAC;CACxE;AAED,2EAA2E;AAC3E,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,KAAK,CAAC;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,OAAO,CAAC;QACjB,eAAe,CAAC,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;SAAE,CAAC,CAAC;KAC7D,CAAC,CAAC;IACH,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACpC;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,GAAE,gBAAqB,GAAG,aAAa,CAiEjF;AAED,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAC3E,YAAY,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvG,eAAe,mBAAmB,CAAC"}

package/dist/index.js

@@ -0,0 +1,99 @@
+// @ts-nocheck
+/**
+ * @fileoverview Vision & OCR Pipeline Extension Pack: the vision-pipeline tool
+ * on AgentOS's `createVisionPipeline`.
+ *
+ * Pipeline tiers:
+ *   Tier 1 (local, fast):   PaddleOCR or Tesseract for printed text
+ *   Tier 2 (local, medium): TrOCR for handwriting, Florence-2 for layout, CLIP embeddings
+ *   Tier 3 (cloud, slow):   a vision model of the provider whose key is set
+ *
+ * The local tiers need their optional packages (ppu-paddle-ocr or tesseract.js,
+ * and @huggingface/transformers); AgentOS uses the ones it finds installed.
+ * The cloud tier reads OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY (or
+ * GEMINI_API_KEY) or OPENROUTER_API_KEY from the environment.
+ *
+ * @module @framers/agentos-ext-vision-pipeline
+ */
+import { createRequire } from 'node:module';
+import { createVisionPipeline } from '@framers/agentos';
+import { VisionPipelineTool } from './tools/visionPipeline.js';
+const { version } = createRequire(import.meta.url)('../package.json');
+/**
+ * Create the Vision & OCR Pipeline extension pack.
+ *
+ * The pack builds a pipeline per strategy the first time a call needs it:
+ * building one probes for the optional OCR and model packages, which loading
+ * the pack does not do. A build that fails is tried again on the next call.
+ */
+export function createExtensionPack(context = {}) {
+    // An OpenAI key from the options or the secrets makes OpenAI the cloud tier's
+    // provider, with that key. Without one, AgentOS detects the provider and its
+    // key from the environment. AgentOS releases whose VisionPipelineConfig has
+    // no cloudApiKey ignore it and read OPENAI_API_KEY.
+    const firstKey = (...values) => values.map((value) => (typeof value === 'string' ? value.trim() : '')).find(Boolean) || undefined;
+    const openaiKey = firstKey(context.options?.openaiApiKey, context.getSecret?.('openai.apiKey'));
+    const cloud = openaiKey ? { cloudProvider: 'openai', cloudApiKey: openaiKey } : {};
+    const pipelines = new Map();
+    // Set while the pack is deactivated: a call then gets no new pipeline,
+    // which deactivation would not dispose of.
+    let deactivated = false;
+    const pipelineFor = (strategy) => {
+        if (deactivated)
+            return Promise.reject(new Error('The Vision & OCR Pipeline pack is deactivated.'));
+        let pipeline = pipelines.get(strategy);
+        if (!pipeline) {
+            pipeline = createVisionPipeline({ strategy, ...cloud });
+            pipelines.set(strategy, pipeline);
+            const built = pipeline;
+            built.catch(() => {
+                if (pipelines.get(strategy) === built)
+                    pipelines.delete(strategy);
+            });
+        }
+        return pipeline;
+    };
+    const tool = new VisionPipelineTool(pipelineFor);
+    return {
+        name: '@framers/agentos-ext-vision-pipeline',
+        version,
+        descriptors: [
+            {
+                id: tool.name,
+                kind: 'tool',
+                priority: context.options?.priority ?? 45,
+                payload: tool,
+                requiredSecrets: [{ id: 'openai.apiKey', optional: true }],
+            },
+        ],
+        onActivate: async () => {
+            deactivated = false;
+            context.logger?.info('Vision & OCR Pipeline Extension activated');
+        },
+        onDeactivate: async () => {
+            deactivated = true;
+            const built = [...pipelines.values()];
+            pipelines.clear();
+            for (const pipeline of built) {
+                let instance;
+                try {
+                    instance = await pipeline;
+                }
+                catch {
+                    continue; // A pipeline that failed to build has nothing to release.
+                }
+                try {
+                    await instance.dispose();
+                }
+                catch (error) {
+                    const log = context.logger?.warn ?? context.logger?.info;
+                    log?.call(context.logger, `Vision & OCR Pipeline: disposing of a pipeline failed: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+            context.logger?.info('Vision & OCR Pipeline Extension deactivated');
+        },
+    };
+}
+export { VisionPipelineTool, imageInput } from './tools/visionPipeline.js';
+export default createExtensionPack;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAuB,MAAM,2BAA2B,CAAC;AAEpF,MAAM,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;AA0BtE;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,UAA4B,EAAE;IAChE,8EAA8E;IAC9E,6EAA6E;IAC7E,4EAA4E;IAC5E,oDAAoD;IACpD,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAiB,EAAE,EAAE,CACxC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC;IACpG,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC;IAChG,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC1D,uEAAuE;IACvE,2CAA2C;IAC3C,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,MAAM,WAAW,GAAG,CAAC,QAAwB,EAAE,EAAE;QAC/C,IAAI,WAAW;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC,CAAC;QACpG,IAAI,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,oBAAoB,CAAC,EAAE,QAAQ,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;YACxD,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,QAAQ,CAAC;YACvB,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;gBACf,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK;oBAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpE,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAEjD,OAAO;QACL,IAAI,EAAE,sCAAsC;QAC5C,OAAO;QACP,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,IAAI,CAAC,IAAI;gBACb,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE;gBACzC,OAAO,EAAE,IAAI;gBACb,eAAe,EAAE,CAAC,EAAE,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;aAC3D;SACF;QACD,UAAU,EAAE,KAAK,IAAI,EAAE;YACrB,WAAW,GAAG,KAAK,CAAC;YACpB,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,2CAA2C,CAAC,CAAC;QACpE,CAAC;QACD,YAAY,EAAE,KAAK,IAAI,EAAE;YACvB,WAAW,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;YACtC,SAAS,CAAC,KAAK,EAAE,CAAC;YAClB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,IAAI,QAAQ,CAAC;gBACb,IAAI,CAAC;oBACH,QAAQ,GAAG,MAAM,QAAQ,CAAC;gBAC5B,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,CAAC,0DAA0D;gBACtE,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;gBAC3B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC;oBACzD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,0DAA0D,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAChJ,CAAC;YACH,CAAC;YACD,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,6CAA6C,CAAC,CAAC;QACtE,CAAC;KACF,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE3E,eAAe,mBAAmB,CAAC"}

package/dist/tools/visionPipeline.d.ts

@@ -0,0 +1,72 @@
+/**
+ * @fileoverview The vision-pipeline tool: OCR, handwriting, document layout,
+ * image description and CLIP embeddings through an AgentOS VisionPipeline.
+ */
+import type { ITool, JSONSchemaObject, ToolExecutionContext, ToolExecutionResult } from '@framers/agentos';
+/** What the tool reads from an image; see {@link VisionPipelineTool}. */
+export type VisionMode = 'ocr' | 'handwriting' | 'layout' | 'describe' | 'embed' | 'auto';
+/** The vision-pipeline tool's input. */
+export interface VisionPipelineInput {
+    /** The image: a data:image URL, or an http(s) URL on a public host. */
+    imageUrl: string;
+    /** What to read (default `auto`). */
+    mode?: VisionMode;
+    /** The highest tier to use: 1 local OCR, 2 local vision models, 3 cloud vision (default). */
+    maxTier?: 1 | 2 | 3;
+}
+/** The vision-pipeline tool's output: the text found, or for `embed` the vector. */
+export interface VisionPipelineOutput {
+    mode: VisionMode;
+    text?: string;
+    confidence?: number;
+    category?: string;
+    /** The pipeline tiers that produced the result. */
+    tiers?: string[];
+    /** How many text regions the winning tier located. */
+    regions?: number;
+    durationMs?: number;
+    /** For embed: the CLIP vector. */
+    embedding?: number[];
+    dimensions?: number;
+}
+/** The pipeline strategy a call runs under: tier 3 may reach cloud vision, tiers 1 and 2 stay local. */
+export type VisionStrategy = 'progressive' | 'local-only';
+/** Returns the pipeline for a strategy, made on first use. */
+export type PipelineFor = (strategy: VisionStrategy) => Promise<{
+    process(image: Buffer | string, options?: {
+        forceCategory?: string;
+        tiers?: string[];
+    }): Promise<any>;
+    embed(image: Buffer | string): Promise<number[]>;
+}>;
+/**
+ * Whether a URL's host is this machine or off the public internet: `localhost`
+ * names, and the IPv4 and IPv6 literals above. The URL parser has already
+ * written other IPv4 spellings (`0x7f000001`, `127.1`) as dotted numbers. An
+ * IPv6 literal that cannot be read is treated as not public. The check reads
+ * the host as written: the address a name resolves to, and the target of a
+ * redirect, are not checked here.
+ */
+export declare function isPrivateHost(hostname: string): boolean;
+/**
+ * The image as the pipeline takes it: a data URL decoded to its bytes, an
+ * http(s) URL whose host is not this machine or a private network as is, or
+ * `undefined` for anything else. AgentOS also reads local file paths and
+ * fetches any URL, but a tool the model calls must not send the machine's
+ * files, or what its network serves, to a cloud vision model.
+ */
+export declare function imageInput(value: unknown): Buffer | string | undefined;
+/** The vision-pipeline tool, on an AgentOS `VisionPipeline` per strategy. */
+export declare class VisionPipelineTool implements ITool<VisionPipelineInput, VisionPipelineOutput> {
+    private readonly pipelineFor;
+    readonly id = "tool.vision-pipeline";
+    readonly name = "vision-pipeline";
+    readonly displayName = "Vision & OCR Pipeline";
+    readonly description: string;
+    readonly category = "media";
+    readonly hasSideEffects = false;
+    readonly inputSchema: JSONSchemaObject;
+    constructor(pipelineFor: PipelineFor);
+    execute(args: VisionPipelineInput, _context?: ToolExecutionContext): Promise<ToolExecutionResult<VisionPipelineOutput>>;
+}
+//# sourceMappingURL=visionPipeline.d.ts.map

package/dist/tools/visionPipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"visionPipeline.d.ts","sourceRoot":"","sources":["../../src/tools/visionPipeline.ts"],"names":[],"mappings":"AACA;;;GAGG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAE3G,yEAAyE;AACzE,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,aAAa,GAAG,QAAQ,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;AAE1F,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,qCAAqC;IACrC,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,6FAA6F;IAC7F,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;CACrB;AAED,oFAAoF;AACpF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wGAAwG;AACxG,MAAM,MAAM,cAAc,GAAG,aAAa,GAAG,YAAY,CAAC;AAE1D,8DAA8D;AAC9D,MAAM,MAAM,WAAW,GAAG,CAAC,QAAQ,EAAE,cAAc,KAAK,OAAO,CAAC;IAC9D,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACtG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAClD,CAAC,CAAC;AA+EH;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAQvD;AAiCD;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAetE;AAED,6EAA6E;AAC7E,qBAAa,kBAAmB,YAAW,KAAK,CAAC,mBAAmB,EAAE,oBAAoB,CAAC;IA8B7E,OAAO,CAAC,QAAQ,CAAC,WAAW;IA7BxC,QAAQ,CAAC,EAAE,0BAA0B;IACrC,QAAQ,CAAC,IAAI,qBAAqB;IAClC,QAAQ,CAAC,WAAW,2BAA2B;IAC/C,QAAQ,CAAC,WAAW,SAE8F;IAClH,QAAQ,CAAC,QAAQ,WAAW;IAC5B,QAAQ,CAAC,cAAc,SAAS;IAEhC,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAkBpC;gBAE2B,WAAW,EAAE,WAAW;IAE/C,OAAO,CAAC,IAAI,EAAE,mBAAmB,EAAE,QAAQ,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;CAkD9H"}

package/dist/tools/visionPipeline.js

@@ -0,0 +1,247 @@
+// @ts-nocheck
+/**
+ * @fileoverview The vision-pipeline tool: OCR, handwriting, document layout,
+ * image description and CLIP embeddings through an AgentOS VisionPipeline.
+ */
+/**
+ * The lowest tier each mode needs: 1 is local OCR, 2 local vision models
+ * (TrOCR, Florence-2, CLIP), 3 cloud vision.
+ */
+const TIER_NEEDED = {
+    auto: 1,
+    ocr: 1,
+    handwriting: 2,
+    layout: 2,
+    embed: 2,
+    describe: 3,
+};
+/**
+ * Whether an IPv4 address, as its four numbers, is off the public internet:
+ * "this network", private, carrier-grade NAT, loopback, link-local (cloud
+ * metadata services answer at 169.254.169.254), the IETF protocol block, the
+ * documentation and benchmarking ranges, multicast, reserved and broadcast.
+ */
+function isNonPublicIPv4([a, b, c]) {
+    return (a === 0 ||
+        a === 10 ||
+        a === 127 ||
+        (a === 100 && b >= 64 && b <= 127) ||
+        (a === 169 && b === 254) ||
+        (a === 172 && b >= 16 && b <= 31) ||
+        (a === 192 && b === 0 && (c === 0 || c === 2)) ||
+        (a === 192 && b === 168) ||
+        (a === 198 && (b === 18 || b === 19)) ||
+        (a === 198 && b === 51 && c === 100) ||
+        (a === 203 && b === 0 && c === 113) ||
+        a >= 224);
+}
+/** The eight 16-bit groups of an IPv6 address, or `undefined` when the text is not one. */
+function ipv6Groups(address) {
+    let text = address;
+    // A trailing dotted IPv4 (::ffff:1.2.3.4) is two groups.
+    const dotted = /^(.*:)(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(text);
+    if (dotted) {
+        const [w, x, y, z] = dotted.slice(2).map(Number);
+        if ([w, x, y, z].some((n) => n > 255))
+            return undefined;
+        text = `${dotted[1]}${((w << 8) | x).toString(16)}:${((y << 8) | z).toString(16)}`;
+    }
+    const halves = text.split('::');
+    if (halves.length > 2)
+        return undefined;
+    const head = halves[0] ? halves[0].split(':') : [];
+    const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
+    const fill = halves.length === 2 ? 8 - head.length - tail.length : 0;
+    if (halves.length === 2 ? fill < 1 : head.length !== 8)
+        return undefined;
+    const groups = [...head, ...Array(fill).fill('0'), ...tail];
+    if (!groups.every((group) => /^[0-9a-f]{1,4}$/.test(group)))
+        return undefined;
+    return groups.map((group) => parseInt(group, 16));
+}
+/**
+ * Whether an IPv6 address is off the public internet, the IPv4 address it
+ * carries included: unspecified, loopback, IPv4-compatible and IPv4-mapped,
+ * NAT64 (64:ff9b::/96 and 64:ff9b:1::/48), 6to4 (2002::/16), discard
+ * (100::/64), documentation (2001:db8::/32), unique local, link-local,
+ * site-local and multicast.
+ */
+function isNonPublicIPv6(groups) {
+    const carried = (high, low) => isNonPublicIPv4([high >> 8, high & 255, low >> 8, low & 255]);
+    const [g0, g1, g2, g3, g4, g5, g6, g7] = groups;
+    const zeros = (from, to) => groups.slice(from, to).every((group) => group === 0);
+    if (zeros(0, 6))
+        return (g6 === 0 && g7 <= 1) || carried(g6, g7);
+    if (zeros(0, 5) && g5 === 0xffff)
+        return carried(g6, g7);
+    if (g0 === 0x64 && g1 === 0xff9b)
+        return g2 === 1 || !zeros(2, 6) || carried(g6, g7);
+    if (g0 === 0x2002)
+        return carried(g1, g2);
+    if (g0 === 0x100 && zeros(1, 4))
+        return true;
+    if (g0 === 0x2001 && g1 === 0xdb8)
+        return true;
+    return (g0 & 0xfe00) === 0xfc00 || (g0 & 0xffc0) === 0xfe80 || (g0 & 0xffc0) === 0xfec0 || (g0 & 0xff00) === 0xff00;
+}
+/**
+ * Whether a URL's host is this machine or off the public internet: `localhost`
+ * names, and the IPv4 and IPv6 literals above. The URL parser has already
+ * written other IPv4 spellings (`0x7f000001`, `127.1`) as dotted numbers. An
+ * IPv6 literal that cannot be read is treated as not public. The check reads
+ * the host as written: the address a name resolves to, and the target of a
+ * redirect, are not checked here.
+ */
+export function isPrivateHost(hostname) {
+    const host = hostname.toLowerCase().replace(/^\[|\]$/g, '').replace(/\.$/, '');
+    if (host === 'localhost' || host.endsWith('.localhost'))
+        return true;
+    const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
+    if (v4)
+        return isNonPublicIPv4(v4.slice(1).map(Number));
+    if (!host.includes(':'))
+        return false;
+    const groups = ipv6Groups(host);
+    return groups === undefined || isNonPublicIPv6(groups);
+}
+/** The value of an ASCII hex digit, or -1. */
+function hexValue(byte) {
+    if (byte >= 0x30 && byte <= 0x39)
+        return byte - 0x30;
+    if (byte >= 0x41 && byte <= 0x46)
+        return byte - 0x37;
+    if (byte >= 0x61 && byte <= 0x66)
+        return byte - 0x57;
+    return -1;
+}
+/**
+ * The bytes of a percent-encoded data URL payload: each `%XX` is one byte,
+ * any other character its UTF-8 bytes. A `%` that starts no valid escape is
+ * kept as it is, so the decoding never throws. One pass over two buffers.
+ */
+function percentDecode(payload) {
+    const text = Buffer.from(payload, 'utf8');
+    const out = Buffer.allocUnsafe(text.length);
+    let length = 0;
+    for (let i = 0; i < text.length; i += 1) {
+        const high = text[i] === 0x25 && i + 2 < text.length ? hexValue(text[i + 1]) : -1;
+        const low = high >= 0 ? hexValue(text[i + 2]) : -1;
+        if (low >= 0) {
+            out[length] = high * 16 + low;
+            i += 2;
+        }
+        else {
+            out[length] = text[i];
+        }
+        length += 1;
+    }
+    return out.subarray(0, length);
+}
+/**
+ * The image as the pipeline takes it: a data URL decoded to its bytes, an
+ * http(s) URL whose host is not this machine or a private network as is, or
+ * `undefined` for anything else. AgentOS also reads local file paths and
+ * fetches any URL, but a tool the model calls must not send the machine's
+ * files, or what its network serves, to a cloud vision model.
+ */
+export function imageInput(value) {
+    if (typeof value !== 'string')
+        return undefined;
+    const source = value.trim();
+    if (/^https?:\/\//i.test(source)) {
+        let url;
+        try {
+            url = new URL(source);
+        }
+        catch {
+            return undefined;
+        }
+        return isPrivateHost(url.hostname) ? undefined : source;
+    }
+    const data = /^data:image\/[^,]*?(;base64)?,(.*)$/is.exec(source);
+    if (!data)
+        return undefined;
+    return data[1] ? Buffer.from(data[2], 'base64') : percentDecode(data[2]);
+}
+/** The vision-pipeline tool, on an AgentOS `VisionPipeline` per strategy. */
+export class VisionPipelineTool {
+    pipelineFor;
+    id = 'tool.vision-pipeline';
+    name = 'vision-pipeline';
+    displayName = 'Vision & OCR Pipeline';
+    description = 'Read and understand an image: printed text (ocr), handwriting, document layout, a description of the image, ' +
+        'or a CLIP embedding vector. Local OCR runs first; local vision models and cloud vision run as maxTier allows.';
+    category = 'media';
+    hasSideEffects = false;
+    inputSchema = {
+        type: 'object',
+        properties: {
+            imageUrl: { type: 'string', description: 'The image: an http(s) URL or a data:image URL.' },
+            mode: {
+                type: 'string',
+                enum: ['ocr', 'handwriting', 'layout', 'describe', 'embed', 'auto'],
+                description: 'ocr for printed text, handwriting, layout for document structure, describe for what the image shows, ' +
+                    'embed for a CLIP vector, or auto (default) to let the pipeline decide.',
+            },
+            maxTier: {
+                type: 'number',
+                enum: [1, 2, 3],
+                description: 'The highest tier to use: 1 local OCR only, 2 local vision models, 3 cloud vision (default).',
+            },
+        },
+        required: ['imageUrl'],
+    };
+    constructor(pipelineFor) {
+        this.pipelineFor = pipelineFor;
+    }
+    async execute(args, _context) {
+        const image = imageInput(args.imageUrl);
+        if (image === undefined) {
+            return {
+                success: false,
+                error: 'imageUrl must be a data:image URL or an http(s) URL on a public host: local file paths are not read, nor local or private network addresses.',
+            };
+        }
+        const mode = args.mode ?? 'auto';
+        if (typeof mode !== 'string' || !Object.hasOwn(TIER_NEEDED, mode)) {
+            return { success: false, error: `mode must be one of ${Object.keys(TIER_NEEDED).join(', ')}.` };
+        }
+        const maxTier = args.maxTier === 1 || args.maxTier === 2 ? args.maxTier : 3;
+        if (TIER_NEEDED[mode] > maxTier) {
+            return { success: false, error: `${mode} needs maxTier ${TIER_NEEDED[mode]} or higher; this call allows ${maxTier}.` };
+        }
+        try {
+            const pipeline = await this.pipelineFor(maxTier === 3 ? 'progressive' : 'local-only');
+            if (mode === 'embed') {
+                const embedding = await pipeline.embed(image);
+                return { success: true, output: { mode, embedding, dimensions: embedding.length } };
+            }
+            const options = maxTier === 1 || mode === 'ocr'
+                ? { tiers: ['ocr'] }
+                : mode === 'handwriting'
+                    ? { forceCategory: 'handwritten' }
+                    : mode === 'layout'
+                        ? { forceCategory: 'document-layout' }
+                        : mode === 'describe'
+                            ? { tiers: ['cloud-vision'] }
+                            : undefined;
+            const result = await pipeline.process(image, options);
+            return {
+                success: true,
+                output: {
+                    mode,
+                    text: result.text,
+                    confidence: result.confidence,
+                    category: result.category,
+                    tiers: result.tiers,
+                    regions: result.regions?.length ?? 0,
+                    durationMs: result.durationMs,
+                },
+            };
+        }
+        catch (error) {
+            return { success: false, error: error instanceof Error ? error.message : String(error) };
+        }
+    }
+}
+//# sourceMappingURL=visionPipeline.js.map

package/dist/tools/visionPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"visionPipeline.js","sourceRoot":"","sources":["../../src/tools/visionPipeline.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;;GAGG;AA0CH;;;GAGG;AACH,MAAM,WAAW,GAAkC;IACjD,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,CAAC;IACN,WAAW,EAAE,CAAC;IACd,MAAM,EAAE,CAAC;IACT,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,CAAC;CACZ,CAAC;AAEF;;;;;GAKG;AACH,SAAS,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAW;IAC1C,OAAO,CACL,CAAC,KAAK,CAAC;QACP,CAAC,KAAK,EAAE;QACR,CAAC,KAAK,GAAG;QACT,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC;QAClC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QACxB,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QACxB,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACrC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC;QACpC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC;QACnC,CAAC,IAAI,GAAG,CACT,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,SAAS,UAAU,CAAC,OAAe;IACjC,IAAI,IAAI,GAAG,OAAO,CAAC;IACnB,yDAAyD;IACzD,MAAM,MAAM,GAAG,mDAAmD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9E,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;YAAE,OAAO,SAAS,CAAC;QACxD,IAAI,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1E,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9E,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;AACpD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,MAAgB;IACvC,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,GAAW,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,IAAI,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;IAC7G,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC;IAChD,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAU,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;IACjG,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACjE,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,MAAM;QAAE,OAAO,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACzD,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,MAAM;QAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACrF,IAAI,EAAE,KAAK,MAAM;QAAE,OAAO,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,EAAE,KAAK,KAAK,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IAC/C,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,MAAM,CAAC;AACtH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC/E,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IACrE,MAAM,EAAE,GAAG,8CAA8C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,EAAE;QAAE,OAAO,eAAe,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,MAAM,KAAK,SAAS,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,8CAA8C;AAC9C,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,IAAI,GAAG,IAAI,CAAC;IACrD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,IAAI,GAAG,IAAI,CAAC;IACrD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,IAAI,GAAG,IAAI,CAAC;IACrD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC;YAC9B,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;QACD,MAAM,IAAI,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,KAAc;IACvC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACjC,IAAI,GAAQ,CAAC;QACb,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1D,CAAC;IACD,MAAM,IAAI,GAAG,uCAAuC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED,6EAA6E;AAC7E,MAAM,OAAO,kBAAkB;IA8BA;IA7BpB,EAAE,GAAG,sBAAsB,CAAC;IAC5B,IAAI,GAAG,iBAAiB,CAAC;IACzB,WAAW,GAAG,uBAAuB,CAAC;IACtC,WAAW,GAClB,8GAA8G;QAC9G,+GAA+G,CAAC;IACzG,QAAQ,GAAG,OAAO,CAAC;IACnB,cAAc,GAAG,KAAK,CAAC;IAEvB,WAAW,GAAqB;QACvC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gDAAgD,EAAE;YAC3F,IAAI,EAAE;gBACJ,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC;gBACnE,WAAW,EACT,uGAAuG;oBACvG,wEAAwE;aAC3E;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACf,WAAW,EAAE,6FAA6F;aAC3G;SACF;QACD,QAAQ,EAAE,CAAC,UAAU,CAAC;KACvB,CAAC;IAEF,YAA6B,WAAwB;QAAxB,gBAAW,GAAX,WAAW,CAAa;IAAG,CAAC;IAEzD,KAAK,CAAC,OAAO,CAAC,IAAyB,EAAE,QAA+B;QACtE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,8IAA8I;aACtJ,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAe,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC;QAC7C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAClG,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAI,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC;YAChC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,kBAAkB,WAAW,CAAC,IAAI,CAAC,gCAAgC,OAAO,GAAG,EAAE,CAAC;QACzH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YACtF,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC9C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YACtF,CAAC;YACD,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,IAAI,IAAI,KAAK,KAAK;gBAC7B,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;gBACpB,CAAC,CAAC,IAAI,KAAK,aAAa;oBACtB,CAAC,CAAC,EAAE,aAAa,EAAE,aAAa,EAAE;oBAClC,CAAC,CAAC,IAAI,KAAK,QAAQ;wBACjB,CAAC,CAAC,EAAE,aAAa,EAAE,iBAAiB,EAAE;wBACtC,CAAC,CAAC,IAAI,KAAK,UAAU;4BACnB,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,cAAc,CAAC,EAAE;4BAC7B,CAAC,CAAC,SAAS,CAAC;YACtB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,IAAI;oBACJ,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC;oBACpC,UAAU,EAAE,MAAM,CAAC,UAAU;iBAC9B;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}

package/manifest.json

@@ -2,16 +2,16 @@
   "$schema": "https://agentos.sh/schemas/extension-manifest-v1.json",
   "id": "com.framers.tools.vision-pipeline",
   "name": "Vision & OCR Pipeline",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Progressive vision pipeline with PaddleOCR, TrOCR, Florence-2, CLIP, and cloud vision (GPT-4o, Claude).",
   "author": {
     "name": "Frame.dev",
     "email": "support@frame.dev",
-    "url": "https://github.com/framersai"
+    "url": "https://github.com/framerslab"
   },
   "license": "MIT",
   "keywords": ["vision", "ocr", "paddle", "trocr", "florence", "clip", "document-ai"],
-  "agentosVersion": "^2.0.0",
+  "agentosVersion": ">=0.10.40",
   "categories": ["tool", "media"],
   "platforms": [],
   "sdk": "none",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@framers/agentos-ext-vision-pipeline",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Vision & OCR pipeline extension for AgentOS — PaddleOCR, TrOCR, Florence-2, CLIP, and cloud vision",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -38,26 +38,35 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/framersai/agentos-extensions.git",
+    "url": "git+https://github.com/framerslab/agentos-extensions.git",
     "directory": "registry/curated/tools/vision-pipeline"
   },
   "publishConfig": {
     "access": "public"
   },
   "peerDependencies": {
-    "@framers/agentos": "^0.5.2"
+    "@framers/agentos": ">=0.10.40"
   },
   "devDependencies": {
-    "@framers/agentos": "^0.5.2",
+    "@framers/agentos": "^0.12.8",
     "@types/node": "^20.12.12",
     "rimraf": "^5.0.7",
     "typescript": "^5.4.5",
-    "vitest": "^1.6.0"
+    "vitest": "^3.2.7"
   },
+  "homepage": "https://github.com/framerslab/agentos-extensions/tree/master/registry/curated/tools/vision-pipeline#readme",
+  "bugs": {
+    "url": "https://github.com/framerslab/agentos-extensions/issues"
+  },
+  "files": [
+    "dist",
+    "manifest.json",
+    "SKILL.md"
+  ],
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "test": "vitest run",
+    "test": "vitest run --passWithNoTests",
     "test:watch": "vitest",
     "clean": "rimraf dist"
   }

package/src/index.ts

@@ -1,116 +0,0 @@
-// @ts-nocheck
-/**
- * @fileoverview Vision & OCR Pipeline Extension Pack — progressive multi-tier
- * vision pipeline as an agent tool.
- *
- * This is a minimal stub pack. The actual vision pipeline implementation is
- * provided by the core AgentOS vision module; this extension pack registers
- * the tool descriptor so the agent can discover and invoke it.
- *
- * Pipeline tiers:
- *   Tier 1 (local, fast):   PaddleOCR for printed text
- *   Tier 2 (local, medium): TrOCR for handwriting, Florence-2 for layout
- *   Tier 3 (cloud, slow):   GPT-4o / Claude vision for complex understanding
- *
- * CLIP embeddings are generated alongside any tier for semantic image search.
- *
- * @module @framers/agentos-ext-vision-pipeline
- */
-
-/* ------------------------------------------------------------------ */
-/*  Extension pack types                                               */
-/* ------------------------------------------------------------------ */
-
-export interface ExtensionContext {
-  options?: Record<string, unknown>;
-  secrets?: Record<string, string>;
-  getSecret?: (key: string) => string | undefined;
-  logger?: { info: (msg: string) => void };
-}
-
-export interface ExtensionPack {
-  name: string;
-  version: string;
-  descriptors: Array<{
-    id: string;
-    kind: string;
-    priority?: number;
-    payload: unknown;
-    requiredSecrets?: Array<{ id: string; optional?: boolean }>;
-  }>;
-  onActivate?: () => Promise<void>;
-  onDeactivate?: () => Promise<void>;
-}
-
-/* ------------------------------------------------------------------ */
-/*  Stub tool definition                                               */
-/* ------------------------------------------------------------------ */
-
-/**
- * Stub tool for the vision & OCR pipeline.
- * Implementation is resolved at runtime by the AgentOS vision module.
- */
-const visionPipelineTool = {
-  name: 'vision-pipeline',
-  description:
-    'Extract text, understand layout, and generate embeddings from images using a progressive pipeline: PaddleOCR -> TrOCR/Florence-2 -> GPT-4o/Claude vision.',
-  parameters: {
-    type: 'object' as const,
-    properties: {
-      imageUrl: {
-        type: 'string',
-        description: 'URL or local path of the image to analyze',
-      },
-      mode: {
-        type: 'string',
-        enum: ['ocr', 'handwriting', 'layout', 'describe', 'embed', 'auto'],
-        description:
-          'Analysis mode — "ocr" for printed text, "handwriting" for handwritten text, "layout" for document structure, "describe" for general image understanding, "embed" for CLIP vector, "auto" to let the pipeline decide (default: auto)',
-      },
-      maxTier: {
-        type: 'number',
-        enum: [1, 2, 3],
-        description:
-          'Maximum pipeline tier to use — 1 (local OCR only), 2 (local vision models), 3 (cloud vision). Default: 3',
-      },
-      language: {
-        type: 'string',
-        description: 'Language hint for OCR (e.g. "en", "zh", "ja"). Default: "en"',
-      },
-    },
-    required: ['imageUrl'],
-  },
-  execute: async () => {
-    throw new Error('vision-pipeline stub — wire to VisionPipelineService at runtime.');
-  },
-};
-
-/* ------------------------------------------------------------------ */
-/*  Factory                                                            */
-/* ------------------------------------------------------------------ */
-
-/**
- * Create the Vision & OCR Pipeline extension pack.
- *
- * Registers the vision-pipeline tool. Actual implementation is provided
- * by the core AgentOS vision module and wired at runtime.
- */
-export function createExtensionPack(context: ExtensionContext): ExtensionPack {
-  return {
-    name: '@framers/agentos-ext-vision-pipeline',
-    version: '0.1.0',
-    descriptors: [
-      {
-        id: visionPipelineTool.name,
-        kind: 'tool' as const,
-        priority: 45,
-        payload: visionPipelineTool,
-        requiredSecrets: [{ id: 'openai.apiKey', optional: true }],
-      },
-    ],
-    onActivate: async () => context.logger?.info('Vision & OCR Pipeline Extension activated'),
-    onDeactivate: async () => context.logger?.info('Vision & OCR Pipeline Extension deactivated'),
-  };
-}
-
-export default createExtensionPack;