@framers/agentos-ext-vision-pipeline 0.1.0 → 0.2.1

package/SKILL.md

@@ -8,13 +8,24 @@ tools_required: [vision-pipeline]
 
 # Vision & OCR
 
-Extract text from images, documents, and handwritten notes using a progressive 3-tier pipeline: local OCR (PaddleOCR) -> local vision models (TrOCR, Florence-2) -> cloud vision (GPT-4o, Claude).
+Extract text from images, documents, and handwritten notes with a progressive 3-tier pipeline: local OCR (PaddleOCR or Tesseract), then local vision models (TrOCR, Florence-2), then cloud vision.
 
-## Capabilities
-- **Printed text OCR**: Extract text from documents, receipts, screenshots
-- **Handwriting recognition**: Read handwritten notes and forms via TrOCR
-- **Document layout**: Understand tables, figures, headings via Florence-2
-- **Image embeddings**: Generate CLIP vectors for semantic image search
+## Tool
+**vision-pipeline** takes `imageUrl`, `mode` and `maxTier`. `imageUrl` is a `data:image/...` URL or an http(s) URL on a public host: local file paths and addresses on this machine or a private network are refused, so a model cannot send the machine's files, or what its network serves, to a cloud model. With AgentOS 0.13.16 or later, the tool fetches an http(s) image through `imageToBuffer`'s untrusted mode and hands the pipeline the bytes: every address the host resolves to is checked when the connection is made, each redirect is checked the same way, and the fetch stops at 50 MiB and 30 seconds. With an older AgentOS the check reads the URL's host as written, so the address a public name resolves to and a redirect's target are not checked.
+
+| mode | what it returns | lowest maxTier |
+|------|-----------------|----------------|
+| `auto` (default) | the pipeline's best text | 1 |
+| `ocr` | printed text | 1 |
+| `handwriting` | handwritten text (TrOCR) | 2 |
+| `layout` | document text and structure (Florence-2) | 2 |
+| `embed` | a CLIP embedding vector | 2 |
+| `describe` | what the image shows (cloud vision) | 3 |
+
+`maxTier` 1 runs local OCR alone, 2 adds the local vision models, 3 (default) may use cloud vision.
+
+## Setup
+The local tiers use the optional packages AgentOS finds installed: `ppu-paddle-ocr` or `tesseract.js` for OCR, and `@huggingface/transformers` for handwriting, layout and embeddings. With an OpenAI key in the pack option `openaiApiKey` or the secret `openai.apiKey`, the cloud tier uses OpenAI with that key. Without one, it uses the first provider whose key is in the environment: `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY` (or `GEMINI_API_KEY`), `OPENROUTER_API_KEY`. `describe` and the cloud tier need `@framers/agentos` 0.12.14 or later: earlier releases send the image to the cloud model as text, and read the key from the environment only.
 
 ## Example
 "Read the text from this receipt"

package/dist/index.d.ts

@@ -0,0 +1,59 @@
+/**
+ * @fileoverview Vision & OCR Pipeline Extension Pack: the vision-pipeline tool
+ * on AgentOS's `createVisionPipeline`.
+ *
+ * Pipeline tiers:
+ *   Tier 1 (local, fast):   PaddleOCR or Tesseract for printed text
+ *   Tier 2 (local, medium): TrOCR for handwriting, Florence-2 for layout, CLIP embeddings
+ *   Tier 3 (cloud, slow):   a vision model of the provider whose key is set
+ *
+ * The local tiers need their optional packages (ppu-paddle-ocr or tesseract.js,
+ * and @huggingface/transformers); AgentOS uses the ones it finds installed.
+ * The cloud tier reads OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY (or
+ * GEMINI_API_KEY) or OPENROUTER_API_KEY from the environment.
+ *
+ * @module @framers/agentos-ext-vision-pipeline
+ */
+/** What the extension manager passes the pack factory. */
+export interface ExtensionContext {
+    /** `priority` of the tool descriptor (default 45); `openaiApiKey` for the cloud tier. */
+    options?: {
+        priority?: number;
+        openaiApiKey?: string;
+    } & Record<string, unknown>;
+    /** Reads a secret, such as `openai.apiKey`. */
+    getSecret?: (key: string) => string | undefined;
+    logger?: {
+        info: (msg: string) => void;
+        warn?: (msg: string) => void;
+    };
+}
+/** The pack the factory returns: one `vision-pipeline` tool descriptor. */
+export interface ExtensionPack {
+    name: string;
+    version: string;
+    descriptors: Array<{
+        id: string;
+        kind: string;
+        priority?: number;
+        payload: unknown;
+        requiredSecrets?: Array<{
+            id: string;
+            optional?: boolean;
+        }>;
+    }>;
+    onActivate?: () => Promise<void>;
+    onDeactivate?: () => Promise<void>;
+}
+/**
+ * Create the Vision & OCR Pipeline extension pack.
+ *
+ * The pack builds a pipeline per strategy the first time a call needs it:
+ * building one probes for the optional OCR and model packages, which loading
+ * the pack does not do. A build that fails is tried again on the next call.
+ */
+export declare function createExtensionPack(context?: ExtensionContext): ExtensionPack;
+export { VisionPipelineTool, imageInput } from './tools/visionPipeline.js';
+export type { VisionPipelineInput, VisionPipelineOutput, VisionMode } from './tools/visionPipeline.js';
+export default createExtensionPack;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA;;;;;;;;;;;;;;;GAeG;AAQH,0DAA0D;AAC1D,MAAM,WAAW,gBAAgB;IAC/B,yFAAyF;IACzF,OAAO,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjF,+CAA+C;IAC/C,SAAS,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAC;IAChD,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;QAAC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAA;KAAE,CAAC;CACxE;AAED,2EAA2E;AAC3E,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,KAAK,CAAC;QACjB,EAAE,EAAE,MAAM,CAAC;QACX,IAAI,EAAE,MAAM,CAAC;QACb,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,OAAO,CAAC;QACjB,eAAe,CAAC,EAAE,KAAK,CAAC;YAAE,EAAE,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;SAAE,CAAC,CAAC;KAC7D,CAAC,CAAC;IACH,UAAU,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IACjC,YAAY,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CACpC;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,GAAE,gBAAqB,GAAG,aAAa,CAiEjF;AAED,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAC3E,YAAY,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvG,eAAe,mBAAmB,CAAC"}

package/dist/index.js

@@ -0,0 +1,99 @@
+// @ts-nocheck
+/**
+ * @fileoverview Vision & OCR Pipeline Extension Pack: the vision-pipeline tool
+ * on AgentOS's `createVisionPipeline`.
+ *
+ * Pipeline tiers:
+ *   Tier 1 (local, fast):   PaddleOCR or Tesseract for printed text
+ *   Tier 2 (local, medium): TrOCR for handwriting, Florence-2 for layout, CLIP embeddings
+ *   Tier 3 (cloud, slow):   a vision model of the provider whose key is set
+ *
+ * The local tiers need their optional packages (ppu-paddle-ocr or tesseract.js,
+ * and @huggingface/transformers); AgentOS uses the ones it finds installed.
+ * The cloud tier reads OPENAI_API_KEY, ANTHROPIC_API_KEY, GOOGLE_API_KEY (or
+ * GEMINI_API_KEY) or OPENROUTER_API_KEY from the environment.
+ *
+ * @module @framers/agentos-ext-vision-pipeline
+ */
+import { createRequire } from 'node:module';
+import { createVisionPipeline } from '@framers/agentos';
+import { VisionPipelineTool } from './tools/visionPipeline.js';
+const { version } = createRequire(import.meta.url)('../package.json');
+/**
+ * Create the Vision & OCR Pipeline extension pack.
+ *
+ * The pack builds a pipeline per strategy the first time a call needs it:
+ * building one probes for the optional OCR and model packages, which loading
+ * the pack does not do. A build that fails is tried again on the next call.
+ */
+export function createExtensionPack(context = {}) {
+    // An OpenAI key from the options or the secrets makes OpenAI the cloud tier's
+    // provider, with that key. Without one, AgentOS detects the provider and its
+    // key from the environment. AgentOS releases whose VisionPipelineConfig has
+    // no cloudApiKey ignore it and read OPENAI_API_KEY.
+    const firstKey = (...values) => values.map((value) => (typeof value === 'string' ? value.trim() : '')).find(Boolean) || undefined;
+    const openaiKey = firstKey(context.options?.openaiApiKey, context.getSecret?.('openai.apiKey'));
+    const cloud = openaiKey ? { cloudProvider: 'openai', cloudApiKey: openaiKey } : {};
+    const pipelines = new Map();
+    // Set while the pack is deactivated: a call then gets no new pipeline,
+    // which deactivation would not dispose of.
+    let deactivated = false;
+    const pipelineFor = (strategy) => {
+        if (deactivated)
+            return Promise.reject(new Error('The Vision & OCR Pipeline pack is deactivated.'));
+        let pipeline = pipelines.get(strategy);
+        if (!pipeline) {
+            pipeline = createVisionPipeline({ strategy, ...cloud });
+            pipelines.set(strategy, pipeline);
+            const built = pipeline;
+            built.catch(() => {
+                if (pipelines.get(strategy) === built)
+                    pipelines.delete(strategy);
+            });
+        }
+        return pipeline;
+    };
+    const tool = new VisionPipelineTool(pipelineFor);
+    return {
+        name: '@framers/agentos-ext-vision-pipeline',
+        version,
+        descriptors: [
+            {
+                id: tool.name,
+                kind: 'tool',
+                priority: context.options?.priority ?? 45,
+                payload: tool,
+                requiredSecrets: [{ id: 'openai.apiKey', optional: true }],
+            },
+        ],
+        onActivate: async () => {
+            deactivated = false;
+            context.logger?.info('Vision & OCR Pipeline Extension activated');
+        },
+        onDeactivate: async () => {
+            deactivated = true;
+            const built = [...pipelines.values()];
+            pipelines.clear();
+            for (const pipeline of built) {
+                let instance;
+                try {
+                    instance = await pipeline;
+                }
+                catch {
+                    continue; // A pipeline that failed to build has nothing to release.
+                }
+                try {
+                    await instance.dispose();
+                }
+                catch (error) {
+                    const log = context.logger?.warn ?? context.logger?.info;
+                    log?.call(context.logger, `Vision & OCR Pipeline: disposing of a pipeline failed: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+            context.logger?.info('Vision & OCR Pipeline Extension deactivated');
+        },
+    };
+}
+export { VisionPipelineTool, imageInput } from './tools/visionPipeline.js';
+export default createExtensionPack;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAuB,MAAM,2BAA2B,CAAC;AAEpF,MAAM,EAAE,OAAO,EAAE,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAC,CAAC;AA0BtE;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,UAA4B,EAAE;IAChE,8EAA8E;IAC9E,6EAA6E;IAC7E,4EAA4E;IAC5E,oDAAoD;IACpD,MAAM,QAAQ,GAAG,CAAC,GAAG,MAAiB,EAAE,EAAE,CACxC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC;IACpG,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,YAAY,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC;IAChG,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACnF,MAAM,SAAS,GAAG,IAAI,GAAG,EAAgC,CAAC;IAC1D,uEAAuE;IACvE,2CAA2C;IAC3C,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,MAAM,WAAW,GAAG,CAAC,QAAwB,EAAE,EAAE;QAC/C,IAAI,WAAW;YAAE,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC,CAAC;QACpG,IAAI,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,oBAAoB,CAAC,EAAE,QAAQ,EAAE,GAAG,KAAK,EAAE,CAAC,CAAC;YACxD,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAClC,MAAM,KAAK,GAAG,QAAQ,CAAC;YACvB,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;gBACf,IAAI,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,KAAK;oBAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACpE,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,IAAI,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAEjD,OAAO;QACL,IAAI,EAAE,sCAAsC;QAC5C,OAAO;QACP,WAAW,EAAE;YACX;gBACE,EAAE,EAAE,IAAI,CAAC,IAAI;gBACb,IAAI,EAAE,MAAM;gBACZ,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,IAAI,EAAE;gBACzC,OAAO,EAAE,IAAI;gBACb,eAAe,EAAE,CAAC,EAAE,EAAE,EAAE,eAAe,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;aAC3D;SACF;QACD,UAAU,EAAE,KAAK,IAAI,EAAE;YACrB,WAAW,GAAG,KAAK,CAAC;YACpB,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,2CAA2C,CAAC,CAAC;QACpE,CAAC;QACD,YAAY,EAAE,KAAK,IAAI,EAAE;YACvB,WAAW,GAAG,IAAI,CAAC;YACnB,MAAM,KAAK,GAAG,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;YACtC,SAAS,CAAC,KAAK,EAAE,CAAC;YAClB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;gBAC7B,IAAI,QAAQ,CAAC;gBACb,IAAI,CAAC;oBACH,QAAQ,GAAG,MAAM,QAAQ,CAAC;gBAC5B,CAAC;gBAAC,MAAM,CAAC;oBACP,SAAS,CAAC,0DAA0D;gBACtE,CAAC;gBACD,IAAI,CAAC;oBACH,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAC;gBAC3B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC;oBACzD,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,0DAA0D,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAChJ,CAAC;YACH,CAAC;YACD,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,6CAA6C,CAAC,CAAC;QACtE,CAAC;KACF,CAAC;AACJ,CAAC;AAED,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AAE3E,eAAe,mBAAmB,CAAC"}

package/dist/tools/visionPipeline.d.ts

@@ -0,0 +1,82 @@
+/**
+ * @fileoverview The vision-pipeline tool: OCR, handwriting, document layout,
+ * image description and CLIP embeddings through an AgentOS VisionPipeline.
+ */
+import type { ITool, JSONSchemaObject, ToolExecutionContext, ToolExecutionResult } from '@framers/agentos';
+/** What the tool reads from an image; see {@link VisionPipelineTool}. */
+export type VisionMode = 'ocr' | 'handwriting' | 'layout' | 'describe' | 'embed' | 'auto';
+/** The vision-pipeline tool's input. */
+export interface VisionPipelineInput {
+    /** The image: a data:image URL, or an http(s) URL on a public host. */
+    imageUrl: string;
+    /** What to read (default `auto`). */
+    mode?: VisionMode;
+    /** The highest tier to use: 1 local OCR, 2 local vision models, 3 cloud vision (default). */
+    maxTier?: 1 | 2 | 3;
+}
+/** The vision-pipeline tool's output: the text found, or for `embed` the vector. */
+export interface VisionPipelineOutput {
+    mode: VisionMode;
+    text?: string;
+    confidence?: number;
+    category?: string;
+    /** The pipeline tiers that produced the result. */
+    tiers?: string[];
+    /** How many text regions the winning tier located. */
+    regions?: number;
+    durationMs?: number;
+    /** For embed: the CLIP vector. */
+    embedding?: number[];
+    dimensions?: number;
+}
+/** The pipeline strategy a call runs under: tier 3 may reach cloud vision, tiers 1 and 2 stay local. */
+export type VisionStrategy = 'progressive' | 'local-only';
+/** Returns the pipeline for a strategy, made on first use. */
+export type PipelineFor = (strategy: VisionStrategy) => Promise<{
+    process(image: Buffer | string, options?: {
+        forceCategory?: string;
+        tiers?: string[];
+    }): Promise<any>;
+    embed(image: Buffer | string): Promise<number[]>;
+}>;
+/**
+ * Whether a URL's host is this machine or off the public internet: `localhost`
+ * names, and the IPv4 and IPv6 literals above. The URL parser has already
+ * written other IPv4 spellings (`0x7f000001`, `127.1`) as dotted numbers. An
+ * IPv6 literal that cannot be read is treated as not public. The check reads
+ * the host as written: the address a name resolves to, and the target of a
+ * redirect, are not checked here.
+ */
+export declare function isPrivateHost(hostname: string): boolean;
+/**
+ * The image as the pipeline takes it: a data URL decoded to its bytes, an
+ * http(s) URL whose host is not this machine or a private network as is, or
+ * `undefined` for anything else. AgentOS also reads local file paths and
+ * fetches any URL, but a tool the model calls must not send the machine's
+ * files, or what its network serves, to a cloud vision model.
+ */
+export declare function imageInput(value: unknown): Buffer | string | undefined;
+/**
+ * The image as the tool hands it to the pipeline. A Buffer goes on as it is. An
+ * http(s) URL is fetched here through AgentOS's `imageToBuffer` with
+ * `untrusted: true`, which connects only to public network addresses (every
+ * address the host resolves to, and every redirect, is checked) and stops at
+ * 50 MiB and 30 seconds; the bytes go on. With an AgentOS without that mode
+ * (it has no `isPublicNetworkAddress` export) the pipeline gets the URL,
+ * already checked against the host as written.
+ */
+export declare function loadImage(image: Buffer | string): Promise<Buffer | string>;
+/** The vision-pipeline tool, on an AgentOS `VisionPipeline` per strategy. */
+export declare class VisionPipelineTool implements ITool<VisionPipelineInput, VisionPipelineOutput> {
+    private readonly pipelineFor;
+    readonly id = "tool.vision-pipeline";
+    readonly name = "vision-pipeline";
+    readonly displayName = "Vision & OCR Pipeline";
+    readonly description: string;
+    readonly category = "media";
+    readonly hasSideEffects = false;
+    readonly inputSchema: JSONSchemaObject;
+    constructor(pipelineFor: PipelineFor);
+    execute(args: VisionPipelineInput, _context?: ToolExecutionContext): Promise<ToolExecutionResult<VisionPipelineOutput>>;
+}
+//# sourceMappingURL=visionPipeline.d.ts.map

package/dist/tools/visionPipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"visionPipeline.d.ts","sourceRoot":"","sources":["../../src/tools/visionPipeline.ts"],"names":[],"mappings":"AACA;;;GAGG;AAGH,OAAO,KAAK,EAAE,KAAK,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAC;AAE3G,yEAAyE;AACzE,MAAM,MAAM,UAAU,GAAG,KAAK,GAAG,aAAa,GAAG,QAAQ,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC;AAE1F,wCAAwC;AACxC,MAAM,WAAW,mBAAmB;IAClC,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,qCAAqC;IACrC,IAAI,CAAC,EAAE,UAAU,CAAC;IAClB,6FAA6F;IAC7F,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;CACrB;AAED,oFAAoF;AACpF,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,UAAU,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,mDAAmD;IACnD,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,sDAAsD;IACtD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,kCAAkC;IAClC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wGAAwG;AACxG,MAAM,MAAM,cAAc,GAAG,aAAa,GAAG,YAAY,CAAC;AAE1D,8DAA8D;AAC9D,MAAM,MAAM,WAAW,GAAG,CAAC,QAAQ,EAAE,cAAc,KAAK,OAAO,CAAC;IAC9D,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACtG,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAClD,CAAC,CAAC;AA+EH;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAQvD;AAiCD;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAetE;AAED;;;;;;;;GAQG;AACH,wBAAsB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAIhF;AAED,6EAA6E;AAC7E,qBAAa,kBAAmB,YAAW,KAAK,CAAC,mBAAmB,EAAE,oBAAoB,CAAC;IA8B7E,OAAO,CAAC,QAAQ,CAAC,WAAW;IA7BxC,QAAQ,CAAC,EAAE,0BAA0B;IACrC,QAAQ,CAAC,IAAI,qBAAqB;IAClC,QAAQ,CAAC,WAAW,2BAA2B;IAC/C,QAAQ,CAAC,WAAW,SAE8F;IAClH,QAAQ,CAAC,QAAQ,WAAW;IAC5B,QAAQ,CAAC,cAAc,SAAS;IAEhC,QAAQ,CAAC,WAAW,EAAE,gBAAgB,CAkBpC;gBAE2B,WAAW,EAAE,WAAW;IAE/C,OAAO,CAAC,IAAI,EAAE,mBAAmB,EAAE,QAAQ,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,mBAAmB,CAAC,oBAAoB,CAAC,CAAC;CAmD9H"}

package/dist/tools/visionPipeline.js

@@ -0,0 +1,264 @@
+// @ts-nocheck
+/**
+ * @fileoverview The vision-pipeline tool: OCR, handwriting, document layout,
+ * image description and CLIP embeddings through an AgentOS VisionPipeline.
+ */
+import * as agentos from '@framers/agentos';
+/**
+ * The lowest tier each mode needs: 1 is local OCR, 2 local vision models
+ * (TrOCR, Florence-2, CLIP), 3 cloud vision.
+ */
+const TIER_NEEDED = {
+    auto: 1,
+    ocr: 1,
+    handwriting: 2,
+    layout: 2,
+    embed: 2,
+    describe: 3,
+};
+/**
+ * Whether an IPv4 address, as its four numbers, is off the public internet:
+ * "this network", private, carrier-grade NAT, loopback, link-local (cloud
+ * metadata services answer at 169.254.169.254), the IETF protocol block, the
+ * documentation and benchmarking ranges, multicast, reserved and broadcast.
+ */
+function isNonPublicIPv4([a, b, c]) {
+    return (a === 0 ||
+        a === 10 ||
+        a === 127 ||
+        (a === 100 && b >= 64 && b <= 127) ||
+        (a === 169 && b === 254) ||
+        (a === 172 && b >= 16 && b <= 31) ||
+        (a === 192 && b === 0 && (c === 0 || c === 2)) ||
+        (a === 192 && b === 168) ||
+        (a === 198 && (b === 18 || b === 19)) ||
+        (a === 198 && b === 51 && c === 100) ||
+        (a === 203 && b === 0 && c === 113) ||
+        a >= 224);
+}
+/** The eight 16-bit groups of an IPv6 address, or `undefined` when the text is not one. */
+function ipv6Groups(address) {
+    let text = address;
+    // A trailing dotted IPv4 (::ffff:1.2.3.4) is two groups.
+    const dotted = /^(.*:)(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(text);
+    if (dotted) {
+        const [w, x, y, z] = dotted.slice(2).map(Number);
+        if ([w, x, y, z].some((n) => n > 255))
+            return undefined;
+        text = `${dotted[1]}${((w << 8) | x).toString(16)}:${((y << 8) | z).toString(16)}`;
+    }
+    const halves = text.split('::');
+    if (halves.length > 2)
+        return undefined;
+    const head = halves[0] ? halves[0].split(':') : [];
+    const tail = halves.length === 2 && halves[1] ? halves[1].split(':') : [];
+    const fill = halves.length === 2 ? 8 - head.length - tail.length : 0;
+    if (halves.length === 2 ? fill < 1 : head.length !== 8)
+        return undefined;
+    const groups = [...head, ...Array(fill).fill('0'), ...tail];
+    if (!groups.every((group) => /^[0-9a-f]{1,4}$/.test(group)))
+        return undefined;
+    return groups.map((group) => parseInt(group, 16));
+}
+/**
+ * Whether an IPv6 address is off the public internet, the IPv4 address it
+ * carries included: unspecified, loopback, IPv4-compatible and IPv4-mapped,
+ * NAT64 (64:ff9b::/96 and 64:ff9b:1::/48), 6to4 (2002::/16), discard
+ * (100::/64), documentation (2001:db8::/32), unique local, link-local,
+ * site-local and multicast.
+ */
+function isNonPublicIPv6(groups) {
+    const carried = (high, low) => isNonPublicIPv4([high >> 8, high & 255, low >> 8, low & 255]);
+    const [g0, g1, g2, g3, g4, g5, g6, g7] = groups;
+    const zeros = (from, to) => groups.slice(from, to).every((group) => group === 0);
+    if (zeros(0, 6))
+        return (g6 === 0 && g7 <= 1) || carried(g6, g7);
+    if (zeros(0, 5) && g5 === 0xffff)
+        return carried(g6, g7);
+    if (g0 === 0x64 && g1 === 0xff9b)
+        return g2 === 1 || !zeros(2, 6) || carried(g6, g7);
+    if (g0 === 0x2002)
+        return carried(g1, g2);
+    if (g0 === 0x100 && zeros(1, 4))
+        return true;
+    if (g0 === 0x2001 && g1 === 0xdb8)
+        return true;
+    return (g0 & 0xfe00) === 0xfc00 || (g0 & 0xffc0) === 0xfe80 || (g0 & 0xffc0) === 0xfec0 || (g0 & 0xff00) === 0xff00;
+}
+/**
+ * Whether a URL's host is this machine or off the public internet: `localhost`
+ * names, and the IPv4 and IPv6 literals above. The URL parser has already
+ * written other IPv4 spellings (`0x7f000001`, `127.1`) as dotted numbers. An
+ * IPv6 literal that cannot be read is treated as not public. The check reads
+ * the host as written: the address a name resolves to, and the target of a
+ * redirect, are not checked here.
+ */
+export function isPrivateHost(hostname) {
+    const host = hostname.toLowerCase().replace(/^\[|\]$/g, '').replace(/\.$/, '');
+    if (host === 'localhost' || host.endsWith('.localhost'))
+        return true;
+    const v4 = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(host);
+    if (v4)
+        return isNonPublicIPv4(v4.slice(1).map(Number));
+    if (!host.includes(':'))
+        return false;
+    const groups = ipv6Groups(host);
+    return groups === undefined || isNonPublicIPv6(groups);
+}
+/** The value of an ASCII hex digit, or -1. */
+function hexValue(byte) {
+    if (byte >= 0x30 && byte <= 0x39)
+        return byte - 0x30;
+    if (byte >= 0x41 && byte <= 0x46)
+        return byte - 0x37;
+    if (byte >= 0x61 && byte <= 0x66)
+        return byte - 0x57;
+    return -1;
+}
+/**
+ * The bytes of a percent-encoded data URL payload: each `%XX` is one byte,
+ * any other character its UTF-8 bytes. A `%` that starts no valid escape is
+ * kept as it is, so the decoding never throws. One pass over two buffers.
+ */
+function percentDecode(payload) {
+    const text = Buffer.from(payload, 'utf8');
+    const out = Buffer.allocUnsafe(text.length);
+    let length = 0;
+    for (let i = 0; i < text.length; i += 1) {
+        const high = text[i] === 0x25 && i + 2 < text.length ? hexValue(text[i + 1]) : -1;
+        const low = high >= 0 ? hexValue(text[i + 2]) : -1;
+        if (low >= 0) {
+            out[length] = high * 16 + low;
+            i += 2;
+        }
+        else {
+            out[length] = text[i];
+        }
+        length += 1;
+    }
+    return out.subarray(0, length);
+}
+/**
+ * The image as the pipeline takes it: a data URL decoded to its bytes, an
+ * http(s) URL whose host is not this machine or a private network as is, or
+ * `undefined` for anything else. AgentOS also reads local file paths and
+ * fetches any URL, but a tool the model calls must not send the machine's
+ * files, or what its network serves, to a cloud vision model.
+ */
+export function imageInput(value) {
+    if (typeof value !== 'string')
+        return undefined;
+    const source = value.trim();
+    if (/^https?:\/\//i.test(source)) {
+        let url;
+        try {
+            url = new URL(source);
+        }
+        catch {
+            return undefined;
+        }
+        return isPrivateHost(url.hostname) ? undefined : source;
+    }
+    const data = /^data:image\/[^,]*?(;base64)?,(.*)$/is.exec(source);
+    if (!data)
+        return undefined;
+    return data[1] ? Buffer.from(data[2], 'base64') : percentDecode(data[2]);
+}
+/**
+ * The image as the tool hands it to the pipeline. A Buffer goes on as it is. An
+ * http(s) URL is fetched here through AgentOS's `imageToBuffer` with
+ * `untrusted: true`, which connects only to public network addresses (every
+ * address the host resolves to, and every redirect, is checked) and stops at
+ * 50 MiB and 30 seconds; the bytes go on. With an AgentOS without that mode
+ * (it has no `isPublicNetworkAddress` export) the pipeline gets the URL,
+ * already checked against the host as written.
+ */
+export async function loadImage(image) {
+    const { imageToBuffer, isPublicNetworkAddress } = agentos;
+    if (typeof image !== 'string' || typeof isPublicNetworkAddress !== 'function')
+        return image;
+    return imageToBuffer(image, { untrusted: true });
+}
+/** The vision-pipeline tool, on an AgentOS `VisionPipeline` per strategy. */
+export class VisionPipelineTool {
+    pipelineFor;
+    id = 'tool.vision-pipeline';
+    name = 'vision-pipeline';
+    displayName = 'Vision & OCR Pipeline';
+    description = 'Read and understand an image: printed text (ocr), handwriting, document layout, a description of the image, ' +
+        'or a CLIP embedding vector. Local OCR runs first; local vision models and cloud vision run as maxTier allows.';
+    category = 'media';
+    hasSideEffects = false;
+    inputSchema = {
+        type: 'object',
+        properties: {
+            imageUrl: { type: 'string', description: 'The image: an http(s) URL or a data:image URL.' },
+            mode: {
+                type: 'string',
+                enum: ['ocr', 'handwriting', 'layout', 'describe', 'embed', 'auto'],
+                description: 'ocr for printed text, handwriting, layout for document structure, describe for what the image shows, ' +
+                    'embed for a CLIP vector, or auto (default) to let the pipeline decide.',
+            },
+            maxTier: {
+                type: 'number',
+                enum: [1, 2, 3],
+                description: 'The highest tier to use: 1 local OCR only, 2 local vision models, 3 cloud vision (default).',
+            },
+        },
+        required: ['imageUrl'],
+    };
+    constructor(pipelineFor) {
+        this.pipelineFor = pipelineFor;
+    }
+    async execute(args, _context) {
+        const image = imageInput(args.imageUrl);
+        if (image === undefined) {
+            return {
+                success: false,
+                error: 'imageUrl must be a data:image URL or an http(s) URL on a public host: local file paths are not read, nor local or private network addresses.',
+            };
+        }
+        const mode = args.mode ?? 'auto';
+        if (typeof mode !== 'string' || !Object.hasOwn(TIER_NEEDED, mode)) {
+            return { success: false, error: `mode must be one of ${Object.keys(TIER_NEEDED).join(', ')}.` };
+        }
+        const maxTier = args.maxTier === 1 || args.maxTier === 2 ? args.maxTier : 3;
+        if (TIER_NEEDED[mode] > maxTier) {
+            return { success: false, error: `${mode} needs maxTier ${TIER_NEEDED[mode]} or higher; this call allows ${maxTier}.` };
+        }
+        try {
+            const input = await loadImage(image);
+            const pipeline = await this.pipelineFor(maxTier === 3 ? 'progressive' : 'local-only');
+            if (mode === 'embed') {
+                const embedding = await pipeline.embed(input);
+                return { success: true, output: { mode, embedding, dimensions: embedding.length } };
+            }
+            const options = maxTier === 1 || mode === 'ocr'
+                ? { tiers: ['ocr'] }
+                : mode === 'handwriting'
+                    ? { forceCategory: 'handwritten' }
+                    : mode === 'layout'
+                        ? { forceCategory: 'document-layout' }
+                        : mode === 'describe'
+                            ? { tiers: ['cloud-vision'] }
+                            : undefined;
+            const result = await pipeline.process(input, options);
+            return {
+                success: true,
+                output: {
+                    mode,
+                    text: result.text,
+                    confidence: result.confidence,
+                    category: result.category,
+                    tiers: result.tiers,
+                    regions: result.regions?.length ?? 0,
+                    durationMs: result.durationMs,
+                },
+            };
+        }
+        catch (error) {
+            return { success: false, error: error instanceof Error ? error.message : String(error) };
+        }
+    }
+}
+//# sourceMappingURL=visionPipeline.js.map

package/dist/tools/visionPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"visionPipeline.js","sourceRoot":"","sources":["../../src/tools/visionPipeline.ts"],"names":[],"mappings":"AAAA,cAAc;AACd;;;GAGG;AAEH,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAC;AAyC5C;;;GAGG;AACH,MAAM,WAAW,GAAkC;IACjD,IAAI,EAAE,CAAC;IACP,GAAG,EAAE,CAAC;IACN,WAAW,EAAE,CAAC;IACd,MAAM,EAAE,CAAC;IACT,KAAK,EAAE,CAAC;IACR,QAAQ,EAAE,CAAC;CACZ,CAAC;AAEF;;;;;GAKG;AACH,SAAS,eAAe,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAW;IAC1C,OAAO,CACL,CAAC,KAAK,CAAC;QACP,CAAC,KAAK,EAAE;QACR,CAAC,KAAK,GAAG;QACT,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC;QAClC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QACxB,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;QACjC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9C,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC;QACxB,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC;QACrC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC;QACpC,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC;QACnC,CAAC,IAAI,GAAG,CACT,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,SAAS,UAAU,CAAC,OAAe;IACjC,IAAI,IAAI,GAAG,OAAO,CAAC;IACnB,yDAAyD;IACzD,MAAM,MAAM,GAAG,mDAAmD,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9E,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC;YAAE,OAAO,SAAS,CAAC;QACxD,IAAI,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1E,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACrE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9E,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;AACpD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,MAAgB;IACvC,MAAM,OAAO,GAAG,CAAC,IAAY,EAAE,GAAW,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,IAAI,IAAI,CAAC,EAAE,IAAI,GAAG,GAAG,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC;IAC7G,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC;IAChD,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,EAAU,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;IACjG,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,CAAC,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACjE,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,MAAM;QAAE,OAAO,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACzD,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,MAAM;QAAE,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IACrF,IAAI,EAAE,KAAK,MAAM;QAAE,OAAO,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1C,IAAI,EAAE,KAAK,KAAK,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAC7C,IAAI,EAAE,KAAK,MAAM,IAAI,EAAE,KAAK,KAAK;QAAE,OAAO,IAAI,CAAC;IAC/C,OAAO,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,MAAM,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,KAAK,MAAM,CAAC;AACtH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB;IAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAC/E,IAAI,IAAI,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAC;IACrE,MAAM,EAAE,GAAG,8CAA8C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrE,IAAI,EAAE;QAAE,OAAO,eAAe,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAChC,OAAO,MAAM,KAAK,SAAS,IAAI,eAAe,CAAC,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,8CAA8C;AAC9C,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,IAAI,GAAG,IAAI,CAAC;IACrD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,IAAI,GAAG,IAAI,CAAC;IACrD,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI;QAAE,OAAO,IAAI,GAAG,IAAI,CAAC;IACrD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,OAAe;IACpC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClF,MAAM,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;YACb,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC;YAC9B,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC;QACD,MAAM,IAAI,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,KAAc;IACvC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAChD,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC5B,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QACjC,IAAI,GAAQ,CAAC;QACb,IAAI,CAAC;YACH,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1D,CAAC;IACD,MAAM,IAAI,GAAG,uCAAuC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAClE,IAAI,CAAC,IAAI;QAAE,OAAO,SAAS,CAAC;IAC5B,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,KAAsB;IACpD,MAAM,EAAE,aAAa,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC;IAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,sBAAsB,KAAK,UAAU;QAAE,OAAO,KAAK,CAAC;IAC5F,OAAO,aAAa,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;AACnD,CAAC;AAED,6EAA6E;AAC7E,MAAM,OAAO,kBAAkB;IA8BA;IA7BpB,EAAE,GAAG,sBAAsB,CAAC;IAC5B,IAAI,GAAG,iBAAiB,CAAC;IACzB,WAAW,GAAG,uBAAuB,CAAC;IACtC,WAAW,GAClB,8GAA8G;QAC9G,+GAA+G,CAAC;IACzG,QAAQ,GAAG,OAAO,CAAC;IACnB,cAAc,GAAG,KAAK,CAAC;IAEvB,WAAW,GAAqB;QACvC,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gDAAgD,EAAE;YAC3F,IAAI,EAAE;gBACJ,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC;gBACnE,WAAW,EACT,uGAAuG;oBACvG,wEAAwE;aAC3E;YACD,OAAO,EAAE;gBACP,IAAI,EAAE,QAAQ;gBACd,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;gBACf,WAAW,EAAE,6FAA6F;aAC3G;SACF;QACD,QAAQ,EAAE,CAAC,UAAU,CAAC;KACvB,CAAC;IAEF,YAA6B,WAAwB;QAAxB,gBAAW,GAAX,WAAW,CAAa;IAAG,CAAC;IAEzD,KAAK,CAAC,OAAO,CAAC,IAAyB,EAAE,QAA+B;QACtE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACxC,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO;gBACL,OAAO,EAAE,KAAK;gBACd,KAAK,EAAE,8IAA8I;aACtJ,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,GAAe,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC;QAC7C,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,WAAW,EAAE,IAAI,CAAC,EAAE,CAAC;YAClE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uBAAuB,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;QAClG,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAI,WAAW,CAAC,IAAI,CAAC,GAAG,OAAO,EAAE,CAAC;YAChC,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,IAAI,kBAAkB,WAAW,CAAC,IAAI,CAAC,gCAAgC,OAAO,GAAG,EAAE,CAAC;QACzH,CAAC;QAED,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;YACtF,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC9C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YACtF,CAAC;YACD,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,IAAI,IAAI,KAAK,KAAK;gBAC7B,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE;gBACpB,CAAC,CAAC,IAAI,KAAK,aAAa;oBACtB,CAAC,CAAC,EAAE,aAAa,EAAE,aAAa,EAAE;oBAClC,CAAC,CAAC,IAAI,KAAK,QAAQ;wBACjB,CAAC,CAAC,EAAE,aAAa,EAAE,iBAAiB,EAAE;wBACtC,CAAC,CAAC,IAAI,KAAK,UAAU;4BACnB,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,cAAc,CAAC,EAAE;4BAC7B,CAAC,CAAC,SAAS,CAAC;YACtB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACtD,OAAO;gBACL,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE;oBACN,IAAI;oBACJ,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,UAAU,EAAE,MAAM,CAAC,UAAU;oBAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,MAAM,IAAI,CAAC;oBACpC,UAAU,EAAE,MAAM,CAAC,UAAU;iBAC9B;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,CAAC;IACH,CAAC;CACF"}

package/manifest.json

@@ -2,16 +2,16 @@
   "$schema": "https://agentos.sh/schemas/extension-manifest-v1.json",
   "id": "com.framers.tools.vision-pipeline",
   "name": "Vision & OCR Pipeline",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "Progressive vision pipeline with PaddleOCR, TrOCR, Florence-2, CLIP, and cloud vision (GPT-4o, Claude).",
   "author": {
     "name": "Frame.dev",
     "email": "support@frame.dev",
-    "url": "https://github.com/framersai"
+    "url": "https://github.com/framerslab"
   },
   "license": "MIT",
   "keywords": ["vision", "ocr", "paddle", "trocr", "florence", "clip", "document-ai"],
-  "agentosVersion": "^2.0.0",
+  "agentosVersion": ">=0.10.40",
   "categories": ["tool", "media"],
   "platforms": [],
   "sdk": "none",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@framers/agentos-ext-vision-pipeline",
-  "version": "0.1.0",
+  "version": "0.2.1",
   "description": "Vision & OCR pipeline extension for AgentOS — PaddleOCR, TrOCR, Florence-2, CLIP, and cloud vision",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -38,26 +38,35 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/framersai/agentos-extensions.git",
+    "url": "git+https://github.com/framerslab/agentos-extensions.git",
     "directory": "registry/curated/tools/vision-pipeline"
   },
   "publishConfig": {
     "access": "public"
   },
   "peerDependencies": {
-    "@framers/agentos": "^0.5.2"
+    "@framers/agentos": ">=0.10.40"
   },
   "devDependencies": {
-    "@framers/agentos": "^0.5.2",
+    "@framers/agentos": "^0.12.8",
     "@types/node": "^20.12.12",
     "rimraf": "^5.0.7",
     "typescript": "^5.4.5",
-    "vitest": "^1.6.0"
+    "vitest": "^3.2.7"
   },
+  "homepage": "https://github.com/framerslab/agentos-extensions/tree/master/registry/curated/tools/vision-pipeline#readme",
+  "bugs": {
+    "url": "https://github.com/framerslab/agentos-extensions/issues"
+  },
+  "files": [
+    "dist",
+    "manifest.json",
+    "SKILL.md"
+  ],
   "scripts": {
     "build": "tsc",
     "dev": "tsc --watch",
-    "test": "vitest run",
+    "test": "vitest run --passWithNoTests",
     "test:watch": "vitest",
     "clean": "rimraf dist"
   }

package/src/index.ts

@@ -1,116 +0,0 @@
-// @ts-nocheck
-/**
- * @fileoverview Vision & OCR Pipeline Extension Pack — progressive multi-tier
- * vision pipeline as an agent tool.
- *
- * This is a minimal stub pack. The actual vision pipeline implementation is
- * provided by the core AgentOS vision module; this extension pack registers
- * the tool descriptor so the agent can discover and invoke it.
- *
- * Pipeline tiers:
- *   Tier 1 (local, fast):   PaddleOCR for printed text
- *   Tier 2 (local, medium): TrOCR for handwriting, Florence-2 for layout
- *   Tier 3 (cloud, slow):   GPT-4o / Claude vision for complex understanding
- *
- * CLIP embeddings are generated alongside any tier for semantic image search.
- *
- * @module @framers/agentos-ext-vision-pipeline
- */
-
-/* ------------------------------------------------------------------ */
-/*  Extension pack types                                               */
-/* ------------------------------------------------------------------ */
-
-export interface ExtensionContext {
-  options?: Record<string, unknown>;
-  secrets?: Record<string, string>;
-  getSecret?: (key: string) => string | undefined;
-  logger?: { info: (msg: string) => void };
-}
-
-export interface ExtensionPack {
-  name: string;
-  version: string;
-  descriptors: Array<{
-    id: string;
-    kind: string;
-    priority?: number;
-    payload: unknown;
-    requiredSecrets?: Array<{ id: string; optional?: boolean }>;
-  }>;
-  onActivate?: () => Promise<void>;
-  onDeactivate?: () => Promise<void>;
-}
-
-/* ------------------------------------------------------------------ */
-/*  Stub tool definition                                               */
-/* ------------------------------------------------------------------ */
-
-/**
- * Stub tool for the vision & OCR pipeline.
- * Implementation is resolved at runtime by the AgentOS vision module.
- */
-const visionPipelineTool = {
-  name: 'vision-pipeline',
-  description:
-    'Extract text, understand layout, and generate embeddings from images using a progressive pipeline: PaddleOCR -> TrOCR/Florence-2 -> GPT-4o/Claude vision.',
-  parameters: {
-    type: 'object' as const,
-    properties: {
-      imageUrl: {
-        type: 'string',
-        description: 'URL or local path of the image to analyze',
-      },
-      mode: {
-        type: 'string',
-        enum: ['ocr', 'handwriting', 'layout', 'describe', 'embed', 'auto'],
-        description:
-          'Analysis mode — "ocr" for printed text, "handwriting" for handwritten text, "layout" for document structure, "describe" for general image understanding, "embed" for CLIP vector, "auto" to let the pipeline decide (default: auto)',
-      },
-      maxTier: {
-        type: 'number',
-        enum: [1, 2, 3],
-        description:
-          'Maximum pipeline tier to use — 1 (local OCR only), 2 (local vision models), 3 (cloud vision). Default: 3',
-      },
-      language: {
-        type: 'string',
-        description: 'Language hint for OCR (e.g. "en", "zh", "ja"). Default: "en"',
-      },
-    },
-    required: ['imageUrl'],
-  },
-  execute: async () => {
-    throw new Error('vision-pipeline stub — wire to VisionPipelineService at runtime.');
-  },
-};
-
-/* ------------------------------------------------------------------ */
-/*  Factory                                                            */
-/* ------------------------------------------------------------------ */
-
-/**
- * Create the Vision & OCR Pipeline extension pack.
- *
- * Registers the vision-pipeline tool. Actual implementation is provided
- * by the core AgentOS vision module and wired at runtime.
- */
-export function createExtensionPack(context: ExtensionContext): ExtensionPack {
-  return {
-    name: '@framers/agentos-ext-vision-pipeline',
-    version: '0.1.0',
-    descriptors: [
-      {
-        id: visionPipelineTool.name,
-        kind: 'tool' as const,
-        priority: 45,
-        payload: visionPipelineTool,
-        requiredSecrets: [{ id: 'openai.apiKey', optional: true }],
-      },
-    ],
-    onActivate: async () => context.logger?.info('Vision & OCR Pipeline Extension activated'),
-    onDeactivate: async () => context.logger?.info('Vision & OCR Pipeline Extension deactivated'),
-  };
-}
-
-export default createExtensionPack;