@nodetool-ai/runtime 0.8.1 → 0.8.3

package/dist/providers/meta-provider.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"meta-provider.d.ts","sourceRoot":"","sources":["../../src/providers/meta-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,KAAK,2BAA2B,EACjC,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAgB,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAe9D;;;;;;GAMG;AACH,qBAAa,YAAa,SAAQ,oBAAoB;IACpD,OAAgB,eAAe,IAAI,MAAM,EAAE,CAE1C;IAED,YACE,OAAO,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,EAClC,OAAO,GAAE,2BAAgC,EAe1C;IAEQ,eAAe;QACb,YAAY;MACtB;IAEc,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAE9D;IAEc,0BAA0B,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC,CAGpE;IAED,OAAO,CAAC,sBAAsB;CAO/B"}
+{"version":3,"file":"meta-provider.d.ts","sourceRoot":"","sources":["../../src/providers/meta-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,KAAK,2BAA2B,EACjC,MAAM,6BAA6B,CAAC;AAErC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAahE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAEL,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,UAAU,EACf,KAAK,qBAAqB,EAC1B,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,iBAAiB,EACvB,MAAM,YAAY,CAAC;AA2EpB;;;;;;;;;;;GAWG;AACH,qBAAa,YAAa,SAAQ,oBAAoB;IACpD,OAAgB,eAAe,IAAI,MAAM,EAAE,CAE1C;IAED,YACE,OAAO,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAA;KAAE,EAClC,OAAO,GAAE,2BAAgC,EAe1C;IAEQ,eAAe;QACb,YAAY;MACtB;IAEc,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAE9D;IAED;;;OAGG;IACH,UAAmB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAE1D;IAED,IAAa,uBAAuB,IAAI,OAAO,CAE9C;IAED,cAAuB,wBAAwB,IAAI,OAAO,CAEzD;IAED,IAAa,kBAAkB,IAAI,OAAO,CAEzC;IAED,UAAmB,qBAAqB,IAAI,qBAAqB,CAEhE;IAED,UAAmB,oBAAoB,IAAI,SAAS,kBAAkB,EAAE,CAOvE;IAEc,0BAA0B,IAAI,OAAO,CAAC,aAAa,EAAE,CAAC,CAKpE;IAEc,uBAAuB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAe9D;IAEc,qBAAqB,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC,CAQ1D;IAEc,WAAW,CAAC,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,UAAU,CAAC,CAGzE;IAEc,YAAY,CACzB,MAAM,EAAE,iBAAiB,EACzB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,UAAU,EAAE,CAAC,CAmBvB;IAEc,YAAY,CACzB,MAAM,EAAE,UAAU,EAAE,EACpB,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,UAAU,CAAC,CAGrB;IAED;;;OAGG;IACY,aAAa,CAC1B,MAAM,EAAE,UAAU,EAAE,EACpB,MAAM,EAAE,kBAAkB,EAC1B,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,UAAU,EAAE,CAAC,CAmCvB;IAED;;;OAGG;IACY,0BAA0B,CAAC,IAAI,EAAE;QAC9C,KAAK,EAAE,UAAU,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,OAAO,CAAC;KAC3B,GAAG,OAAO,CAAC,SAAS,CAAC,CAmDrB;IAED;;;OAGG;IACY,YAAY,CACzB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,kBAAkB,GACzB,OAAO,CAAC,qBAAqB,EAAE,CAAC,CA4ClC;YAEa,aAAa;YAoBb,QAAQ;YAuBR,QAAQ;IAUtB,OAAO,CAAC,sBAAsB;CAO/B"}

package/dist/providers/meta-provider.js

@@ -1,22 +1,78 @@
 import { OpenAICompatProvider } from "./openai-compat-provider.js";
+import { extractResponsesText } from "./responses-api.js";
+import { bytesToImageDataUri } from "./image-mime.js";
+import { displayedImageSize, samMasksFromOutput, samPromptText } from "./meta-segmentation.js";
+import { decodeTranscribeResponse, keywordsFromPrompt, languageBiasName, toTranscribeWav } from "./meta-transcription.js";
 import { PROVIDER_IDS } from "./types.js";
+import { isRecord, isString } from "@nodetool-ai/protocol";
 const META_BASE_URL = "https://api.meta.ai/v1";
+/** Muse Image: text-to-image and image editing. */
+const MUSE_IMAGE_MODEL = "muse-image-1.0";
+/** Muse Voice Transcribe: speech to text over `/v1/asr/transcribe`. */
+const MUSE_TRANSCRIBE_MODEL = "muse-voice-transcribe-1.0";
+/** SAM 3.1: text- or object-prompted segmentation over the Responses API. */
+const SAM_MODEL = "sam-3.1";
+/** Images per request on `/v1/images/generations` and `/v1/images/edits`. */
+const MAX_IMAGES_PER_REQUEST = 10;
 /**
  * Models Meta serves when `/v1/models` cannot be reached (no network, key not
  * yet accepted). Keeps the model menu usable so a user can pick one and let
  * the call itself report the real error. See https://dev.meta.ai/docs/models.
  */
 const META_FALLBACK_MODELS = [
+    "muse-spark-1.3",
     "muse-spark-1.2",
     "muse-spark-1.2-contributor",
     "muse-spark-1.1"
 ];
 /**
- * Meta AI provider. Speaks the OpenAI Chat Completions dialect against Meta's
- * model API at https://api.meta.ai/v1, which serves the Muse Spark family:
- * multimodal input (text, image, video, PDF), text output, parallel tool
- * calling, and a 1M-token context window.
- * See https://dev.meta.ai/docs/overview.
+ * Whether `model` is a Muse Spark chat model rather than one of the media
+ * models the same `/v1/models` listing can carry.
+ */
+function isMetaChatModel(model) {
+    return !/^(muse-image|muse-voice|sam-)/i.test(model);
+}
+/**
+ * Meta's Images API reads `size` as an aspect ratio, not exact pixels. Pixel
+ * sizes pass through; a ratio becomes a size with a 1024-pixel short side.
+ */
+function imageSize(width, height, aspectRatio) {
+    if (width && height && width > 0 && height > 0) {
+        return `${Math.round(width)}x${Math.round(height)}`;
+    }
+    const match = aspectRatio ? /^(\d+(?:\.\d+)?):(\d+(?:\.\d+)?)$/.exec(aspectRatio.trim()) : null;
+    if (!match)
+        return undefined;
+    const w = Number(match[1]);
+    const h = Number(match[2]);
+    if (!(w > 0 && h > 0))
+        return undefined;
+    return w >= h
+        ? `${Math.round((1024 * w) / h)}x1024`
+        : `1024x${Math.round((1024 * h) / w)}`;
+}
+function promptWithNegative(prompt, negativePrompt) {
+    return negativePrompt?.trim()
+        ? `${prompt.trim()}\n\nDo not include: ${negativePrompt.trim()}`
+        : prompt;
+}
+function imageCount(n) {
+    if (!Number.isInteger(n) || n < 1 || n > MAX_IMAGES_PER_REQUEST) {
+        throw new Error(`Meta image requests return 1 to ${MAX_IMAGES_PER_REQUEST} images, got ${n}.`);
+    }
+    return n;
+}
+/**
+ * Meta AI provider for the Muse family at https://api.meta.ai/v1.
+ *
+ * Chat runs on Meta's Responses API, the only protocol that serves its hosted
+ * tools: a `web_search` tool is answered by Meta's own search grounding, and
+ * tools marked `deferLoading` are loaded on demand by its hosted
+ * `tool_search`. Muse Spark reads video and audio parts natively.
+ *
+ * Media runs on Meta's dedicated endpoints: Muse Image for text-to-image and
+ * editing, Muse Voice Transcribe for speech to text, and SAM 3.1 for image
+ * segmentation. See https://dev.meta.ai/docs/overview.
  */
 export class MetaProvider extends OpenAICompatProvider {
     static requiredSecrets() {
@@ -39,10 +95,237 @@ export class MetaProvider extends OpenAICompatProvider {
     async hasToolSupport(_model) {
         return true;
     }
+    /**
+     * Muse Spark chat goes through the Responses API. Search grounding and tool
+     * search exist only there, not on Meta's Chat Completions endpoint.
+     */
+    usesResponsesApi(model) {
+        return isMetaChatModel(model);
+    }
+    get supportsNativeWebSearch() {
+        return true;
+    }
+    get supportsNativeToolSearch() {
+        return true;
+    }
+    get supportsVideoInput() {
+        return true;
+    }
+    responsesInputOptions() {
+        return { video: true, audio: true };
+    }
+    declaredCapabilities() {
+        return [
+            "text_to_image",
+            "image_to_image",
+            "segment_image",
+            "automatic_speech_recognition"
+        ];
+    }
     async getAvailableLanguageModels() {
-        const models = await this.listCompatModels();
+        const models = (await this.listCompatModels()).filter((model) => isMetaChatModel(model.id));
         return models.length > 0 ? models : this.fallbackLanguageModels();
     }
+    async getAvailableImageModels() {
+        return [
+            {
+                id: MUSE_IMAGE_MODEL,
+                name: "Muse Image 1.0",
+                provider: PROVIDER_IDS.META,
+                supportedTasks: ["text_to_image", "image_to_image"]
+            },
+            {
+                id: SAM_MODEL,
+                name: "SAM 3.1",
+                provider: PROVIDER_IDS.META,
+                supportedTasks: ["segment"]
+            }
+        ];
+    }
+    async getAvailableASRModels() {
+        return [
+            {
+                id: MUSE_TRANSCRIBE_MODEL,
+                name: "Muse Voice Transcribe 1.0",
+                provider: PROVIDER_IDS.META
+            }
+        ];
+    }
+    async textToImage(params) {
+        const [image] = await this.textToImages(params, 1);
+        return image;
+    }
+    async textToImages(params, numImages) {
+        if (!params.prompt?.trim()) {
+            throw new Error("The input prompt cannot be empty.");
+        }
+        const request = {
+            model: params.model.id,
+            prompt: promptWithNegative(params.prompt, params.negativePrompt),
+            n: imageCount(numImages),
+            response_format: "b64_json",
+            output_format: "png"
+        };
+        const size = imageSize(params.width, params.height, params.aspectRatio);
+        if (size)
+            request.size = size;
+        return this.requestImages("images/generations", request, params.model.id, params.signal);
+    }
+    async imageToImage(images, params) {
+        const [image] = await this.imageToImages(images, params, 1);
+        return image;
+    }
+    /**
+     * Edit or compose 1–10 source images. The edits endpoint's JSON form takes
+     * each image as a data URL; it rejects masks.
+     */
+    async imageToImages(images, params, numImages) {
+        const sources = images.filter((image) => image && image.length > 0);
+        if (sources.length === 0) {
+            throw new Error("image must not be empty.");
+        }
+        if (sources.length > MAX_IMAGES_PER_REQUEST) {
+            throw new Error(`Meta image edits accept at most ${MAX_IMAGES_PER_REQUEST} source images, got ${sources.length}.`);
+        }
+        if (!params.prompt?.trim()) {
+            throw new Error("The input prompt cannot be empty.");
+        }
+        const request = {
+            model: params.model.id,
+            prompt: promptWithNegative(params.prompt, params.negativePrompt),
+            images: sources.map((image) => ({
+                image_url: bytesToImageDataUri(image)
+            })),
+            n: imageCount(numImages),
+            response_format: "b64_json",
+            output_format: "png"
+        };
+        const size = imageSize(params.targetWidth, params.targetHeight, params.aspectRatio);
+        if (size)
+            request.size = size;
+        return this.requestImages("images/edits", request, params.model.id, params.signal);
+    }
+    /**
+     * Transcribe a recording with Muse Voice Transcribe. ENDPOINTING mode splits
+     * the transcript into turns, which come back as timed chunks.
+     */
+    async automaticSpeechRecognition(args) {
+        if (!args.audio || args.audio.length === 0) {
+            throw new Error("audio must not be empty");
+        }
+        if (args.word_timestamps) {
+            throw new Error("Meta transcription has no word-level timestamps; it returns turn timings.");
+        }
+        const wav = await toTranscribeWav(args.audio);
+        const settings = {
+            mode: "ENDPOINTING",
+            model: args.model,
+            audioEncoding: "WAV"
+        };
+        if (args.language?.trim()) {
+            settings.languageBias = [languageBiasName(args.language)];
+        }
+        const keywords = args.prompt ? keywordsFromPrompt(args.prompt) : [];
+        if (keywords.length > 0)
+            settings.keywords = keywords;
+        const form = new FormData();
+        form.append("request", new Blob([JSON.stringify(settings)], { type: "application/json" }));
+        form.append("audio", new Blob([Buffer.from(wav)], { type: "audio/wav" }), "audio.wav");
+        this.recordRequestPayload(settings);
+        const response = await this.compatFetch(`${META_BASE_URL}/asr/transcribe`, {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${this.apiKey}`,
+                Accept: "application/json"
+            },
+            body: form
+        });
+        const payload = await this.readJson(response, "transcription");
+        const { result, audioDurationMs } = decodeTranscribeResponse(payload);
+        if (audioDurationMs !== null) {
+            // Billed per whole second of audio, rounded down.
+            this.trackUsage(args.model, {
+                durationSeconds: Math.floor(audioDurationMs / 1000)
+            });
+        }
+        return result;
+    }
+    /**
+     * Segment an image with SAM 3.1. A text prompt finds every instance of a
+     * concept; a box or points pick out one object.
+     */
+    async segmentImage(image, params) {
+        if (image.length === 0) {
+            throw new Error("segmentImage requires a non-empty image");
+        }
+        const size = await displayedImageSize(image);
+        const promptText = samPromptText(params, size);
+        const request = {
+            model: params.model.id,
+            input: [
+                {
+                    type: "message",
+                    role: "user",
+                    content: [
+                        { type: "input_text", text: promptText },
+                        { type: "input_image", image_url: bytesToImageDataUri(image) }
+                    ]
+                }
+            ],
+            stream: false
+        };
+        const payload = await this.postJson("responses", request, "segmentation", params.signal);
+        const outputText = isRecord(payload) && isString(payload.output_text)
+            ? payload.output_text
+            : extractResponsesText(isRecord(payload) ? payload.output : undefined);
+        let masks = await samMasksFromOutput(outputText, size, {
+            label: params.prompt?.trim() || null
+        });
+        const minConfidence = params.minConfidence;
+        if (minConfidence != null) {
+            masks = masks.filter((mask) => mask.confidence == null || mask.confidence >= minConfidence);
+        }
+        if (params.maxMasks != null && params.maxMasks >= 0) {
+            masks = masks.slice(0, params.maxMasks);
+        }
+        return masks;
+    }
+    async requestImages(endpoint, request, model, signal) {
+        const payload = await this.postJson(endpoint, request, "image", signal);
+        const data = isRecord(payload) ? payload.data : undefined;
+        const images = (Array.isArray(data) ? data : [])
+            .filter(isRecord)
+            .map((item) => item.b64_json)
+            .filter(isString)
+            .map((b64) => new Uint8Array(Buffer.from(b64, "base64")));
+        if (images.length === 0) {
+            throw new Error("Meta image generation returned no image data.");
+        }
+        this.trackUsage(model, { imageCount: images.length });
+        return images;
+    }
+    async postJson(endpoint, body, what, signal) {
+        this.recordRequestPayload(body);
+        const init = {
+            method: "POST",
+            headers: {
+                Authorization: `Bearer ${this.apiKey}`,
+                "Content-Type": "application/json"
+            },
+            body: JSON.stringify(body)
+        };
+        if (signal)
+            init.signal = signal;
+        const response = await this.compatFetch(`${META_BASE_URL}/${endpoint}`, init);
+        return this.readJson(response, what);
+    }
+    async readJson(response, what) {
+        if (!response.ok) {
+            const detail = await response.text().catch(() => "");
+            throw new Error(`Meta ${what} request failed: ${response.status}${detail ? ` ${detail}` : ""}`);
+        }
+        return response.json();
+    }
     fallbackLanguageModels() {
         return META_FALLBACK_MODELS.map((id) => ({
             id,

package/dist/providers/meta-provider.js.map

@@ -1 +1 @@
-{"version":3,"file":"meta-provider.js","sourceRoot":"","sources":["../../src/providers/meta-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EAErB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,YAAY,EAAsB,MAAM,YAAY,CAAC;AAE9D,MAAM,aAAa,GAAG,wBAAwB,CAAC;AAE/C;;;;GAIG;AACH,MAAM,oBAAoB,GAAG;IAC3B,gBAAgB;IAChB,4BAA4B;IAC5B,gBAAgB;CACR,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,OAAO,YAAa,SAAQ,oBAAoB;IACpD,MAAM,CAAU,eAAe;QAC7B,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1B,CAAC;IAED,YACE,OAAkC,EAClC,OAAO,GAAgC,EAAE;QAEzC,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;QACpC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,KAAK,CACH;YACE,UAAU,EAAE,YAAY,CAAC,IAAI;YAC7B,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;YACrB,OAAO,EAAE,aAAa;SACvB,EACD,OAAO,CACR,CAAC;IACJ,CAAC;IAEQ,eAAe;QACtB,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IACvC,CAAC;IAEQ,KAAK,CAAC,cAAc,CAAC,MAAc;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAEQ,KAAK,CAAC,0BAA0B;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC7C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpE,CAAC;IAEO,sBAAsB;QAC5B,OAAO,oBAAoB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACvC,EAAE;YACF,IAAI,EAAE,EAAE;YACR,QAAQ,EAAE,YAAY,CAAC,IAAI;SAC5B,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
+{"version":3,"file":"meta-provider.js","sourceRoot":"","sources":["../../src/providers/meta-provider.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EAErB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAE1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EACL,kBAAkB,EAClB,kBAAkB,EAClB,aAAa,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,wBAAwB,EACxB,kBAAkB,EAClB,gBAAgB,EAChB,eAAe,EAChB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EACL,YAAY,EASb,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,aAAa,GAAG,wBAAwB,CAAC;AAE/C,mDAAmD;AACnD,MAAM,gBAAgB,GAAG,gBAAgB,CAAC;AAC1C,uEAAuE;AACvE,MAAM,qBAAqB,GAAG,2BAA2B,CAAC;AAC1D,6EAA6E;AAC7E,MAAM,SAAS,GAAG,SAAS,CAAC;AAE5B,6EAA6E;AAC7E,MAAM,sBAAsB,GAAG,EAAE,CAAC;AAElC;;;;GAIG;AACH,MAAM,oBAAoB,GAAG;IAC3B,gBAAgB;IAChB,gBAAgB;IAChB,4BAA4B;IAC5B,gBAAgB;CACR,CAAC;AAEX;;;GAGG;AACH,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,CAAC,gCAAgC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACvD,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAChB,KAAgC,EAChC,MAAiC,EACjC,WAAsC;IAEtC,IAAI,KAAK,IAAI,MAAM,IAAI,KAAK,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/C,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;IACtD,CAAC;IACD,MAAM,KAAK,GAAG,WAAW,CAAC,CAAC,CAAC,mCAAmC,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAChG,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,OAAO,CAAC,IAAI,CAAC;QACX,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO;QACtC,CAAC,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAc,EACd,cAAyC;IAEzC,OAAO,cAAc,EAAE,IAAI,EAAE;QAC3B,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,uBAAuB,cAAc,CAAC,IAAI,EAAE,EAAE;QAChE,CAAC,CAAC,MAAM,CAAC;AACb,CAAC;AAED,SAAS,UAAU,CAAC,CAAS;IAC3B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,sBAAsB,EAAE,CAAC;QAChE,MAAM,IAAI,KAAK,CACb,mCAAmC,sBAAsB,gBAAgB,CAAC,GAAG,CAC9E,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,YAAa,SAAQ,oBAAoB;IACpD,MAAM,CAAU,eAAe;QAC7B,OAAO,CAAC,cAAc,CAAC,CAAC;IAC1B,CAAC;IAED,YACE,OAAkC,EAClC,OAAO,GAAgC,EAAE;QAEzC,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;QACpC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QAED,KAAK,CACH;YACE,UAAU,EAAE,YAAY,CAAC,IAAI;YAC7B,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;YACrB,OAAO,EAAE,aAAa;SACvB,EACD,OAAO,CACR,CAAC;IACJ,CAAC;IAEQ,eAAe;QACtB,OAAO,EAAE,YAAY,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IACvC,CAAC;IAEQ,KAAK,CAAC,cAAc,CAAC,MAAc;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;OAGG;IACgB,gBAAgB,CAAC,KAAa;QAC/C,OAAO,eAAe,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAa,uBAAuB;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAuB,wBAAwB;QAC7C,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAa,kBAAkB;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAEkB,qBAAqB;QACtC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACtC,CAAC;IAEkB,oBAAoB;QACrC,OAAO;YACL,eAAe;YACf,gBAAgB;YAChB,eAAe;YACf,8BAA8B;SAC/B,CAAC;IACJ,CAAC;IAEQ,KAAK,CAAC,0BAA0B;QACvC,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAC9D,eAAe,CAAC,KAAK,CAAC,EAAE,CAAC,CAC1B,CAAC;QACF,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACpE,CAAC;IAEQ,KAAK,CAAC,uBAAuB;QACpC,OAAO;YACL;gBACE,EAAE,EAAE,gBAAgB;gBACpB,IAAI,EAAE,gBAAgB;gBACtB,QAAQ,EAAE,YAAY,CAAC,IAAI;gBAC3B,cAAc,EAAE,CAAC,eAAe,EAAE,gBAAgB,CAAC;aACpD;YACD;gBACE,EAAE,EAAE,SAAS;gBACb,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,YAAY,CAAC,IAAI;gBAC3B,cAAc,EAAE,CAAC,SAAS,CAAC;aAC5B;SACF,CAAC;IACJ,CAAC;IAEQ,KAAK,CAAC,qBAAqB;QAClC,OAAO;YACL;gBACE,EAAE,EAAE,qBAAqB;gBACzB,IAAI,EAAE,2BAA2B;gBACjC,QAAQ,EAAE,YAAY,CAAC,IAAI;aAC5B;SACF,CAAC;IACJ,CAAC;IAEQ,KAAK,CAAC,WAAW,CAAC,MAAyB;QAClD,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC;IAEQ,KAAK,CAAC,YAAY,CACzB,MAAyB,EACzB,SAAiB;QAEjB,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,OAAO,GAA4B;YACvC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE;YACtB,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;YAChE,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC;YACxB,eAAe,EAAE,UAAU;YAC3B,aAAa,EAAE,KAAK;SACrB,CAAC;QACF,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC;QACxE,IAAI,IAAI;YAAE,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;QAC9B,OAAO,IAAI,CAAC,aAAa,CACvB,oBAAoB,EACpB,OAAO,EACP,MAAM,CAAC,KAAK,CAAC,EAAE,EACf,MAAM,CAAC,MAAM,CACd,CAAC;IACJ,CAAC;IAEQ,KAAK,CAAC,YAAY,CACzB,MAAoB,EACpB,MAA0B;QAE1B,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5D,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,aAAa,CAC1B,MAAoB,EACpB,MAA0B,EAC1B,SAAiB;QAEjB,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CACb,mCAAmC,sBAAsB,uBAAuB,OAAO,CAAC,MAAM,GAAG,CAClG,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,MAAM,OAAO,GAA4B;YACvC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE;YACtB,MAAM,EAAE,kBAAkB,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,cAAc,CAAC;YAChE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC9B,SAAS,EAAE,mBAAmB,CAAC,KAAK,CAAC;aACtC,CAAC,CAAC;YACH,CAAC,EAAE,UAAU,CAAC,SAAS,CAAC;YACxB,eAAe,EAAE,UAAU;YAC3B,aAAa,EAAE,KAAK;SACrB,CAAC;QACF,MAAM,IAAI,GAAG,SAAS,CACpB,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,WAAW,CACnB,CAAC;QACF,IAAI,IAAI;YAAE,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;QAC9B,OAAO,IAAI,CAAC,aAAa,CACvB,cAAc,EACd,OAAO,EACP,MAAM,CAAC,KAAK,CAAC,EAAE,EACf,MAAM,CAAC,MAAM,CACd,CAAC;IACJ,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,0BAA0B,CAAC,IAOzC;QACC,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;QAC7C,CAAC;QACD,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CACb,2EAA2E,CAC5E,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE9C,MAAM,QAAQ,GAA4B;YACxC,IAAI,EAAE,aAAa;YACnB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,aAAa,EAAE,KAAK;SACrB,CAAC;QACF,IAAI,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YAC1B,QAAQ,CAAC,YAAY,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAEtD,MAAM,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC5B,IAAI,CAAC,MAAM,CACT,SAAS,EACT,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,CACnE,CAAC;QACF,IAAI,CAAC,MAAM,CACT,OAAO,EACP,IAAI,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EACnD,WAAW,CACZ,CAAC;QAEF,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,aAAa,iBAAiB,EAAE;YACzE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;gBACtC,MAAM,EAAE,kBAAkB;aAC3B;YACD,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;QAC/D,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;QACtE,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;YAC7B,kDAAkD;YAClD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,EAAE;gBAC1B,eAAe,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;aACpD,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACM,KAAK,CAAC,YAAY,CACzB,KAAiB,EACjB,MAA0B;QAE1B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE;YACtB,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,SAAS;oBACf,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE;wBACP,EAAE,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,UAAU,EAAE;wBACxC,EAAE,IAAI,EAAE,aAAa,EAAE,SAAS,EAAE,mBAAmB,CAAC,KAAK,CAAC,EAAE;qBAC/D;iBACF;aACF;YACD,MAAM,EAAE,KAAK;SACd,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CACjC,WAAW,EACX,OAAO,EACP,cAAc,EACd,MAAM,CAAC,MAAM,CACd,CAAC;QACF,MAAM,UAAU,GACd,QAAQ,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC;YAChD,CAAC,CAAC,OAAO,CAAC,WAAW;YACrB,CAAC,CAAC,oBAAoB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAE3E,IAAI,KAAK,GAAG,MAAM,kBAAkB,CAAC,UAAU,EAAE,IAAI,EAAE;YACrD,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,IAAI;SACrC,CAAC,CAAC;QACH,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC3C,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,KAAK,GAAG,KAAK,CAAC,MAAM,CAClB,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,IAAI,IAAI,IAAI,IAAI,CAAC,UAAU,IAAI,aAAa,CACtE,CAAC;QACJ,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,IAAI,IAAI,IAAI,MAAM,CAAC,QAAQ,IAAI,CAAC,EAAE,CAAC;YACpD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,QAA+C,EAC/C,OAAgC,EAChC,KAAa,EACb,MAAoB;QAEpB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACxE,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1D,MAAM,MAAM,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7C,MAAM,CAAC,QAAQ,CAAC;aAChB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;aAC5B,MAAM,CAAC,QAAQ,CAAC;aAChB,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QACnE,CAAC;QACD,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACtD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,QAAgB,EAChB,IAA6B,EAC7B,IAAY,EACZ,MAAoB;QAEpB,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;QAChC,MAAM,IAAI,GAAgB;YACxB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;gBACtC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC;QACF,IAAI,MAAM;YAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACjC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CACrC,GAAG,aAAa,IAAI,QAAQ,EAAE,EAC9B,IAAI,CACL,CAAC;QACF,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAkB,EAAE,IAAY;QACrD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YACrD,MAAM,IAAI,KAAK,CACb,QAAQ,IAAI,oBAAoB,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/E,CAAC;QACJ,CAAC;QACD,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,sBAAsB;QAC5B,OAAO,oBAAoB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;YACvC,EAAE;YACF,IAAI,EAAE,EAAE;YACR,QAAQ,EAAE,YAAY,CAAC,IAAI;SAC5B,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}

package/dist/providers/meta-segmentation.d.ts

@@ -0,0 +1,27 @@
+import type { ImageSegmentationMask, SegmentImageParams } from "./types.js";
+export interface ImageSize {
+    width: number;
+    height: number;
+}
+/**
+ * The image's size as displayed, which is the space SAM reads prompt
+ * coordinates in and reports boxes in. EXIF orientations 5–8 rotate by a
+ * quarter turn, so the stored width and height swap.
+ */
+export declare function displayedImageSize(image: Uint8Array): Promise<ImageSize>;
+/**
+ * The `input_text` for a segmentation request. A text prompt wins: SAM's API
+ * refuses a request that mixes a noun phrase with object prompts. Without
+ * one, the box and points become a single object's prompt on frame 0.
+ */
+export declare function samPromptText(params: Pick<SegmentImageParams, "prompt" | "box" | "points">, size: ImageSize): string;
+/**
+ * Decode SAM's output text into one full-frame mask per object, in the order
+ * the model reported them. Each mask raster covers only its object's box, so
+ * it is placed at that box on a black canvas the size of the source image.
+ * Empty output is SAM's answer for "no match" and yields an empty list.
+ */
+export declare function samMasksFromOutput(outputText: string, size: ImageSize, options?: {
+    label?: string | null;
+}): Promise<ImageSegmentationMask[]>;
+//# sourceMappingURL=meta-segmentation.d.ts.map

package/dist/providers/meta-segmentation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"meta-segmentation.d.ts","sourceRoot":"","sources":["../../src/providers/meta-segmentation.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EACV,qBAAqB,EACrB,kBAAkB,EACnB,MAAM,YAAY,CAAC;AAapB,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAU9E;AAMD;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,MAAM,EAAE,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,KAAK,GAAG,QAAQ,CAAC,EAC7D,IAAI,EAAE,SAAS,GACd,MAAM,CAgCR;AAUD;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,SAAS,EACf,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAO,GACtC,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAoDlC"}

package/dist/providers/meta-segmentation.js

@@ -0,0 +1,141 @@
+/**
+ * Request text and response decoding for Meta's SAM 3.1 segmentation model.
+ *
+ * SAM 3.1 runs on Meta's Responses API. The request carries one image and an
+ * `input_text` prompt: a short noun phrase, or an object prompt built from a
+ * box and points. The answer is not JSON but one line of special-token text,
+ * `<0f>` followed by `id<|box;…|><|mask;…|>` records, whose masks use a custom
+ * base85 encoding. `@meta-sam/parser` is Meta's own reader for that grammar;
+ * this module turns its records into full-frame mask PNGs.
+ * See https://dev.meta.ai/docs/media-segmentation and
+ * https://dev.meta.ai/docs/sam/reading-segmentation.
+ */
+import { buildObjectPrompt, decodeMaskToRaster, parseImageStream, recordsOfKind } from "@meta-sam/parser";
+import { importHidden } from "@nodetool-ai/config";
+async function loadSharp() {
+    const mod = await importHidden("sharp");
+    if (!mod)
+        throw new Error("Meta segmentation requires sharp");
+    return "default" in mod ? mod.default : mod;
+}
+/**
+ * The image's size as displayed, which is the space SAM reads prompt
+ * coordinates in and reports boxes in. EXIF orientations 5–8 rotate by a
+ * quarter turn, so the stored width and height swap.
+ */
+export async function displayedImageSize(image) {
+    const sharp = await loadSharp();
+    const meta = await sharp(Buffer.from(image)).metadata();
+    if (!meta.width || !meta.height) {
+        throw new Error("Meta segmentation could not read the image size");
+    }
+    const rotated = (meta.orientation ?? 1) >= 5;
+    return rotated
+        ? { width: meta.height, height: meta.width }
+        : { width: meta.width, height: meta.height };
+}
+function clampInt(value, min, max) {
+    return Math.max(min, Math.min(max, Math.floor(value)));
+}
+/**
+ * The `input_text` for a segmentation request. A text prompt wins: SAM's API
+ * refuses a request that mixes a noun phrase with object prompts. Without
+ * one, the box and points become a single object's prompt on frame 0.
+ */
+export function samPromptText(params, size) {
+    const prompt = params.prompt?.trim();
+    if (prompt)
+        return prompt;
+    const framePrompt = {};
+    if (params.box) {
+        const left = clampInt(params.box.x, 0, size.width - 1);
+        const top = clampInt(params.box.y, 0, size.height - 1);
+        framePrompt.box = {
+            left,
+            top,
+            right: clampInt(params.box.x + params.box.width, left + 1, size.width),
+            bottom: clampInt(params.box.y + params.box.height, top + 1, size.height)
+        };
+    }
+    if (params.points && params.points.length > 0) {
+        framePrompt.points = params.points.map((point) => ({
+            x: clampInt(point.x, 0, size.width - 1),
+            y: clampInt(point.y, 0, size.height - 1),
+            label: point.include ? "positive" : "negative"
+        }));
+    }
+    if (!framePrompt.box && !framePrompt.points) {
+        throw new Error("Meta SAM 3.1 needs a text prompt (a short noun phrase), a box, or points");
+    }
+    return buildObjectPrompt({
+        size,
+        objects: [{ id: 1, prompts: [{ frame: 0, ...framePrompt }] }]
+    });
+}
+/** The parser reads a Responses event stream; a complete answer is one delta. */
+async function* outputTextEvents(text) {
+    const lane = { item_id: "output", output_index: 0, content_index: 0 };
+    yield { type: "response.output_text.delta", delta: text, ...lane };
+    yield { type: "response.output_text.done", text, ...lane };
+    yield { type: "response.completed" };
+}
+/**
+ * Decode SAM's output text into one full-frame mask per object, in the order
+ * the model reported them. Each mask raster covers only its object's box, so
+ * it is placed at that box on a black canvas the size of the source image.
+ * Empty output is SAM's answer for "no match" and yields an empty list.
+ */
+export async function samMasksFromOutput(outputText, size, options = {}) {
+    if (!outputText.trim())
+        return [];
+    const result = await parseImageStream(outputTextEvents(outputText))
+        .finalResult;
+    const maskRecords = recordsOfKind(result.records, "mask");
+    const errors = result.diagnostics.filter((d) => d.severity === "error");
+    if (maskRecords.length === 0 && errors.length > 0) {
+        throw new Error(`Meta SAM 3.1 returned output that could not be read: ${errors
+            .map((d) => `${d.code}: ${d.message}`)
+            .join("; ")}`);
+    }
+    const sharp = await loadSharp();
+    const masks = [];
+    for (const record of maskRecords) {
+        const raster = decodeMaskToRaster(record.mask);
+        const { width: rasterWidth, height: rasterHeight } = record.mask;
+        const canvas = new Uint8Array(size.width * size.height);
+        for (let row = 0; row < rasterHeight; row++) {
+            const y = record.bounds.top + row;
+            if (y < 0 || y >= size.height)
+                continue;
+            for (let col = 0; col < rasterWidth; col++) {
+                const x = record.bounds.left + col;
+                if (x < 0 || x >= size.width)
+                    continue;
+                if (raster[row * rasterWidth + col] === 1) {
+                    canvas[y * size.width + x] = 255;
+                }
+            }
+        }
+        const png = await sharp(Buffer.from(canvas), {
+            raw: { width: size.width, height: size.height, channels: 1 }
+        })
+            .png()
+            .toBuffer();
+        masks.push({
+            mask: new Uint8Array(png),
+            mimeType: "image/png",
+            width: size.width,
+            height: size.height,
+            label: options.label ?? null,
+            confidence: record.confidence ?? null,
+            box: {
+                x: record.bounds.left,
+                y: record.bounds.top,
+                width: record.bounds.right - record.bounds.left,
+                height: record.bounds.bottom - record.bounds.top
+            }
+        });
+    }
+    return masks;
+}
+//# sourceMappingURL=meta-segmentation.js.map

package/dist/providers/meta-segmentation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"meta-segmentation.js","sourceRoot":"","sources":["../../src/providers/meta-segmentation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,gBAAgB,EAChB,aAAa,EAGd,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AASnD,KAAK,UAAU,SAAS;IACtB,MAAM,GAAG,GAAG,MAAM,YAAY,CAC5B,OAAO,CACR,CAAC;IACF,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC9D,OAAO,SAAS,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9C,CAAC;AAOD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,KAAiB;IACxD,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAChC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACxD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;IAC7C,OAAO,OAAO;QACZ,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE;QAC5C,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW;IACvD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,MAA6D,EAC7D,IAAe;IAEf,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC;IACrC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,MAAM,WAAW,GACf,EAAE,CAAC;IACL,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;QACf,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACvD,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvD,WAAW,CAAC,GAAG,GAAG;YAChB,IAAI;YACJ,GAAG;YACH,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC;YACtE,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;SACzE,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9C,WAAW,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACjD,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YACvC,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;YACxC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU;SAC/C,CAAC,CAAC,CAAC;IACN,CAAC;IACD,IAAI,CAAC,WAAW,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CACb,0EAA0E,CAC3E,CAAC;IACJ,CAAC;IACD,OAAO,iBAAiB,CAAC;QACvB,IAAI;QACJ,OAAO,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,WAAW,EAAE,CAAC,EAAE,CAAC;KAC9D,CAAC,CAAC;AACL,CAAC;AAED,iFAAiF;AACjF,KAAK,SAAS,CAAC,CAAC,gBAAgB,CAAC,IAAY;IAC3C,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;IACtE,MAAM,EAAE,IAAI,EAAE,4BAA4B,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC;IACnE,MAAM,EAAE,IAAI,EAAE,2BAA2B,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,CAAC;IAC3D,MAAM,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAC;AACvC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,UAAkB,EAClB,IAAe,EACf,OAAO,GAA8B,EAAE;IAEvC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;SAChE,WAAW,CAAC;IACf,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC;IACxE,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CACb,wDAAwD,MAAM;aAC3D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;aACrC,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAChC,MAAM,KAAK,GAA4B,EAAE,CAAC;IAC1C,KAAK,MAAM,MAAM,IAAI,WAAW,EAAE,CAAC;QACjC,MAAM,MAAM,GAAG,kBAAkB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC/C,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;QACjE,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC;YAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM;gBAAE,SAAS;YACxC,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,WAAW,EAAE,GAAG,EAAE,EAAE,CAAC;gBAC3C,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,GAAG,GAAG,CAAC;gBACnC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK;oBAAE,SAAS;gBACvC,IAAI,MAAM,CAAC,GAAG,GAAG,WAAW,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC1C,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC;gBACnC,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE;YAC3C,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EAAE;SAC7D,CAAC;aACC,GAAG,EAAE;aACL,QAAQ,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC;YACT,IAAI,EAAE,IAAI,UAAU,CAAC,GAAG,CAAC;YACzB,QAAQ,EAAE,WAAW;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI;YAC5B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;YACrC,GAAG,EAAE;gBACH,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI;gBACrB,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG;gBACpB,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI;gBAC/C,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG;aACjD;SACF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/providers/meta-transcription.d.ts

@@ -0,0 +1,31 @@
+import type { ASRResult } from "./types.js";
+/**
+ * Whether `bytes` is a WAV file the transcribe endpoint accepts unchanged.
+ * Walks the RIFF chunks by their declared sizes (word-aligned) to find `fmt `,
+ * and answers false for anything it cannot read, which sends the audio
+ * through ffmpeg rather than to a 400.
+ */
+export declare function isTranscribeReadyWav(bytes: Uint8Array): boolean;
+/** ffmpeg is not installed. Names what needed it and the way out. */
+export declare class TranscribeAudioConversionUnavailableError extends Error {
+    constructor();
+}
+/** `bytes` in the one WAV shape the transcribe endpoint accepts. */
+export declare function toTranscribeWav(bytes: Uint8Array, signal?: AbortSignal): Promise<Uint8Array>;
+/**
+ * Meta's `languageBias` takes English language names ("English", "French"),
+ * while NodeTool's ASR callers pass ISO 639 codes ("en", "fr-CA"). A value
+ * that is already a name, or a code Intl cannot name, passes through.
+ */
+export declare function languageBiasName(language: string): string;
+/** Recognition-bias terms from a free-text prompt: comma or line separated. */
+export declare function keywordsFromPrompt(prompt: string): string[];
+/**
+ * Decode the transcribe endpoint's JSON answer. `turns` carry millisecond
+ * timings in ENDPOINTING and DIARIZATION modes and become chunks in seconds.
+ */
+export declare function decodeTranscribeResponse(payload: unknown): {
+    result: ASRResult;
+    audioDurationMs: number | null;
+};
+//# sourceMappingURL=meta-transcription.d.ts.map

package/dist/providers/meta-transcription.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"meta-transcription.d.ts","sourceRoot":"","sources":["../../src/providers/meta-transcription.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,SAAS,EAAc,MAAM,YAAY,CAAC;AASxD;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CA2B/D;AAED,qEAAqE;AACrE,qBAAa,yCAA0C,SAAQ,KAAK;IAClE,cAQC;CACF;AAkED,oEAAoE;AACpE,wBAAsB,eAAe,CACnC,KAAK,EAAE,UAAU,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,UAAU,CAAC,CAGrB;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAezD;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAK3D;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,OAAO,GAAG;IAC1D,MAAM,EAAE,SAAS,CAAC;IAClB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC,CA6BA"}