@loonylabs/tti-middleware 1.9.0 → 1.11.0

package/README.md

@@ -24,6 +24,7 @@
 - [Configuration](#%EF%B8%8F-configuration)
 - [Providers & Models](#-providers--models)
 - [Character Consistency](#-character-consistency)
+- [Inpainting / Image Editing](#inpainting--image-editing)
 - [GDPR / Compliance](#-gdpr--compliance)
 - [API Reference](#-api-reference)
 - [Advanced Features](#-advanced-features)
@@ -44,6 +45,7 @@
   - **Eden AI**: Aggregator with access to OpenAI, Stability AI, Replicate (experimental)
   - **IONOS**: German cloud provider with OpenAI-compatible API (experimental)
 - **Character Consistency**: Generate consistent characters across multiple images (perfect for children's book illustrations)
+- **Inpainting**: Fix specific areas of a generated image without regenerating the entire scene — via Vertex AI `imagen-capability` model. Supports optional subject reference images (`maskReferenceImages`) to guide *what* gets inserted into the masked area
 - **GDPR/DSGVO Compliance**: Built-in EU region support with automatic fallback
 - **Region Rotation**: Opt-in region rotation on quota errors (429) for Google Cloud — rotate through regions instead of retrying the same exhausted region
 - **Retry Logic**: Exponential backoff with jitter for transient errors (429, 408, 5xx, timeouts)
@@ -307,6 +309,72 @@ const duelScene = await service.generate({
 
 - Model must be `gemini-flash-image` (only model supporting character consistency)
 
+## Inpainting / Image Editing
+
+The `imagen-capability` model supports mask-based inpainting via Vertex AI. This is the **only** model that supports pixel-precise editing with a mask image.
+
+### Basic Inpainting
+
+```typescript
+const result = await service.generate({
+  model: 'imagen-capability',
+  prompt: 'Remove the extra arm and fill with matching forest background',
+  baseImage: { base64: originalImageBase64, mimeType: 'image/png' },
+  maskImage: { base64: maskBase64,          mimeType: 'image/png' },
+  editMode: 'inpainting-remove',  // default: 'inpainting-insert'
+  maskDilation: 0.02,             // optional, 0.0–1.0, default 0.01
+});
+```
+
+**How the mask works:**
+- White pixels = area the model will regenerate
+- Black pixels = area preserved exactly as-is
+- Mask must have identical dimensions to `baseImage`
+
+### Guided Inpainting with Subject References
+
+Use `maskReferenceImages` to provide a reference photo of the subject to insert — e.g. "place **this** character into the masked region":
+
+```typescript
+const result = await service.generate({
+  model: 'imagen-capability',
+  prompt: 'The character standing in a bright forest clearing, photorealistic',
+  baseImage:  { base64: sceneBase64,        mimeType: 'image/png' },
+  maskImage:  { base64: maskBase64,         mimeType: 'image/png' },
+  editMode: 'inpainting-insert',
+  maskReferenceImages: [
+    {
+      base64: characterRefBase64,
+      mimeType: 'image/png',
+      subjectType: 'person',   // 'person' | 'animal' | 'product' | 'default'
+    },
+  ],
+});
+```
+
+**Subject types:**
+
+| `subjectType` | Use case |
+|---------------|----------|
+| `'person'` | Human character |
+| `'animal'` | Animal or creature |
+| `'product'` | Object, item, product |
+| `'default'` | Let the model decide (safe fallback) |
+
+**Notes:**
+- `maskReferenceImages` only works with `editMode: 'inpainting-insert'`
+- Gemini models do **not** support mask-based inpainting or `maskReferenceImages`
+- `maskReferenceImages` without `baseImage` throws a validation error
+
+### Supported `editMode` Values
+
+| Value | Description |
+|-------|-------------|
+| `'inpainting-insert'` | Add or replace content in masked area (default) |
+| `'inpainting-remove'` | Remove content and fill with matching background |
+| `'background-swap'` | Replace background, preserve foreground |
+| `'outpainting'` | Extend image beyond its boundaries into the masked area |
+
 ## GDPR / Compliance
 
 ### Provider Compliance Overview
@@ -365,15 +433,30 @@ interface TTIRequest {
   n?: number;               // Number of images (default: 1)
   aspectRatio?: string;     // '1:1', '16:9', '4:3', etc.
 
-  // Character consistency
+  // Character consistency (Gemini models only)
   referenceImages?: TTIReferenceImage[];
   subjectDescription?: string;
 
+  // Inpainting / image editing (imagen-capability only)
+  baseImage?: TTIReferenceImage;   // Activates edit mode when set
+  maskImage?: TTIReferenceImage;   // Required when baseImage is set
+  maskDilation?: number;           // 0.0–1.0, default 0.01
+  editMode?: 'inpainting-insert' | 'inpainting-remove' | 'background-swap' | 'outpainting';
+  maskReferenceImages?: TTIMaskReferenceImage[];  // Subject refs for guided inpainting
+
   // Retry configuration
   retry?: boolean | RetryOptions;  // true (default), false, or custom config
 
   providerOptions?: Record<string, unknown>;
 }
+
+type TTISubjectType = 'person' | 'animal' | 'product' | 'default';
+
+interface TTIMaskReferenceImage {
+  base64: string;
+  mimeType?: string;
+  subjectType?: TTISubjectType;  // defaults to 'default'
+}
 ```
 
 ### TTIResponse

package/dist/middleware/services/tti/providers/base-tti-provider.js

@@ -196,6 +196,38 @@ class BaseTTIProvider {
         if (!request.prompt || request.prompt.trim().length === 0) {
             throw new InvalidConfigError(this.providerName, 'Prompt cannot be empty');
         }
+        // If baseImage is provided, validate inpainting requirements
+        if (request.baseImage) {
+            if (!request.baseImage.base64 || request.baseImage.base64.trim().length === 0) {
+                throw new InvalidConfigError(this.providerName, 'baseImage has empty base64 data');
+            }
+            const modelId = request.model || this.getDefaultModel();
+            if (!this.modelSupportsCapability(modelId, 'imageEditing')) {
+                throw new CapabilityNotSupportedError(this.providerName, 'imageEditing', modelId);
+            }
+            if (!request.maskImage) {
+                throw new InvalidConfigError(this.providerName, 'maskImage is required when baseImage is set');
+            }
+            if (!request.maskImage.base64 || request.maskImage.base64.trim().length === 0) {
+                throw new InvalidConfigError(this.providerName, 'maskImage has empty base64 data');
+            }
+            if (request.maskDilation !== undefined) {
+                if (request.maskDilation < 0 || request.maskDilation > 1) {
+                    throw new InvalidConfigError(this.providerName, 'maskDilation must be between 0.0 and 1.0');
+                }
+            }
+            if (request.maskReferenceImages && request.maskReferenceImages.length > 0) {
+                for (let i = 0; i < request.maskReferenceImages.length; i++) {
+                    const ref = request.maskReferenceImages[i];
+                    if (!ref.base64 || ref.base64.trim().length === 0) {
+                        throw new InvalidConfigError(this.providerName, `maskReferenceImages[${i}] has empty base64 data`);
+                    }
+                }
+            }
+        }
+        if (request.maskReferenceImages && request.maskReferenceImages.length > 0 && !request.baseImage) {
+            throw new InvalidConfigError(this.providerName, 'maskReferenceImages requires baseImage to be set');
+        }
         // If reference images are provided, validate them
         if (request.referenceImages && request.referenceImages.length > 0) {
             const modelId = request.model || this.getDefaultModel();

package/dist/middleware/services/tti/providers/google-cloud-provider.d.ts

@@ -63,6 +63,10 @@ export declare class GoogleCloudTTIProvider extends BaseTTIProvider {
     private generateWithImagen;
     private getAiplatformClient;
     private processImagenResponse;
+    /** Maps our editMode values to the Vertex AI API constants */
+    private static readonly EDIT_MODE_MAP;
+    private static readonly SUBJECT_TYPE_MAP;
+    private editWithImagen;
     private generateWithGemini;
     private getGenaiClient;
     private buildCharacterConsistencyPrompt;

package/dist/middleware/services/tti/providers/google-cloud-provider.js

@@ -131,6 +131,27 @@ const GOOGLE_CLOUD_MODELS = [
         availableRegions: IMAGEN_4_REGIONS,
         pricingUrl: 'https://cloud.google.com/vertex-ai/generative-ai/pricing',
     },
+    // ── Imagen Capability model (Vertex AI editing / inpainting) ──
+    {
+        id: 'imagen-capability',
+        displayName: 'Imagen 3 Capability (Editing)',
+        capabilities: {
+            textToImage: false,
+            characterConsistency: false,
+            imageEditing: true,
+            maxImagesPerRequest: 4,
+        },
+        availableRegions: [
+            'europe-west1',
+            'europe-west2',
+            'europe-west3',
+            'europe-west4',
+            'europe-west9',
+            'us-central1',
+            'us-east4',
+        ],
+        pricingUrl: 'https://cloud.google.com/vertex-ai/generative-ai/pricing',
+    },
     // ── Gemini models (Vertex AI generateContent API) ──────────
     {
         id: 'gemini-flash-image',
@@ -185,6 +206,7 @@ const MODEL_ID_MAP = {
     'imagen-4': 'imagen-4.0-generate-001',
     'imagen-4-fast': 'imagen-4.0-fast-generate-001',
     'imagen-4-ultra': 'imagen-4.0-ultra-generate-001',
+    'imagen-capability': 'imagen-3.0-capability-001',
     'gemini-flash-image': 'gemini-2.5-flash-image',
     'gemini-pro-image': 'gemini-3-pro-image-preview',
     'gemini-flash-image-2': 'gemini-3.1-flash-image-preview',
@@ -296,10 +318,18 @@ class GoogleCloudTTIProvider extends base_tti_provider_1.BaseTTIProvider {
             hasReferenceImages: (0, base_tti_provider_1.hasReferenceImages)(request),
         });
         const isGeminiModel = GEMINI_API_MODELS.has(modelId);
-        const operationName = isGeminiModel ? 'Gemini API call' : 'Imagen API call';
+        const isEditRequest = !!request.baseImage;
+        const operationName = isEditRequest
+            ? 'Imagen edit API call'
+            : isGeminiModel
+                ? 'Gemini API call'
+                : 'Imagen API call';
         // Operation lambda reads currentRegion from closure
         const operation = () => {
-            if (isGeminiModel) {
+            if (isEditRequest) {
+                return this.editWithImagen(request, modelId, currentRegion);
+            }
+            else if (isGeminiModel) {
                 return this.generateWithGemini(request, modelId, currentRegion);
             }
             else {
@@ -528,6 +558,94 @@ class GoogleCloudTTIProvider extends base_tti_provider_1.BaseTTIProvider {
             usage,
         };
     }
+    async editWithImagen(request, modelId, region) {
+        const startTime = Date.now();
+        const internalModelId = MODEL_ID_MAP[modelId];
+        this.lastUsedRegion = region;
+        try {
+            const { client, helpers } = await this.getAiplatformClient(region);
+            const endpoint = `projects/${this.config.projectId}/locations/${region}/publishers/google/models/${internalModelId}`;
+            // Build referenceImages array: [RAW base image, MASK image]
+            const referenceImages = [
+                {
+                    referenceType: 'REFERENCE_TYPE_RAW',
+                    referenceId: 1,
+                    referenceImage: {
+                        bytesBase64Encoded: request.baseImage.base64,
+                    },
+                },
+                {
+                    referenceType: 'REFERENCE_TYPE_MASK',
+                    referenceId: 2,
+                    referenceImage: {
+                        bytesBase64Encoded: request.maskImage.base64,
+                    },
+                    maskImageConfig: {
+                        maskMode: 'MASK_MODE_USER_PROVIDED',
+                        dilation: request.maskDilation ?? 0.01,
+                    },
+                },
+            ];
+            // Append optional subject reference images for guided inpainting
+            if (request.maskReferenceImages && request.maskReferenceImages.length > 0) {
+                for (const [i, ref] of request.maskReferenceImages.entries()) {
+                    const subjectType = GoogleCloudTTIProvider.SUBJECT_TYPE_MAP[ref.subjectType ?? 'default'] ??
+                        'SUBJECT_TYPE_DEFAULT';
+                    referenceImages.push({
+                        referenceType: 'REFERENCE_TYPE_SUBJECT',
+                        referenceId: 3 + i,
+                        referenceImage: {
+                            bytesBase64Encoded: ref.base64,
+                        },
+                        subjectImageConfig: {
+                            subjectType,
+                        },
+                    });
+                }
+            }
+            const instanceValue = {
+                prompt: request.prompt,
+                referenceImages,
+            };
+            const instance = helpers.toValue(instanceValue);
+            // Map editMode to Vertex AI constant, default to inpainting-insert
+            const editModeKey = request.editMode ?? 'inpainting-insert';
+            const vertexEditMode = GoogleCloudTTIProvider.EDIT_MODE_MAP[editModeKey] ?? 'EDIT_MODE_INPAINT_INSERTION';
+            const parameterValue = {
+                editMode: vertexEditMode,
+                sampleCount: request.n || 1,
+                editConfig: {
+                    baseSteps: request.providerOptions?.baseSteps ?? 35,
+                },
+            };
+            const parameters = helpers.toValue(parameterValue);
+            this.log('info', 'Sending Imagen edit request to Vertex AI', {
+                endpoint,
+                editMode: vertexEditMode,
+                dilation: request.maskDilation ?? 0.01,
+            });
+            const [response] = await client.predict({
+                endpoint,
+                instances: [instance],
+                parameters,
+            });
+            const duration = Date.now() - startTime;
+            this.log('info', `Imagen edit response received in ${duration}ms`, {
+                duration,
+                hasPredictions: !!response.predictions?.length,
+            });
+            if (!response.predictions || response.predictions.length === 0) {
+                throw new base_tti_provider_1.GenerationFailedError(this.providerName, 'No images returned from Imagen edit API');
+            }
+            return this.processImagenResponse(response.predictions, helpers, modelId, duration);
+        }
+        catch (error) {
+            if (error instanceof base_tti_provider_1.InvalidConfigError || error instanceof base_tti_provider_1.GenerationFailedError) {
+                throw error;
+            }
+            throw this.handleError(error, 'during Imagen edit API call');
+        }
+    }
     // ============================================================
     // PRIVATE: GEMINI IMAGE IMPLEMENTATION
     // ============================================================
@@ -702,3 +820,19 @@ IMPORTANT: Maintain exact visual consistency with the subject in the reference -
     }
 }
 exports.GoogleCloudTTIProvider = GoogleCloudTTIProvider;
+// ============================================================
+// PRIVATE: IMAGEN EDITING / INPAINTING IMPLEMENTATION
+// ============================================================
+/** Maps our editMode values to the Vertex AI API constants */
+GoogleCloudTTIProvider.EDIT_MODE_MAP = {
+    'inpainting-insert': 'EDIT_MODE_INPAINT_INSERTION',
+    'inpainting-remove': 'EDIT_MODE_INPAINT_REMOVAL',
+    'background-swap': 'EDIT_MODE_BGSWAP',
+    'outpainting': 'EDIT_MODE_OUTPAINT',
+};
+GoogleCloudTTIProvider.SUBJECT_TYPE_MAP = {
+    person: 'SUBJECT_TYPE_PERSON',
+    animal: 'SUBJECT_TYPE_ANIMAL',
+    product: 'SUBJECT_TYPE_PRODUCT',
+    default: 'SUBJECT_TYPE_DEFAULT',
+};

package/dist/middleware/types/index.d.ts

@@ -82,6 +82,30 @@ export interface TTIReferenceImage {
     /** MIME type of the image (e.g., 'image/png', 'image/jpeg') */
     mimeType?: string;
 }
+/**
+ * Subject type hint for mask reference images.
+ * Helps the model understand what kind of subject is shown in the reference image.
+ * - 'person'  — a human character
+ * - 'animal'  — an animal or creature
+ * - 'product' — an object, product, or item
+ * - 'default' — let the model decide (fallback)
+ */
+export type TTISubjectType = 'person' | 'animal' | 'product' | 'default';
+/**
+ * Reference image for mask-based inpainting (subject reference).
+ * Used with maskReferenceImages to guide what the model inserts into the masked area.
+ */
+export interface TTIMaskReferenceImage {
+    /** Base64-encoded image data of the subject to insert */
+    base64: string;
+    /** MIME type of the image (e.g., 'image/png', 'image/jpeg') */
+    mimeType?: string;
+    /**
+     * Subject type hint for the model.
+     * Defaults to 'default' if omitted.
+     */
+    subjectType?: TTISubjectType;
+}
 /**
  * Unified TTI generation request
  * Works for both simple text-to-image and character consistency
@@ -105,6 +129,38 @@ export interface TTIRequest {
      * Required when using referenceImages (e.g., "cute cartoon bear with red hat")
      */
     subjectDescription?: string;
+    /**
+     * Base image to edit. When present, activates "edit mode" instead of text-to-image generation.
+     * Requires maskImage and a model that supports imageEditing capability.
+     */
+    baseImage?: TTIReferenceImage;
+    /**
+     * Mask image for inpainting. White pixels = regenerate, black pixels = preserve.
+     * Must have identical dimensions to baseImage.
+     * Required when baseImage is set.
+     */
+    maskImage?: TTIReferenceImage;
+    /**
+     * Mask dilation: expands the mask boundary to smooth hard edges (0.0–1.0, default 0.01).
+     * Useful when hand-drawn masks have jagged edges.
+     */
+    maskDilation?: number;
+    /**
+     * Edit operation to perform on the masked region.
+     * - 'inpainting-insert': add or replace content in the masked area (default)
+     * - 'inpainting-remove': remove content and fill with matching background
+     * - 'background-swap': replace background while preserving foreground
+     * - 'outpainting': extend image beyond its boundaries into the masked area
+     */
+    editMode?: 'inpainting-insert' | 'inpainting-remove' | 'background-swap' | 'outpainting';
+    /**
+     * Optional subject reference images for mask-based inpainting.
+     * Only valid when baseImage and maskImage are set.
+     * Each entry guides the model to insert a specific subject into the masked area
+     * (e.g., "place the character from this reference image into the mask").
+     * Only supported by the 'imagen-capability' model.
+     */
+    maskReferenceImages?: TTIMaskReferenceImage[];
     /** Additional provider-specific options */
     providerOptions?: Record<string, unknown>;
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loonylabs/tti-middleware",
-  "version": "1.9.0",
+  "version": "1.11.0",
   "description": "Provider-agnostic Text-to-Image middleware with GDPR compliance. Supports Google Cloud (Imagen, Gemini), Eden AI, and IONOS.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",