@loonylabs/tti-middleware 1.9.0 → 1.10.0

package/README.md

@@ -44,6 +44,7 @@
   - **Eden AI**: Aggregator with access to OpenAI, Stability AI, Replicate (experimental)
   - **IONOS**: German cloud provider with OpenAI-compatible API (experimental)
 - **Character Consistency**: Generate consistent characters across multiple images (perfect for children's book illustrations)
+- **Inpainting**: Fix specific areas of a generated image without regenerating the entire scene — via Vertex AI `imagen-capability` model
 - **GDPR/DSGVO Compliance**: Built-in EU region support with automatic fallback
 - **Region Rotation**: Opt-in region rotation on quota errors (429) for Google Cloud — rotate through regions instead of retrying the same exhausted region
 - **Retry Logic**: Exponential backoff with jitter for transient errors (429, 408, 5xx, timeouts)

package/dist/middleware/services/tti/providers/base-tti-provider.js

@@ -196,6 +196,27 @@ class BaseTTIProvider {
         if (!request.prompt || request.prompt.trim().length === 0) {
             throw new InvalidConfigError(this.providerName, 'Prompt cannot be empty');
         }
+        // If baseImage is provided, validate inpainting requirements
+        if (request.baseImage) {
+            if (!request.baseImage.base64 || request.baseImage.base64.trim().length === 0) {
+                throw new InvalidConfigError(this.providerName, 'baseImage has empty base64 data');
+            }
+            const modelId = request.model || this.getDefaultModel();
+            if (!this.modelSupportsCapability(modelId, 'imageEditing')) {
+                throw new CapabilityNotSupportedError(this.providerName, 'imageEditing', modelId);
+            }
+            if (!request.maskImage) {
+                throw new InvalidConfigError(this.providerName, 'maskImage is required when baseImage is set');
+            }
+            if (!request.maskImage.base64 || request.maskImage.base64.trim().length === 0) {
+                throw new InvalidConfigError(this.providerName, 'maskImage has empty base64 data');
+            }
+            if (request.maskDilation !== undefined) {
+                if (request.maskDilation < 0 || request.maskDilation > 1) {
+                    throw new InvalidConfigError(this.providerName, 'maskDilation must be between 0.0 and 1.0');
+                }
+            }
+        }
         // If reference images are provided, validate them
         if (request.referenceImages && request.referenceImages.length > 0) {
             const modelId = request.model || this.getDefaultModel();

package/dist/middleware/services/tti/providers/google-cloud-provider.d.ts

@@ -63,6 +63,9 @@ export declare class GoogleCloudTTIProvider extends BaseTTIProvider {
     private generateWithImagen;
     private getAiplatformClient;
     private processImagenResponse;
+    /** Maps our editMode values to the Vertex AI API constants */
+    private static readonly EDIT_MODE_MAP;
+    private editWithImagen;
     private generateWithGemini;
     private getGenaiClient;
     private buildCharacterConsistencyPrompt;

package/dist/middleware/services/tti/providers/google-cloud-provider.js

@@ -131,6 +131,27 @@ const GOOGLE_CLOUD_MODELS = [
         availableRegions: IMAGEN_4_REGIONS,
         pricingUrl: 'https://cloud.google.com/vertex-ai/generative-ai/pricing',
     },
+    // ── Imagen Capability model (Vertex AI editing / inpainting) ──
+    {
+        id: 'imagen-capability',
+        displayName: 'Imagen 3 Capability (Editing)',
+        capabilities: {
+            textToImage: false,
+            characterConsistency: false,
+            imageEditing: true,
+            maxImagesPerRequest: 4,
+        },
+        availableRegions: [
+            'europe-west1',
+            'europe-west2',
+            'europe-west3',
+            'europe-west4',
+            'europe-west9',
+            'us-central1',
+            'us-east4',
+        ],
+        pricingUrl: 'https://cloud.google.com/vertex-ai/generative-ai/pricing',
+    },
     // ── Gemini models (Vertex AI generateContent API) ──────────
     {
         id: 'gemini-flash-image',
@@ -185,6 +206,7 @@ const MODEL_ID_MAP = {
     'imagen-4': 'imagen-4.0-generate-001',
     'imagen-4-fast': 'imagen-4.0-fast-generate-001',
     'imagen-4-ultra': 'imagen-4.0-ultra-generate-001',
+    'imagen-capability': 'imagen-3.0-capability-001',
     'gemini-flash-image': 'gemini-2.5-flash-image',
     'gemini-pro-image': 'gemini-3-pro-image-preview',
     'gemini-flash-image-2': 'gemini-3.1-flash-image-preview',
@@ -296,10 +318,18 @@ class GoogleCloudTTIProvider extends base_tti_provider_1.BaseTTIProvider {
             hasReferenceImages: (0, base_tti_provider_1.hasReferenceImages)(request),
         });
         const isGeminiModel = GEMINI_API_MODELS.has(modelId);
-        const operationName = isGeminiModel ? 'Gemini API call' : 'Imagen API call';
+        const isEditRequest = !!request.baseImage;
+        const operationName = isEditRequest
+            ? 'Imagen edit API call'
+            : isGeminiModel
+                ? 'Gemini API call'
+                : 'Imagen API call';
         // Operation lambda reads currentRegion from closure
         const operation = () => {
-            if (isGeminiModel) {
+            if (isEditRequest) {
+                return this.editWithImagen(request, modelId, currentRegion);
+            }
+            else if (isGeminiModel) {
                 return this.generateWithGemini(request, modelId, currentRegion);
             }
             else {
@@ -528,6 +558,77 @@ class GoogleCloudTTIProvider extends base_tti_provider_1.BaseTTIProvider {
             usage,
         };
     }
+    async editWithImagen(request, modelId, region) {
+        const startTime = Date.now();
+        const internalModelId = MODEL_ID_MAP[modelId];
+        this.lastUsedRegion = region;
+        try {
+            const { client, helpers } = await this.getAiplatformClient(region);
+            const endpoint = `projects/${this.config.projectId}/locations/${region}/publishers/google/models/${internalModelId}`;
+            // Build referenceImages array: [RAW base image, MASK image]
+            const referenceImages = [
+                {
+                    referenceType: 'REFERENCE_TYPE_RAW',
+                    referenceId: 1,
+                    referenceImage: {
+                        bytesBase64Encoded: request.baseImage.base64,
+                    },
+                },
+                {
+                    referenceType: 'REFERENCE_TYPE_MASK',
+                    referenceId: 2,
+                    referenceImage: {
+                        bytesBase64Encoded: request.maskImage.base64,
+                    },
+                    maskImageConfig: {
+                        maskMode: 'MASK_MODE_USER_PROVIDED',
+                        dilation: request.maskDilation ?? 0.01,
+                    },
+                },
+            ];
+            const instanceValue = {
+                prompt: request.prompt,
+                referenceImages,
+            };
+            const instance = helpers.toValue(instanceValue);
+            // Map editMode to Vertex AI constant, default to inpainting-insert
+            const editModeKey = request.editMode ?? 'inpainting-insert';
+            const vertexEditMode = GoogleCloudTTIProvider.EDIT_MODE_MAP[editModeKey] ?? 'EDIT_MODE_INPAINT_INSERTION';
+            const parameterValue = {
+                editMode: vertexEditMode,
+                sampleCount: request.n || 1,
+                editConfig: {
+                    baseSteps: request.providerOptions?.baseSteps ?? 35,
+                },
+            };
+            const parameters = helpers.toValue(parameterValue);
+            this.log('info', 'Sending Imagen edit request to Vertex AI', {
+                endpoint,
+                editMode: vertexEditMode,
+                dilation: request.maskDilation ?? 0.01,
+            });
+            const [response] = await client.predict({
+                endpoint,
+                instances: [instance],
+                parameters,
+            });
+            const duration = Date.now() - startTime;
+            this.log('info', `Imagen edit response received in ${duration}ms`, {
+                duration,
+                hasPredictions: !!response.predictions?.length,
+            });
+            if (!response.predictions || response.predictions.length === 0) {
+                throw new base_tti_provider_1.GenerationFailedError(this.providerName, 'No images returned from Imagen edit API');
+            }
+            return this.processImagenResponse(response.predictions, helpers, modelId, duration);
+        }
+        catch (error) {
+            if (error instanceof base_tti_provider_1.InvalidConfigError || error instanceof base_tti_provider_1.GenerationFailedError) {
+                throw error;
+            }
+            throw this.handleError(error, 'during Imagen edit API call');
+        }
+    }
     // ============================================================
     // PRIVATE: GEMINI IMAGE IMPLEMENTATION
     // ============================================================
@@ -702,3 +803,13 @@ IMPORTANT: Maintain exact visual consistency with the subject in the reference -
     }
 }
 exports.GoogleCloudTTIProvider = GoogleCloudTTIProvider;
+// ============================================================
+// PRIVATE: IMAGEN EDITING / INPAINTING IMPLEMENTATION
+// ============================================================
+/** Maps our editMode values to the Vertex AI API constants */
+GoogleCloudTTIProvider.EDIT_MODE_MAP = {
+    'inpainting-insert': 'EDIT_MODE_INPAINT_INSERTION',
+    'inpainting-remove': 'EDIT_MODE_INPAINT_REMOVAL',
+    'background-swap': 'EDIT_MODE_BGSWAP',
+    'outpainting': 'EDIT_MODE_OUTPAINT',
+};

package/dist/middleware/types/index.d.ts

@@ -105,6 +105,30 @@ export interface TTIRequest {
      * Required when using referenceImages (e.g., "cute cartoon bear with red hat")
      */
     subjectDescription?: string;
+    /**
+     * Base image to edit. When present, activates "edit mode" instead of text-to-image generation.
+     * Requires maskImage and a model that supports imageEditing capability.
+     */
+    baseImage?: TTIReferenceImage;
+    /**
+     * Mask image for inpainting. White pixels = regenerate, black pixels = preserve.
+     * Must have identical dimensions to baseImage.
+     * Required when baseImage is set.
+     */
+    maskImage?: TTIReferenceImage;
+    /**
+     * Mask dilation: expands the mask boundary to smooth hard edges (0.0–1.0, default 0.01).
+     * Useful when hand-drawn masks have jagged edges.
+     */
+    maskDilation?: number;
+    /**
+     * Edit operation to perform on the masked region.
+     * - 'inpainting-insert': add or replace content in the masked area (default)
+     * - 'inpainting-remove': remove content and fill with matching background
+     * - 'background-swap': replace background while preserving foreground
+     * - 'outpainting': extend image beyond its boundaries into the masked area
+     */
+    editMode?: 'inpainting-insert' | 'inpainting-remove' | 'background-swap' | 'outpainting';
     /** Additional provider-specific options */
     providerOptions?: Record<string, unknown>;
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loonylabs/tti-middleware",
-  "version": "1.9.0",
+  "version": "1.10.0",
   "description": "Provider-agnostic Text-to-Image middleware with GDPR compliance. Supports Google Cloud (Imagen, Gemini), Eden AI, and IONOS.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",