@loonylabs/tti-middleware 1.10.0 → 1.11.0

package/README.md

@@ -24,6 +24,7 @@
 - [Configuration](#%EF%B8%8F-configuration)
 - [Providers & Models](#-providers--models)
 - [Character Consistency](#-character-consistency)
+- [Inpainting / Image Editing](#inpainting--image-editing)
 - [GDPR / Compliance](#-gdpr--compliance)
 - [API Reference](#-api-reference)
 - [Advanced Features](#-advanced-features)
@@ -44,7 +45,7 @@
   - **Eden AI**: Aggregator with access to OpenAI, Stability AI, Replicate (experimental)
   - **IONOS**: German cloud provider with OpenAI-compatible API (experimental)
 - **Character Consistency**: Generate consistent characters across multiple images (perfect for children's book illustrations)
-- **Inpainting**: Fix specific areas of a generated image without regenerating the entire scene — via Vertex AI `imagen-capability` model
+- **Inpainting**: Fix specific areas of a generated image without regenerating the entire scene — via Vertex AI `imagen-capability` model. Supports optional subject reference images (`maskReferenceImages`) to guide *what* gets inserted into the masked area
 - **GDPR/DSGVO Compliance**: Built-in EU region support with automatic fallback
 - **Region Rotation**: Opt-in region rotation on quota errors (429) for Google Cloud — rotate through regions instead of retrying the same exhausted region
 - **Retry Logic**: Exponential backoff with jitter for transient errors (429, 408, 5xx, timeouts)
@@ -308,6 +309,72 @@ const duelScene = await service.generate({
 
 - Model must be `gemini-flash-image` (only model supporting character consistency)
 
+## Inpainting / Image Editing
+
+The `imagen-capability` model supports mask-based inpainting via Vertex AI. This is the **only** model that supports pixel-precise editing with a mask image.
+
+### Basic Inpainting
+
+```typescript
+const result = await service.generate({
+  model: 'imagen-capability',
+  prompt: 'Remove the extra arm and fill with matching forest background',
+  baseImage: { base64: originalImageBase64, mimeType: 'image/png' },
+  maskImage: { base64: maskBase64,          mimeType: 'image/png' },
+  editMode: 'inpainting-remove',  // default: 'inpainting-insert'
+  maskDilation: 0.02,             // optional, 0.0–1.0, default 0.01
+});
+```
+
+**How the mask works:**
+- White pixels = area the model will regenerate
+- Black pixels = area preserved exactly as-is
+- Mask must have identical dimensions to `baseImage`
+
+### Guided Inpainting with Subject References
+
+Use `maskReferenceImages` to provide a reference photo of the subject to insert — e.g. "place **this** character into the masked region":
+
+```typescript
+const result = await service.generate({
+  model: 'imagen-capability',
+  prompt: 'The character standing in a bright forest clearing, photorealistic',
+  baseImage:  { base64: sceneBase64,        mimeType: 'image/png' },
+  maskImage:  { base64: maskBase64,         mimeType: 'image/png' },
+  editMode: 'inpainting-insert',
+  maskReferenceImages: [
+    {
+      base64: characterRefBase64,
+      mimeType: 'image/png',
+      subjectType: 'person',   // 'person' | 'animal' | 'product' | 'default'
+    },
+  ],
+});
+```
+
+**Subject types:**
+
+| `subjectType` | Use case |
+|---------------|----------|
+| `'person'` | Human character |
+| `'animal'` | Animal or creature |
+| `'product'` | Object, item, product |
+| `'default'` | Let the model decide (safe fallback) |
+
+**Notes:**
+- `maskReferenceImages` only works with `editMode: 'inpainting-insert'`
+- Gemini models do **not** support mask-based inpainting or `maskReferenceImages`
+- `maskReferenceImages` without `baseImage` throws a validation error
+
+### Supported `editMode` Values
+
+| Value | Description |
+|-------|-------------|
+| `'inpainting-insert'` | Add or replace content in masked area (default) |
+| `'inpainting-remove'` | Remove content and fill with matching background |
+| `'background-swap'` | Replace background, preserve foreground |
+| `'outpainting'` | Extend image beyond its boundaries into the masked area |
+
 ## GDPR / Compliance
 
 ### Provider Compliance Overview
@@ -366,15 +433,30 @@ interface TTIRequest {
   n?: number;               // Number of images (default: 1)
   aspectRatio?: string;     // '1:1', '16:9', '4:3', etc.
 
-  // Character consistency
+  // Character consistency (Gemini models only)
   referenceImages?: TTIReferenceImage[];
   subjectDescription?: string;
 
+  // Inpainting / image editing (imagen-capability only)
+  baseImage?: TTIReferenceImage;   // Activates edit mode when set
+  maskImage?: TTIReferenceImage;   // Required when baseImage is set
+  maskDilation?: number;           // 0.0–1.0, default 0.01
+  editMode?: 'inpainting-insert' | 'inpainting-remove' | 'background-swap' | 'outpainting';
+  maskReferenceImages?: TTIMaskReferenceImage[];  // Subject refs for guided inpainting
+
   // Retry configuration
   retry?: boolean | RetryOptions;  // true (default), false, or custom config
 
   providerOptions?: Record<string, unknown>;
 }
+
+type TTISubjectType = 'person' | 'animal' | 'product' | 'default';
+
+interface TTIMaskReferenceImage {
+  base64: string;
+  mimeType?: string;
+  subjectType?: TTISubjectType;  // defaults to 'default'
+}
 ```
 
 ### TTIResponse

package/dist/middleware/services/tti/providers/base-tti-provider.js

@@ -216,6 +216,17 @@ class BaseTTIProvider {
                     throw new InvalidConfigError(this.providerName, 'maskDilation must be between 0.0 and 1.0');
                 }
             }
+            if (request.maskReferenceImages && request.maskReferenceImages.length > 0) {
+                for (let i = 0; i < request.maskReferenceImages.length; i++) {
+                    const ref = request.maskReferenceImages[i];
+                    if (!ref.base64 || ref.base64.trim().length === 0) {
+                        throw new InvalidConfigError(this.providerName, `maskReferenceImages[${i}] has empty base64 data`);
+                    }
+                }
+            }
+        }
+        if (request.maskReferenceImages && request.maskReferenceImages.length > 0 && !request.baseImage) {
+            throw new InvalidConfigError(this.providerName, 'maskReferenceImages requires baseImage to be set');
         }
         // If reference images are provided, validate them
         if (request.referenceImages && request.referenceImages.length > 0) {

package/dist/middleware/services/tti/providers/google-cloud-provider.d.ts

@@ -65,6 +65,7 @@ export declare class GoogleCloudTTIProvider extends BaseTTIProvider {
     private processImagenResponse;
     /** Maps our editMode values to the Vertex AI API constants */
     private static readonly EDIT_MODE_MAP;
+    private static readonly SUBJECT_TYPE_MAP;
     private editWithImagen;
     private generateWithGemini;
     private getGenaiClient;

package/dist/middleware/services/tti/providers/google-cloud-provider.js

@@ -586,6 +586,23 @@ class GoogleCloudTTIProvider extends base_tti_provider_1.BaseTTIProvider {
                     },
                 },
             ];
+            // Append optional subject reference images for guided inpainting
+            if (request.maskReferenceImages && request.maskReferenceImages.length > 0) {
+                for (const [i, ref] of request.maskReferenceImages.entries()) {
+                    const subjectType = GoogleCloudTTIProvider.SUBJECT_TYPE_MAP[ref.subjectType ?? 'default'] ??
+                        'SUBJECT_TYPE_DEFAULT';
+                    referenceImages.push({
+                        referenceType: 'REFERENCE_TYPE_SUBJECT',
+                        referenceId: 3 + i,
+                        referenceImage: {
+                            bytesBase64Encoded: ref.base64,
+                        },
+                        subjectImageConfig: {
+                            subjectType,
+                        },
+                    });
+                }
+            }
             const instanceValue = {
                 prompt: request.prompt,
                 referenceImages,
@@ -813,3 +830,9 @@ GoogleCloudTTIProvider.EDIT_MODE_MAP = {
     'background-swap': 'EDIT_MODE_BGSWAP',
     'outpainting': 'EDIT_MODE_OUTPAINT',
 };
+GoogleCloudTTIProvider.SUBJECT_TYPE_MAP = {
+    person: 'SUBJECT_TYPE_PERSON',
+    animal: 'SUBJECT_TYPE_ANIMAL',
+    product: 'SUBJECT_TYPE_PRODUCT',
+    default: 'SUBJECT_TYPE_DEFAULT',
+};

package/dist/middleware/types/index.d.ts

@@ -82,6 +82,30 @@ export interface TTIReferenceImage {
     /** MIME type of the image (e.g., 'image/png', 'image/jpeg') */
     mimeType?: string;
 }
+/**
+ * Subject type hint for mask reference images.
+ * Helps the model understand what kind of subject is shown in the reference image.
+ * - 'person'  — a human character
+ * - 'animal'  — an animal or creature
+ * - 'product' — an object, product, or item
+ * - 'default' — let the model decide (fallback)
+ */
+export type TTISubjectType = 'person' | 'animal' | 'product' | 'default';
+/**
+ * Reference image for mask-based inpainting (subject reference).
+ * Used with maskReferenceImages to guide what the model inserts into the masked area.
+ */
+export interface TTIMaskReferenceImage {
+    /** Base64-encoded image data of the subject to insert */
+    base64: string;
+    /** MIME type of the image (e.g., 'image/png', 'image/jpeg') */
+    mimeType?: string;
+    /**
+     * Subject type hint for the model.
+     * Defaults to 'default' if omitted.
+     */
+    subjectType?: TTISubjectType;
+}
 /**
  * Unified TTI generation request
  * Works for both simple text-to-image and character consistency
@@ -129,6 +153,14 @@ export interface TTIRequest {
      * - 'outpainting': extend image beyond its boundaries into the masked area
      */
     editMode?: 'inpainting-insert' | 'inpainting-remove' | 'background-swap' | 'outpainting';
+    /**
+     * Optional subject reference images for mask-based inpainting.
+     * Only valid when baseImage and maskImage are set.
+     * Each entry guides the model to insert a specific subject into the masked area
+     * (e.g., "place the character from this reference image into the mask").
+     * Only supported by the 'imagen-capability' model.
+     */
+    maskReferenceImages?: TTIMaskReferenceImage[];
     /** Additional provider-specific options */
     providerOptions?: Record<string, unknown>;
     /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loonylabs/tti-middleware",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "description": "Provider-agnostic Text-to-Image middleware with GDPR compliance. Supports Google Cloud (Imagen, Gemini), Eden AI, and IONOS.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",