@loonylabs/tti-middleware 1.10.0 → 1.12.0

package/README.md

@@ -24,6 +24,7 @@
 - [Configuration](#%EF%B8%8F-configuration)
 - [Providers & Models](#-providers--models)
 - [Character Consistency](#-character-consistency)
+- [Inpainting / Image Editing](#inpainting--image-editing)
 - [GDPR / Compliance](#-gdpr--compliance)
 - [API Reference](#-api-reference)
 - [Advanced Features](#-advanced-features)
@@ -44,7 +45,7 @@
   - **Eden AI**: Aggregator with access to OpenAI, Stability AI, Replicate (experimental)
   - **IONOS**: German cloud provider with OpenAI-compatible API (experimental)
 - **Character Consistency**: Generate consistent characters across multiple images (perfect for children's book illustrations)
-- **Inpainting**: Fix specific areas of a generated image without regenerating the entire scene — via Vertex AI `imagen-capability` model
+- **Inpainting**: Fix specific areas of a generated image without regenerating the entire scene — via Vertex AI `imagen-capability` model.
 - **GDPR/DSGVO Compliance**: Built-in EU region support with automatic fallback
 - **Region Rotation**: Opt-in region rotation on quota errors (429) for Google Cloud — rotate through regions instead of retrying the same exhausted region
 - **Retry Logic**: Exponential backoff with jitter for transient errors (429, 408, 5xx, timeouts)
@@ -308,6 +309,37 @@ const duelScene = await service.generate({
 
 - Model must be `gemini-flash-image` (only model supporting character consistency)
 
+## Inpainting / Image Editing
+
+The `imagen-capability` model supports mask-based inpainting via Vertex AI. This is the **only** model that supports pixel-precise editing with a mask image.
+
+### Basic Inpainting
+
+```typescript
+const result = await service.generate({
+  model: 'imagen-capability',
+  prompt: 'Remove the extra arm and fill with matching forest background',
+  baseImage: { base64: originalImageBase64, mimeType: 'image/png' },
+  maskImage: { base64: maskBase64,          mimeType: 'image/png' },
+  editMode: 'inpainting-remove',  // default: 'inpainting-insert'
+  maskDilation: 0.02,             // optional, 0.0–1.0, default 0.01
+});
+```
+
+**How the mask works:**
+- White pixels = area the model will regenerate
+- Black pixels = area preserved exactly as-is
+- Mask must have identical dimensions to `baseImage`
+
+### Supported `editMode` Values
+
+| Value | Description |
+|-------|-------------|
+| `'inpainting-insert'` | Add or replace content in masked area (default) |
+| `'inpainting-remove'` | Remove content and fill with matching background |
+| `'background-swap'` | Replace background, preserve foreground |
+| `'outpainting'` | Extend image beyond its boundaries into the masked area |
+
 ## GDPR / Compliance
 
 ### Provider Compliance Overview
@@ -366,10 +398,16 @@ interface TTIRequest {
   n?: number;               // Number of images (default: 1)
   aspectRatio?: string;     // '1:1', '16:9', '4:3', etc.
 
-  // Character consistency
+  // Character consistency (Gemini models only)
   referenceImages?: TTIReferenceImage[];
   subjectDescription?: string;
 
+  // Inpainting / image editing (imagen-capability only)
+  baseImage?: TTIReferenceImage;   // Activates edit mode when set
+  maskImage?: TTIReferenceImage;   // Required when baseImage is set
+  maskDilation?: number;           // 0.0–1.0, default 0.01
+  editMode?: 'inpainting-insert' | 'inpainting-remove' | 'background-swap' | 'outpainting';
+
   // Retry configuration
   retry?: boolean | RetryOptions;  // true (default), false, or custom config
 

package/dist/middleware/services/tti/providers/google-cloud-provider.d.ts

@@ -65,6 +65,7 @@ export declare class GoogleCloudTTIProvider extends BaseTTIProvider {
     private processImagenResponse;
     /** Maps our editMode values to the Vertex AI API constants */
     private static readonly EDIT_MODE_MAP;
+    private static readonly SUBJECT_TYPE_MAP;
     private editWithImagen;
     private generateWithGemini;
     private getGenaiClient;

package/dist/middleware/services/tti/providers/google-cloud-provider.js

@@ -565,18 +565,31 @@ class GoogleCloudTTIProvider extends base_tti_provider_1.BaseTTIProvider {
         try {
             const { client, helpers } = await this.getAiplatformClient(region);
             const endpoint = `projects/${this.config.projectId}/locations/${region}/publishers/google/models/${internalModelId}`;
-            // Build referenceImages array: [RAW base image, MASK image]
+            // Build referenceImages array.
+            //
+            // Vertex AI uses sequential, distinct referenceId values for each entry.
+            // RAW and MASK have separate IDs — the API pairs them by type/order, not by shared ID.
+            // SUBJECT gets the next ID after MASK and MUST be cited in the prompt as [N].
+            //
+            // Correct layout (confirmed by official Vertex AI docs):
+            //   REFERENCE_TYPE_RAW    referenceId: 1
+            //   REFERENCE_TYPE_MASK   referenceId: 2
+            //   REFERENCE_TYPE_SUBJECT referenceId: 3  ← referenced in prompt as [3]
+            //
+            // See: https://cloud.google.com/vertex-ai/generative-ai/docs/image/edit-insert-objects
+            const RAW_REFERENCE_ID = 1;
+            const MASK_REFERENCE_ID = 2;
             const referenceImages = [
                 {
                     referenceType: 'REFERENCE_TYPE_RAW',
-                    referenceId: 1,
+                    referenceId: RAW_REFERENCE_ID,
                     referenceImage: {
                         bytesBase64Encoded: request.baseImage.base64,
                     },
                 },
                 {
                     referenceType: 'REFERENCE_TYPE_MASK',
-                    referenceId: 2,
+                    referenceId: MASK_REFERENCE_ID,
                     referenceImage: {
                         bytesBase64Encoded: request.maskImage.base64,
                     },
@@ -813,3 +826,10 @@ GoogleCloudTTIProvider.EDIT_MODE_MAP = {
     'background-swap': 'EDIT_MODE_BGSWAP',
     'outpainting': 'EDIT_MODE_OUTPAINT',
 };
+GoogleCloudTTIProvider.SUBJECT_TYPE_MAP = {
+    person: 'SUBJECT_TYPE_PERSON',
+    // Vertex AI API uses ANIMAL_COMPANION, not ANIMAL
+    animal: 'SUBJECT_TYPE_ANIMAL_COMPANION',
+    product: 'SUBJECT_TYPE_PRODUCT',
+    default: 'SUBJECT_TYPE_DEFAULT',
+};

package/package.json

@@ -1,90 +1,90 @@
-{
-  "name": "@loonylabs/tti-middleware",
-  "version": "1.10.0",
-  "description": "Provider-agnostic Text-to-Image middleware with GDPR compliance. Supports Google Cloud (Imagen, Gemini), Eden AI, and IONOS.",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
-  "files": [
-    "dist",
-    "LICENSE",
-    "README.md",
-    ".env.example"
-  ],
-  "scripts": {
-    "build": "tsc",
-    "test": "npm run test:unit",
-    "test:unit": "jest --testPathPattern=tests/unit",
-    "test:unit:watch": "jest --testPathPattern=tests/unit --watch",
-    "test:unit:coverage": "jest --testPathPattern=tests/unit --coverage",
-    "test:integration": "cross-env TTI_INTEGRATION_TESTS=true jest --testPathPattern=tests/integration",
-    "test:ci": "jest --runInBand --ci --coverage --testPathPattern=tests/unit",
-    "test:live": "TTI_INTEGRATION_TESTS=true jest tests/integration/*.integration.test.ts --testTimeout=120000",
-    "test:manual": "ts-node scripts/manual-test-edenai.ts",
-    "test:manual:google-cloud": "ts-node scripts/manual-test-google-cloud.ts",
-    "lint": "eslint src/**/*.ts",
-    "format": "prettier --write \"src/**/*.ts\"",
-    "clean": "node -e \"const fs=require('fs');if(fs.existsSync('dist'))fs.rmSync('dist',{recursive:true})\"",
-    "prepare": "npm run build",
-    "prepublishOnly": "npm run clean && npm run build && npm run test"
-  },
-  "keywords": [
-    "tti",
-    "text-to-image",
-    "image-generation",
-    "ai",
-    "vertex-ai",
-    "google-cloud",
-    "gemini",
-    "imagen",
-    "middleware",
-    "provider-agnostic",
-    "typescript",
-    "gdpr",
-    "dsgvo",
-    "character-consistency"
-  ],
-  "author": "loonylabs-dev",
-  "license": "MIT",
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/loonylabs-dev/tti-middleware.git"
-  },
-  "bugs": {
-    "url": "https://github.com/loonylabs-dev/tti-middleware/issues"
-  },
-  "homepage": "https://github.com/loonylabs-dev/tti-middleware#readme",
-  "peerDependencies": {
-    "@google-cloud/aiplatform": ">=3.0.0",
-    "@google/genai": ">=1.40.0"
-  },
-  "peerDependenciesMeta": {
-    "@google-cloud/aiplatform": {
-      "optional": true
-    },
-    "@google/genai": {
-      "optional": true
-    }
-  },
-  "devDependencies": {
-    "@google-cloud/aiplatform": "^3.29.0",
-    "@google/genai": "^1.40.0",
-    "@types/jest": "^29.5.8",
-    "@types/node": "^20.10.0",
-    "@typescript-eslint/eslint-plugin": "^6.13.0",
-    "@typescript-eslint/parser": "^6.13.0",
-    "cross-env": "^10.1.0",
-    "dotenv": "^17.2.3",
-    "eslint": "^8.54.0",
-    "jest": "^29.7.0",
-    "prettier": "^3.1.0",
-    "ts-jest": "^29.1.1",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.3.2"
-  },
-  "engines": {
-    "node": ">=18.0.0"
-  },
-  "publishConfig": {
-    "access": "public"
-  }
-}
+{
+  "name": "@loonylabs/tti-middleware",
+  "version": "1.12.0",
+  "description": "Provider-agnostic Text-to-Image middleware with GDPR compliance. Supports Google Cloud (Imagen, Gemini), Eden AI, and IONOS.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md",
+    ".env.example"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "test": "npm run test:unit",
+    "test:unit": "jest --testPathPattern=tests/unit",
+    "test:unit:watch": "jest --testPathPattern=tests/unit --watch",
+    "test:unit:coverage": "jest --testPathPattern=tests/unit --coverage",
+    "test:integration": "cross-env TTI_INTEGRATION_TESTS=true jest --testPathPattern=tests/integration",
+    "test:ci": "jest --runInBand --ci --coverage --testPathPattern=tests/unit",
+    "test:live": "TTI_INTEGRATION_TESTS=true jest tests/integration/*.integration.test.ts --testTimeout=120000",
+    "test:manual": "ts-node scripts/manual-test-edenai.ts",
+    "test:manual:google-cloud": "ts-node scripts/manual-test-google-cloud.ts",
+    "lint": "eslint src/**/*.ts",
+    "format": "prettier --write \"src/**/*.ts\"",
+    "clean": "node -e \"const fs=require('fs');if(fs.existsSync('dist'))fs.rmSync('dist',{recursive:true})\"",
+    "prepare": "npm run build",
+    "prepublishOnly": "npm run clean && npm run build && npm run test"
+  },
+  "keywords": [
+    "tti",
+    "text-to-image",
+    "image-generation",
+    "ai",
+    "vertex-ai",
+    "google-cloud",
+    "gemini",
+    "imagen",
+    "middleware",
+    "provider-agnostic",
+    "typescript",
+    "gdpr",
+    "dsgvo",
+    "character-consistency"
+  ],
+  "author": "loonylabs-dev",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/loonylabs-dev/tti-middleware.git"
+  },
+  "bugs": {
+    "url": "https://github.com/loonylabs-dev/tti-middleware/issues"
+  },
+  "homepage": "https://github.com/loonylabs-dev/tti-middleware#readme",
+  "peerDependencies": {
+    "@google-cloud/aiplatform": ">=3.0.0",
+    "@google/genai": ">=1.40.0"
+  },
+  "peerDependenciesMeta": {
+    "@google-cloud/aiplatform": {
+      "optional": true
+    },
+    "@google/genai": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@google-cloud/aiplatform": "^3.29.0",
+    "@google/genai": "^1.40.0",
+    "@types/jest": "^29.5.8",
+    "@types/node": "^20.10.0",
+    "@typescript-eslint/eslint-plugin": "^6.13.0",
+    "@typescript-eslint/parser": "^6.13.0",
+    "cross-env": "^10.1.0",
+    "dotenv": "^17.2.3",
+    "eslint": "^8.54.0",
+    "jest": "^29.7.0",
+    "prettier": "^3.1.0",
+    "ts-jest": "^29.1.1",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.3.2"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}