@loonylabs/tti-middleware 1.8.0 → 1.10.0

package/README.md

@@ -44,6 +44,7 @@
   - **Eden AI**: Aggregator with access to OpenAI, Stability AI, Replicate (experimental)
   - **IONOS**: German cloud provider with OpenAI-compatible API (experimental)
 - **Character Consistency**: Generate consistent characters across multiple images (perfect for children's book illustrations)
+- **Inpainting**: Fix specific areas of a generated image without regenerating the entire scene — via Vertex AI `imagen-capability` model
 - **GDPR/DSGVO Compliance**: Built-in EU region support with automatic fallback
 - **Region Rotation**: Opt-in region rotation on quota errors (429) for Google Cloud — rotate through regions instead of retrying the same exhausted region
 - **Retry Logic**: Exponential backoff with jitter for transient errors (429, 408, 5xx, timeouts)

package/dist/middleware/services/tti/providers/base-tti-provider.d.ts

@@ -102,10 +102,20 @@ export declare abstract class BaseTTIProvider implements ITTIProvider {
      */
     protected sleep(ms: number): Promise<void>;
     /**
-     * Wrap an operation with a timeout. If the operation doesn't resolve
-     * within timeoutMs, the returned promise rejects with a timeout error.
-     * The original operation continues running (promises can't be cancelled),
-     * but its result is ignored.
+     * Wrap an operation with a timeout and optional grace period.
+     *
+     * Normal flow (graceMs = 0):
+     *   If the operation doesn't resolve within `timeoutMs`, the returned promise
+     *   rejects immediately with a timeout error.
+     *
+     * Grace period flow (graceMs > 0):
+     *   When `timeoutMs` fires, instead of rejecting immediately, a grace period
+     *   starts. If the operation resolves successfully within `graceMs`, the result
+     *   is used and no timeout error is thrown. Only if the grace period also expires
+     *   does the promise reject — with an error reflecting the total wait time.
+     *
+     * This prevents paying for (and discarding) a Vertex AI response that arrived
+     * slightly after the timeout threshold.
      */
     private withTimeout;
     /**

package/dist/middleware/services/tti/providers/base-tti-provider.js

@@ -196,6 +196,27 @@ class BaseTTIProvider {
         if (!request.prompt || request.prompt.trim().length === 0) {
             throw new InvalidConfigError(this.providerName, 'Prompt cannot be empty');
         }
+        // If baseImage is provided, validate inpainting requirements
+        if (request.baseImage) {
+            if (!request.baseImage.base64 || request.baseImage.base64.trim().length === 0) {
+                throw new InvalidConfigError(this.providerName, 'baseImage has empty base64 data');
+            }
+            const modelId = request.model || this.getDefaultModel();
+            if (!this.modelSupportsCapability(modelId, 'imageEditing')) {
+                throw new CapabilityNotSupportedError(this.providerName, 'imageEditing', modelId);
+            }
+            if (!request.maskImage) {
+                throw new InvalidConfigError(this.providerName, 'maskImage is required when baseImage is set');
+            }
+            if (!request.maskImage.base64 || request.maskImage.base64.trim().length === 0) {
+                throw new InvalidConfigError(this.providerName, 'maskImage has empty base64 data');
+            }
+            if (request.maskDilation !== undefined) {
+                if (request.maskDilation < 0 || request.maskDilation > 1) {
+                    throw new InvalidConfigError(this.providerName, 'maskDilation must be between 0.0 and 1.0');
+                }
+            }
+        }
         // If reference images are provided, validate them
         if (request.referenceImages && request.referenceImages.length > 0) {
             const modelId = request.model || this.getDefaultModel();
@@ -249,6 +270,7 @@ class BaseTTIProvider {
             jitter: retryOption.jitter ?? types_1.DEFAULT_RETRY_OPTIONS.jitter,
             timeoutMs: retryOption.timeoutMs ?? types_1.DEFAULT_RETRY_OPTIONS.timeoutMs,
             timeoutRetries: retryOption.timeoutRetries ?? types_1.DEFAULT_RETRY_OPTIONS.timeoutRetries,
+            graceMs: retryOption.graceMs ?? types_1.DEFAULT_RETRY_OPTIONS.graceMs,
         };
     }
     /**
@@ -274,25 +296,68 @@ class BaseTTIProvider {
         return new Promise((resolve) => setTimeout(resolve, ms));
     }
     /**
-     * Wrap an operation with a timeout. If the operation doesn't resolve
-     * within timeoutMs, the returned promise rejects with a timeout error.
-     * The original operation continues running (promises can't be cancelled),
-     * but its result is ignored.
+     * Wrap an operation with a timeout and optional grace period.
+     *
+     * Normal flow (graceMs = 0):
+     *   If the operation doesn't resolve within `timeoutMs`, the returned promise
+     *   rejects immediately with a timeout error.
+     *
+     * Grace period flow (graceMs > 0):
+     *   When `timeoutMs` fires, instead of rejecting immediately, a grace period
+     *   starts. If the operation resolves successfully within `graceMs`, the result
+     *   is used and no timeout error is thrown. Only if the grace period also expires
+     *   does the promise reject — with an error reflecting the total wait time.
+     *
+     * This prevents paying for (and discarding) a Vertex AI response that arrived
+     * slightly after the timeout threshold.
      */
-    withTimeout(operation, timeoutMs, operationName) {
+    withTimeout(operation, timeoutMs, graceMs, operationName) {
         return new Promise((resolve, reject) => {
-            const timer = setTimeout(() => {
-                reject(new Error(`timeout: ${operationName} did not complete within ${timeoutMs}ms`));
-            }, timeoutMs);
-            operation()
-                .then((result) => {
-                clearTimeout(timer);
+            let settled = false;
+            let inGracePeriod = false;
+            let mainTimerRef;
+            let graceTimerRef = null;
+            const operationPromise = operation();
+            // Handle operation resolution — can fire at any point, including during grace
+            operationPromise.then((result) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(mainTimerRef);
+                if (graceTimerRef)
+                    clearTimeout(graceTimerRef);
+                if (inGracePeriod) {
+                    this.log('info', `${operationName} completed during grace period`, { graceMs });
+                }
                 resolve(result);
-            })
-                .catch((error) => {
-                clearTimeout(timer);
+            }, (error) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(mainTimerRef);
+                if (graceTimerRef)
+                    clearTimeout(graceTimerRef);
                 reject(error);
             });
+            // Primary timeout
+            mainTimerRef = setTimeout(() => {
+                if (settled)
+                    return;
+                if (graceMs > 0) {
+                    inGracePeriod = true;
+                    this.log('warn', `${operationName} primary timeout after ${timeoutMs}ms, entering grace period (${graceMs}ms)`, { timeoutMs, graceMs });
+                    graceTimerRef = setTimeout(() => {
+                        if (settled)
+                            return;
+                        settled = true;
+                        reject(new Error(`timeout: ${operationName} did not complete within ${timeoutMs + graceMs}ms (including ${graceMs}ms grace period)`));
+                    }, graceMs);
+                }
+                else {
+                    settled = true;
+                    reject(new Error(`timeout: ${operationName} did not complete within ${timeoutMs}ms`));
+                }
+            }, timeoutMs);
         });
     }
     /**
@@ -336,6 +401,7 @@ class BaseTTIProvider {
             return operation();
         }
         const timeoutMs = retryConfig.timeoutMs || 0;
+        const graceMs = retryConfig.graceMs ?? 0;
         const maxTimeoutRetries = retryConfig.timeoutRetries ?? 2;
         let lastError = null;
         let generalRetryCount = 0;
@@ -354,7 +420,7 @@ class BaseTTIProvider {
                 });
                 // Wrap with timeout if configured
                 const result = timeoutMs > 0
-                    ? await this.withTimeout(operation, timeoutMs, operationName)
+                    ? await this.withTimeout(operation, timeoutMs, graceMs, operationName)
                     : await operation();
                 const duration = Date.now() - attemptStart;
                 this.log('info', `${operationName} completed in ${duration}ms`, {

package/dist/middleware/services/tti/providers/google-cloud-provider.d.ts

@@ -63,6 +63,9 @@ export declare class GoogleCloudTTIProvider extends BaseTTIProvider {
     private generateWithImagen;
     private getAiplatformClient;
     private processImagenResponse;
+    /** Maps our editMode values to the Vertex AI API constants */
+    private static readonly EDIT_MODE_MAP;
+    private editWithImagen;
     private generateWithGemini;
     private getGenaiClient;
     private buildCharacterConsistencyPrompt;

package/dist/middleware/services/tti/providers/google-cloud-provider.js

@@ -131,6 +131,27 @@ const GOOGLE_CLOUD_MODELS = [
         availableRegions: IMAGEN_4_REGIONS,
         pricingUrl: 'https://cloud.google.com/vertex-ai/generative-ai/pricing',
     },
+    // ── Imagen Capability model (Vertex AI editing / inpainting) ──
+    {
+        id: 'imagen-capability',
+        displayName: 'Imagen 3 Capability (Editing)',
+        capabilities: {
+            textToImage: false,
+            characterConsistency: false,
+            imageEditing: true,
+            maxImagesPerRequest: 4,
+        },
+        availableRegions: [
+            'europe-west1',
+            'europe-west2',
+            'europe-west3',
+            'europe-west4',
+            'europe-west9',
+            'us-central1',
+            'us-east4',
+        ],
+        pricingUrl: 'https://cloud.google.com/vertex-ai/generative-ai/pricing',
+    },
     // ── Gemini models (Vertex AI generateContent API) ──────────
     {
         id: 'gemini-flash-image',
@@ -185,6 +206,7 @@ const MODEL_ID_MAP = {
     'imagen-4': 'imagen-4.0-generate-001',
     'imagen-4-fast': 'imagen-4.0-fast-generate-001',
     'imagen-4-ultra': 'imagen-4.0-ultra-generate-001',
+    'imagen-capability': 'imagen-3.0-capability-001',
     'gemini-flash-image': 'gemini-2.5-flash-image',
     'gemini-pro-image': 'gemini-3-pro-image-preview',
     'gemini-flash-image-2': 'gemini-3.1-flash-image-preview',
@@ -296,10 +318,18 @@ class GoogleCloudTTIProvider extends base_tti_provider_1.BaseTTIProvider {
             hasReferenceImages: (0, base_tti_provider_1.hasReferenceImages)(request),
         });
         const isGeminiModel = GEMINI_API_MODELS.has(modelId);
-        const operationName = isGeminiModel ? 'Gemini API call' : 'Imagen API call';
+        const isEditRequest = !!request.baseImage;
+        const operationName = isEditRequest
+            ? 'Imagen edit API call'
+            : isGeminiModel
+                ? 'Gemini API call'
+                : 'Imagen API call';
         // Operation lambda reads currentRegion from closure
         const operation = () => {
-            if (isGeminiModel) {
+            if (isEditRequest) {
+                return this.editWithImagen(request, modelId, currentRegion);
+            }
+            else if (isGeminiModel) {
                 return this.generateWithGemini(request, modelId, currentRegion);
             }
             else {
@@ -528,6 +558,77 @@ class GoogleCloudTTIProvider extends base_tti_provider_1.BaseTTIProvider {
             usage,
         };
     }
+    async editWithImagen(request, modelId, region) {
+        const startTime = Date.now();
+        const internalModelId = MODEL_ID_MAP[modelId];
+        this.lastUsedRegion = region;
+        try {
+            const { client, helpers } = await this.getAiplatformClient(region);
+            const endpoint = `projects/${this.config.projectId}/locations/${region}/publishers/google/models/${internalModelId}`;
+            // Build referenceImages array: [RAW base image, MASK image]
+            const referenceImages = [
+                {
+                    referenceType: 'REFERENCE_TYPE_RAW',
+                    referenceId: 1,
+                    referenceImage: {
+                        bytesBase64Encoded: request.baseImage.base64,
+                    },
+                },
+                {
+                    referenceType: 'REFERENCE_TYPE_MASK',
+                    referenceId: 2,
+                    referenceImage: {
+                        bytesBase64Encoded: request.maskImage.base64,
+                    },
+                    maskImageConfig: {
+                        maskMode: 'MASK_MODE_USER_PROVIDED',
+                        dilation: request.maskDilation ?? 0.01,
+                    },
+                },
+            ];
+            const instanceValue = {
+                prompt: request.prompt,
+                referenceImages,
+            };
+            const instance = helpers.toValue(instanceValue);
+            // Map editMode to Vertex AI constant, default to inpainting-insert
+            const editModeKey = request.editMode ?? 'inpainting-insert';
+            const vertexEditMode = GoogleCloudTTIProvider.EDIT_MODE_MAP[editModeKey] ?? 'EDIT_MODE_INPAINT_INSERTION';
+            const parameterValue = {
+                editMode: vertexEditMode,
+                sampleCount: request.n || 1,
+                editConfig: {
+                    baseSteps: request.providerOptions?.baseSteps ?? 35,
+                },
+            };
+            const parameters = helpers.toValue(parameterValue);
+            this.log('info', 'Sending Imagen edit request to Vertex AI', {
+                endpoint,
+                editMode: vertexEditMode,
+                dilation: request.maskDilation ?? 0.01,
+            });
+            const [response] = await client.predict({
+                endpoint,
+                instances: [instance],
+                parameters,
+            });
+            const duration = Date.now() - startTime;
+            this.log('info', `Imagen edit response received in ${duration}ms`, {
+                duration,
+                hasPredictions: !!response.predictions?.length,
+            });
+            if (!response.predictions || response.predictions.length === 0) {
+                throw new base_tti_provider_1.GenerationFailedError(this.providerName, 'No images returned from Imagen edit API');
+            }
+            return this.processImagenResponse(response.predictions, helpers, modelId, duration);
+        }
+        catch (error) {
+            if (error instanceof base_tti_provider_1.InvalidConfigError || error instanceof base_tti_provider_1.GenerationFailedError) {
+                throw error;
+            }
+            throw this.handleError(error, 'during Imagen edit API call');
+        }
+    }
     // ============================================================
     // PRIVATE: GEMINI IMAGE IMPLEMENTATION
     // ============================================================
@@ -702,3 +803,13 @@ IMPORTANT: Maintain exact visual consistency with the subject in the reference -
     }
 }
 exports.GoogleCloudTTIProvider = GoogleCloudTTIProvider;
+// ============================================================
+// PRIVATE: IMAGEN EDITING / INPAINTING IMPLEMENTATION
+// ============================================================
+/** Maps our editMode values to the Vertex AI API constants */
+GoogleCloudTTIProvider.EDIT_MODE_MAP = {
+    'inpainting-insert': 'EDIT_MODE_INPAINT_INSERTION',
+    'inpainting-remove': 'EDIT_MODE_INPAINT_REMOVAL',
+    'background-swap': 'EDIT_MODE_BGSWAP',
+    'outpainting': 'EDIT_MODE_OUTPAINT',
+};

package/dist/middleware/types/index.d.ts

@@ -105,6 +105,30 @@ export interface TTIRequest {
      * Required when using referenceImages (e.g., "cute cartoon bear with red hat")
      */
     subjectDescription?: string;
+    /**
+     * Base image to edit. When present, activates "edit mode" instead of text-to-image generation.
+     * Requires maskImage and a model that supports imageEditing capability.
+     */
+    baseImage?: TTIReferenceImage;
+    /**
+     * Mask image for inpainting. White pixels = regenerate, black pixels = preserve.
+     * Must have identical dimensions to baseImage.
+     * Required when baseImage is set.
+     */
+    maskImage?: TTIReferenceImage;
+    /**
+     * Mask dilation: expands the mask boundary to smooth hard edges (0.0–1.0, default 0.01).
+     * Useful when hand-drawn masks have jagged edges.
+     */
+    maskDilation?: number;
+    /**
+     * Edit operation to perform on the masked region.
+     * - 'inpainting-insert': add or replace content in the masked area (default)
+     * - 'inpainting-remove': remove content and fill with matching background
+     * - 'background-swap': replace background while preserving foreground
+     * - 'outpainting': extend image beyond its boundaries into the masked area
+     */
+    editMode?: 'inpainting-insert' | 'inpainting-remove' | 'background-swap' | 'outpainting';
     /** Additional provider-specific options */
     providerOptions?: Record<string, unknown>;
     /**
@@ -235,6 +259,23 @@ export interface RetryOptions {
      * long waits, while still allowing many retries for quota errors.
      */
     timeoutRetries?: number;
+    /**
+     * Grace period in milliseconds after a timeout before abandoning the attempt (default: 0).
+     *
+     * When `timeoutMs` fires, instead of immediately failing and starting a new retry,
+     * the middleware waits an additional `graceMs`. If the in-flight request resolves
+     * successfully within this window, the result is used and no retry is needed.
+     *
+     * This prevents paying for (and discarding) a successful response that arrived
+     * slightly after the timeout threshold — which is common under quota pressure when
+     * Vertex AI eventually returns a valid result after a long backoff.
+     *
+     * Example: `timeoutMs: 210000, graceMs: 60000`
+     * → waits up to 210s normally, then up to 60s more to capture a late success.
+     *
+     * Set to 0 (default) to disable — timeout retries fire immediately as before.
+     */
+    graceMs?: number;
     /**
      * @deprecated Use `backoffMultiplier` instead. Will be removed in v2.0.
      * When true, equivalent to backoffMultiplier of 1.0 with linear scaling (delayMs * attempt).

package/dist/middleware/types/index.js

@@ -39,4 +39,5 @@ exports.DEFAULT_RETRY_OPTIONS = {
     jitter: true,
     timeoutMs: 45000,
     timeoutRetries: 2,
+    graceMs: 0,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loonylabs/tti-middleware",
-  "version": "1.8.0",
+  "version": "1.10.0",
   "description": "Provider-agnostic Text-to-Image middleware with GDPR compliance. Supports Google Cloud (Imagen, Gemini), Eden AI, and IONOS.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",