@xuda.io/ai_module 1.1.5607 → 1.1.5608

package/index.mjs

@@ -7,6 +7,10 @@ import { pathToFileURL } from 'node:url';
 import { spawn } from 'node:child_process';
 
 import sharp from 'sharp';
+import { createRequire } from 'node:module';
+import { removeBackground as imglyRemoveBackground } from '@imgly/background-removal-node';
+
+const imglyPublicPath = pathToFileURL(path.dirname(createRequire(import.meta.url).resolve('@imgly/background-removal-node')) + path.sep).href;
 
 import puppeteer from 'puppeteer-extra';
 import StealthPlugin from 'puppeteer-extra-plugin-stealth';
@@ -7265,7 +7269,7 @@ export const get_profile_avatar = async function (profile_picture, uid, prompt,
   const tempOutputPath = path.join(tempDir, `output_${uniqueId}.png`);
   const { is_user } = metadata;
   let filename = email || _id || name;
-  const detect_real_person_in_image = async function (base64) {
+  const inspect_person_in_image = async function (base64) {
     const ret = await submit_chat_gpt_prompt({
       uid,
       model: _conf.default_ai_model,
@@ -7275,7 +7279,7 @@ export const get_profile_avatar = async function (profile_picture, uid, prompt,
           content: [
             {
               type: 'input_text',
-              text: `detect if a postrate of real person shown in the picture `,
+              text: `Inspect this profile image. Return whether it contains a real human portrait suitable for an authentic account avatar. Mark it unsuitable if the face is not visible, side-facing, heavily cropped, very blurry, tiny, or not a real person.`,
             },
             {
               type: 'input_image',
@@ -7285,14 +7289,66 @@ export const get_profile_avatar = async function (profile_picture, uid, prompt,
         },
       ],
       response_format: z.object({
-        is_real_person_in_picture: z.boolean().describe('if a postrate of real person in the picture return true otherwise return false'),
+        is_real_person_in_picture: z.boolean().describe('true if a real human portrait is visible'),
+        is_front_facing: z.boolean().describe('true if the face is mostly front-facing or only slightly angled'),
+        is_face_too_cropped: z.boolean().describe('true if important face/head parts are cut off'),
+        is_too_blurry: z.boolean().describe('true if the face is too blurry for an authentic avatar'),
+        is_too_small: z.boolean().describe('true if the portrait is too small or low-detail'),
+        needs_restoration: z.boolean().describe('true if the photo is old, scanned, scratched, faded, grainy, low-detail, or otherwise degraded such that face-restoration would noticeably improve it'),
       }),
       metadata: { _id, func: 'detect_real_person_in_image' },
       account_profile_info,
     });
 
     const res = JSON5.parse(ret.data);
-    return res?.is_real_person_in_picture;
+    return {
+      is_real_person_in_picture: Boolean(res?.is_real_person_in_picture),
+      is_front_facing: Boolean(res?.is_front_facing),
+      is_face_too_cropped: Boolean(res?.is_face_too_cropped),
+      is_too_blurry: Boolean(res?.is_too_blurry),
+      is_too_small: Boolean(res?.is_too_small),
+      needs_restoration: Boolean(res?.needs_restoration),
+    };
+  };
+
+  const detect_face_box = async function (base64) {
+    try {
+      const ret = await submit_chat_gpt_prompt({
+        uid,
+        model: _conf.default_ai_model,
+        prompt: [
+          {
+            role: 'user',
+            content: [
+              {
+                type: 'input_text',
+                text: `Return the bounding box of the visible face in this portrait, from the top of the head (including hair) to just below the chin, ear to ear. Use normalized coordinates in [0,1] where (0,0) is the top-left corner and (1,1) is the bottom-right corner of the image.`,
+              },
+              { type: 'input_image', image_url: `data:image/png;base64,${base64}` },
+            ],
+          },
+        ],
+        response_format: z.object({
+          face_top: z.number().min(0).max(1).describe('Top edge of face bbox, normalized'),
+          face_left: z.number().min(0).max(1).describe('Left edge of face bbox, normalized'),
+          face_width: z.number().min(0).max(1).describe('Width of face bbox, normalized'),
+          face_height: z.number().min(0).max(1).describe('Height of face bbox, normalized'),
+        }),
+        metadata: { _id, func: 'detect_face_box' },
+        account_profile_info,
+      });
+      const res = JSON5.parse(ret.data);
+      if (!res || typeof res.face_height !== 'number' || res.face_height <= 0) return null;
+      return {
+        face_top: Number(res.face_top) || 0,
+        face_left: Number(res.face_left) || 0,
+        face_width: Number(res.face_width) || 0,
+        face_height: Number(res.face_height) || 0,
+      };
+    } catch (err) {
+      console.error('detect_face_box failed:', err.message);
+      return null;
+    }
   };
 
   try {
@@ -7482,57 +7538,48 @@ export const get_profile_avatar = async function (profile_picture, uid, prompt,
         }
       } else {
         /// PERSONAL
-        const create_avatar_from_pic = async function () {
-          const { bio, country, mainCategory, subCategory } = metadata;
-
-          const prompt = `Completely remove the background, enhance and color-correct the subject, and return only a centered head-and-shoulders portrait facing the camera. Add extra top margin, include no labels or text, and export the result as a clean transparent PNG.
-             Ensure the subject is large enough to fill most of the frame. Use the supplied name, country, and context to guide the character’s appearance.
-            Context: ${JSON.stringify({ name, bio, country, mainCategory, subCategory })}`;
-
-          // const { first_name, last_name, bio, country, mainCategory, subCategory } = account_obj.account_info;
-          // const model = 'gpt-image-1-mini';
-          const model = 'chatgpt-image-latest';
-          let ai_avatar_response;
-          try {
-            ai_avatar_response = await client.images.edit({
-              model,
-              image: image_blob_ret.image_blob,
+        const source_quality = await inspectAvatarSourceQuality(imageBase64);
 
-              prompt,
-              background: 'transparent',
-            });
-            report_ai_status(model);
-          } catch (err) {
-            report_ai_status(model, err);
-            throw err;
-          }
-          account_msa.record_ai_usage(uid, ai_avatar_response.usage.input_tokens, ai_avatar_response.usage.output_tokens, 'profile avatar', prompt, model, { profile_picture }, account_profile_info);
-          imageBase64 = ai_avatar_response.data[0].b64_json;
-          avatar_source = 'ai profile';
-        };
         const create_fictional_avatar = async function () {
           // const { name, bio, country, mainCategory, subCategory } = account_obj.account_info;
 
-          const prompt = `analyze the the person info :  ${JSON.stringify(metadata)} 
+          const prompt = `analyze the the person info :  ${JSON.stringify(metadata)}
             and create realistic fictional profile picture transparent background ,person in the center of the picture return only a centered head-and-shoulders portrait facing the camera. include no labels or text add top margin the person should cover the whole picture`;
 
           imageBase64 = await create_image(uid, prompt, undefined, undefined, 1, undefined, undefined, { url: profile_picture }, account_profile_info);
           avatar_source = 'fictional';
         };
-
         // personal account
-        const is_real_person_in_picture = await detect_real_person_in_image(imageBase64);
-        if (is_real_person_in_picture) {
-          if (image_blob_ret.is_transparent) {
-            // keep the original
-            imageBase64 = Buffer.from(await image_blob_ret.image_blob.arrayBuffer()).toString('base64');
-            avatar_source = 'authentic profile';
-          } else {
-            // create avatar from profile picture
-            await create_avatar_from_pic();
+        let person_inspection = await inspect_person_in_image(imageBase64);
+        const wants_restoration = person_inspection.needs_restoration || person_inspection.is_too_blurry;
+
+        if (wants_restoration && person_inspection.is_real_person_in_picture && !person_inspection.is_too_small) {
+          console.log('Source photo flagged for restoration, calling gpt-image-1...');
+          try {
+            const restored_base64 = await restoreFaceWithOpenAI(imageBase64, { uid, account_profile_info, profile_picture });
+            const verify = await inspect_person_in_image(restored_base64);
+            if (verify.is_real_person_in_picture && !verify.is_too_blurry && !verify.is_face_too_cropped) {
+              imageBase64 = restored_base64;
+              person_inspection = verify;
+              console.log('Face restoration applied');
+            } else {
+              console.log('Restoration verify failed, using original photo');
+            }
+          } catch (err) {
+            console.error('Face restoration failed, using original photo:', err.message);
           }
+        }
+
+        const can_create_authentic_avatar = source_quality.is_usable && person_inspection.is_real_person_in_picture && !person_inspection.is_too_blurry && !person_inspection.is_too_small;
+
+        if (can_create_authentic_avatar) {
+          const face_box = await detect_face_box(imageBase64);
+          imageBase64 = await normalizeAuthenticProfileAvatar(imageBase64, {
+            remove_background: !image_blob_ret.is_transparent,
+            face_box,
+          });
+          avatar_source = 'authentic profile';
         } else {
-          // create avatar from profile picture
           await create_fictional_avatar();
         }
       }
@@ -8618,6 +8665,185 @@ function normalizeFilename(name) {
     .replace(/^_+|_+$/g, ''); // trim leading/trailing underscores
 }
 
+async function inspectAvatarSourceQuality(base64Image) {
+  const inputBuffer = Buffer.from(base64Image, 'base64');
+  const metadata = await sharp(inputBuffer).metadata();
+  const width = metadata.width || 0;
+  const height = metadata.height || 0;
+  const shortest_side = Math.min(width, height);
+  const longest_side = Math.max(width, height);
+  const aspect_ratio = shortest_side ? longest_side / shortest_side : 0;
+
+  return {
+    width,
+    height,
+    shortest_side,
+    aspect_ratio,
+    has_alpha: Boolean(metadata.hasAlpha),
+    is_usable: shortest_side >= 180 && aspect_ratio <= 3,
+  };
+}
+
+async function normalizeAuthenticProfileAvatar(base64Image, options = {}) {
+  const { remove_background = true, face_box = null } = options;
+  const inputBuffer = Buffer.from(base64Image, 'base64');
+  const orientedBuffer = await sharp(inputBuffer).rotate().png({ force: true }).toBuffer();
+  const orientedMeta = await sharp(orientedBuffer).metadata();
+  const segmentedBuffer = remove_background ? await removePortraitBackground(orientedBuffer) : orientedBuffer;
+
+  const passportBuffer = face_box ? await cropToPassportFrame(segmentedBuffer, face_box, orientedMeta) : await cropToOpaqueBounds(segmentedBuffer);
+
+  const subjectBuffer = await sharp(passportBuffer).modulate({ brightness: 1.02, saturation: 1.04 }).sharpen({ sigma: 0.35, m1: 0.4, m2: 0.2 }).png({ quality: 98, compressionLevel: 8, force: true }).toBuffer();
+
+  return subjectBuffer.toString('base64');
+}
+
+async function cropToPassportFrame(segmentedBuffer, face_box, sourceMeta) {
+  const segmentedMeta = await sharp(segmentedBuffer).metadata();
+  const sourceW = segmentedMeta.width || sourceMeta.width || 0;
+  const sourceH = segmentedMeta.height || sourceMeta.height || 0;
+  if (!sourceW || !sourceH) return cropToOpaqueBounds(segmentedBuffer);
+
+  const faceCenterX = (face_box.face_left + face_box.face_width / 2) * sourceW;
+  const faceCenterY = (face_box.face_top + face_box.face_height / 2) * sourceH;
+  const faceHeightPx = face_box.face_height * sourceH;
+  if (!(faceHeightPx > 4)) return cropToOpaqueBounds(segmentedBuffer);
+
+  const faceFraction = 0.62;
+  const frameSize = Math.max(8, Math.round(faceHeightPx / faceFraction));
+  const verticalAnchor = 0.42;
+  const cropLeft = Math.round(faceCenterX - frameSize / 2);
+  const cropTop = Math.round(faceCenterY - frameSize * verticalAnchor);
+
+  const padLeft = Math.max(0, -cropLeft);
+  const padTop = Math.max(0, -cropTop);
+  const padRight = Math.max(0, cropLeft + frameSize - sourceW);
+  const padBottom = Math.max(0, cropTop + frameSize - sourceH);
+
+  let workingBuffer = segmentedBuffer;
+  if (padLeft || padTop || padRight || padBottom) {
+    workingBuffer = await sharp(segmentedBuffer)
+      .ensureAlpha()
+      .extend({
+        left: padLeft,
+        top: padTop,
+        right: padRight,
+        bottom: padBottom,
+        background: { r: 0, g: 0, b: 0, alpha: 0 },
+      })
+      .png({ force: true })
+      .toBuffer();
+  }
+
+  const workingMeta = await sharp(workingBuffer).metadata();
+  const wW = workingMeta.width || 0;
+  const wH = workingMeta.height || 0;
+
+  const wantLeft = cropLeft + padLeft;
+  const wantTop = cropTop + padTop;
+  const extractLeft = Math.max(0, Math.min(Math.max(0, wW - 1), wantLeft));
+  const extractTop = Math.max(0, Math.min(Math.max(0, wH - 1), wantTop));
+  const extractWidth = Math.max(1, Math.min(wW - extractLeft, frameSize));
+  const extractHeight = Math.max(1, Math.min(wH - extractTop, frameSize));
+
+  if (extractWidth <= 0 || extractHeight <= 0) {
+    console.warn('cropToPassportFrame: degenerate extract region, falling back', { sourceW, sourceH, frameSize, cropLeft, cropTop, wW, wH });
+    return cropToOpaqueBounds(segmentedBuffer);
+  }
+
+  const framed = await sharp(workingBuffer).extract({ left: extractLeft, top: extractTop, width: extractWidth, height: extractHeight }).png({ force: true }).toBuffer();
+
+  return sharp(framed)
+    .resize(1024, 1024, { fit: 'contain', background: { r: 0, g: 0, b: 0, alpha: 0 } })
+    .png({ force: true })
+    .toBuffer();
+}
+
+async function removePortraitBackground(inputBuffer) {
+  const orientedBuffer = await sharp(inputBuffer).rotate().png({ force: true }).toBuffer();
+  const inputBlob = new Blob([orientedBuffer], { type: 'image/png' });
+
+  const outputBlob = await imglyRemoveBackground(inputBlob, {
+    model: 'medium',
+    publicPath: imglyPublicPath,
+    output: { format: 'image/png' },
+  });
+
+  return Buffer.from(await outputBlob.arrayBuffer());
+}
+
+async function cropToOpaqueBounds(inputBuffer) {
+  const image = sharp(inputBuffer).ensureAlpha();
+  const { data, info } = await image.raw().toBuffer({ resolveWithObject: true });
+  const { width, height, channels } = info;
+  const alphaThreshold = 8;
+
+  let minX = width;
+  let minY = height;
+  let maxX = -1;
+  let maxY = -1;
+
+  for (let y = 0; y < height; y++) {
+    for (let x = 0; x < width; x++) {
+      const alpha = data[(y * width + x) * channels + (channels - 1)];
+      if (alpha > alphaThreshold) {
+        if (x < minX) minX = x;
+        if (x > maxX) maxX = x;
+        if (y < minY) minY = y;
+        if (y > maxY) maxY = y;
+      }
+    }
+  }
+
+  if (maxX < 0 || maxY < 0) {
+    return sharp(inputBuffer).png({ force: true }).toBuffer();
+  }
+
+  const padX = Math.round((maxX - minX + 1) * 0.04);
+  const padY = Math.round((maxY - minY + 1) * 0.04);
+  const cropLeft = Math.max(0, minX - padX);
+  const cropTop = Math.max(0, minY - padY);
+  const cropWidth = Math.min(width - cropLeft, maxX - minX + 1 + padX * 2);
+  const cropHeight = Math.min(height - cropTop, maxY - minY + 1 + padY * 2);
+
+  return sharp(inputBuffer).ensureAlpha().extract({ left: cropLeft, top: cropTop, width: cropWidth, height: cropHeight }).png({ force: true }).toBuffer();
+}
+
+async function restoreFaceWithOpenAI(base64Image, ctx = {}) {
+  const { uid, account_profile_info, profile_picture } = ctx;
+  const inputBuffer = Buffer.from(base64Image, 'base64');
+  const oriented = await sharp(inputBuffer).rotate().png({ force: true }).toBuffer();
+  const meta = await sharp(oriented).metadata();
+  const maxDim = Math.max(meta.width || 0, meta.height || 0);
+  const sourceBuffer = maxDim > 1536 ? await sharp(oriented).resize({ width: 1536, height: 1536, fit: 'inside' }).png({ force: true }).toBuffer() : oriented;
+
+  const imageFile = await toFile(sourceBuffer, 'portrait.png', { type: 'image/png' });
+  const model = 'gpt-image-1';
+  let response;
+  try {
+    response = await client.images.edit({
+      model,
+      image: imageFile,
+      input_fidelity: 'high',
+      quality: 'high',
+      size: 'auto',
+      prompt:
+        'Restore this portrait photograph. Remove scratches, dust, grain, noise, faded color, and any age-related damage. Recover natural skin texture, sharpen face details, and balance lighting and color. Preserve the exact same person completely unchanged: same face shape, same eye shape and color, same nose, same mouth, same hairline and hair, same skin tone, same age, same gender, same facial expression, and same clothing. Do not alter, beautify, age, or de-age the person. Output a clean, naturally lit photograph of the same person.',
+    });
+    report_ai_status(model);
+  } catch (err) {
+    report_ai_status(model, err);
+    throw err;
+  }
+
+  const outBase64 = response.data?.[0]?.b64_json;
+  if (!outBase64) throw new Error('OpenAI returned no restored image');
+  if (response.usage && account_msa?.record_ai_usage) {
+    account_msa.record_ai_usage(uid, response.usage.input_tokens, response.usage.output_tokens, 'avatar face restoration', 'restore portrait', model, { profile_picture }, account_profile_info);
+  }
+  return outBase64;
+}
+
 async function normalizeBase64To1024(
   base64Image,
   resize = {

package/package.json

@@ -1,31 +1,31 @@
 {
   "name": "@xuda.io/ai_module",
-  "version": "1.1.5607",
+  "version": "1.1.5608",
   "description": "Xuda AI Module",
   "main": "index.mjs",
   "type": "module",
   "dependencies": {
-    "lodash": "^4.17.21",
+    "@imgly/background-removal-node": "^1.4.5",
+    "@openai/agents": "^0.3.4",
+    "amqplib": "^0.10.9",
     "dotenv": "^16.0.3",
-    "openai": "^6.7.0",
+    "firebase-admin": "^9.6.0",
     "fs-extra": "^11.2.0",
-    "utf8": "^3.0.0",
-    "@openai/agents": "^0.3.4",
-    "uglify-js": "^3.19.3",
-    "json5": "^2.2.3",
-    "zod": "^3.25.67",
-    "zod-to-json-schema": "^3.24.6",
-    "sharp": "^0.34.4",
     "https-proxy-agent": "^7.0.6",
+    "json5": "^2.2.3",
+    "lodash": "^4.17.21",
+    "md-to-pdf": "^5.2.5",
+    "openai": "^6.7.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
-    "youtube-dl-exec": "^2.1.3",
-    "md-to-pdf": "^5.2.5",
+    "sharp": "^0.34.4",
+    "uglify-js": "^3.19.3",
+    "utf8": "^3.0.0",
     "vm2": "^3.10.0",
-    "amqplib": "^0.10.9",
-    "firebase-admin": "^9.6.0"
+    "youtube-dl-exec": "^2.1.3",
+    "zod": "^3.25.67",
+    "zod-to-json-schema": "^3.24.6"
   },
-  "devDependencies": {},
   "scripts": {
     "pub": "npm version patch --force && npm publish",
     "test:codex": "node tests/execute-codex-request.mjs"