@forgeax-extension/character-3d 0.1.2

package/server/providers/meshy.ts

@@ -0,0 +1,548 @@
+// MeshyProvider — real submit/poll client for the LiteLLM 3D gateway.
+//
+// Decoupling (ADR-0001): like HunyuanWorkflowProvider, this talks to the remote
+// gateway and returns a pure ProviderResult with downloaded bytes. It knows
+// nothing about cache, asset-store, or manifests. Auth is plain `Authorization:
+// Bearer <key>`.
+//
+// Differences from direct Meshy API (now routed through LiteLLM gateway):
+//   - All submits go to POST /v1/3d/generations with a `model` field.
+//   - All polls go to GET /v1/3d/tasks/{taskId}; task id is in `id`.
+//   - status is lower-case: succeeded/failed/canceled.
+//   - outputs come back as `data[]` (type+format-tagged), not model_urls dict.
+//   - text is two-stage: preview (white mesh) then refine (adds texture).
+//   - Rig/animate also go through the gateway with model=meshy-3d-auto-rigging
+//     /meshy-3d-animation (response format in data[] entries).
+
+import { clampTargetPolycount, type ProviderResult, type ProviderResultFile } from '../../shared/catalog';
+import type { FileFormat, FileRole, GenerationMode } from '../../shared/manifest';
+import { MESHY_ACTIONS, MESHY_ACTION_BASE } from '../../shared/meshy-actions';
+import type { MeshyEnv } from '../env';
+import { audit } from '../audit';
+import { RateGuard } from '../rate-guard';
+import { extractGatewayUrls, extractRigUrls } from './gateway-data';
+import { GatewayClient, type FetchLike } from './gateway-client';
+import { DirectMeshyClient } from './meshy-direct-client';
+import { MESHY_GATEWAY_MODELS } from './gateway-models';
+
+const GATEWAY_SUBMIT = '/v1/3d/generations';
+const GATEWAY_POLL = '/v1/3d/tasks';
+
+const MODE_TO_MODEL: Record<string, string> = {
+  text: MESHY_GATEWAY_MODELS.text,
+  image: MESHY_GATEWAY_MODELS.image,
+  views: MESHY_GATEWAY_MODELS.views,
+  refine: MESHY_GATEWAY_MODELS.refine,
+};
+
+export type MeshyMode = 'text' | 'image' | 'views' | 'refine';
+
+export interface MeshyGenerateInput {
+  mode: MeshyMode;
+  prompt?: string;
+  imageUrl?: string;
+  imageUrls?: string[];
+  previewTaskId?: string;
+  texturePrompt?: string;
+  enablePbr?: boolean;
+  shouldTexture?: boolean;
+  targetPolycount?: number;
+  aiModel?: string;
+  params?: Record<string, string | number | boolean>;
+}
+
+// ── Rig / animation I/O (ADR-0006) ──────────────────────────────────────────
+
+export interface MeshyRigInput {
+  inputTaskId?: string;
+  modelUrl?: string;
+  heightMeters?: number;
+}
+
+export interface MeshyAnimateInput {
+  rigTaskId: string;
+  actionId: number;
+}
+
+export interface MeshyBasicAnimation {
+  category: 'walking' | 'running';
+  glb: Uint8Array;
+  fbx: Uint8Array | null;
+}
+
+export interface MeshyRigResult {
+  sourceJobId: string;
+  rigType: string | null;
+  expiresAt: number | null;
+  glb: Uint8Array;
+  fbx: Uint8Array | null;
+  basicAnimations: MeshyBasicAnimation[];
+}
+
+export interface MeshyAnimateResult {
+  sourceJobId: string;
+  expiresAt: number | null;
+  glb: Uint8Array;
+  fbx: Uint8Array | null;
+}
+
+export interface MeshyAction {
+  id: number;
+  name: string;
+  category: string | null;
+  rigType: string | null;
+  isFree: boolean;
+  previewGifUrl: string | null;
+}
+
+// Injectable transports so smokes can run without a real network call.
+export type DownloadLike = (url: string) => Promise<Uint8Array>;
+
+export interface MeshyProviderDeps {
+  env: MeshyEnv;
+  slug: string;
+  fetchImpl?: FetchLike;
+  downloadImpl?: DownloadLike;
+  rateGuard?: RateGuard;
+  sleep?: (ms: number) => Promise<void>;
+}
+
+// Map a gateway data[] entry format to a durable manifest file role+format.
+const DATA_TYPE_TO_FILE: Record<string, { role: FileRole; format: FileFormat }> = {
+  glb: { role: 'source_mesh', format: 'glb' },
+  fbx: { role: 'source_mesh', format: 'fbx' },
+  obj: { role: 'source_mesh', format: 'obj' },
+  usdz: { role: 'source_mesh', format: 'usdz' },
+  stl: { role: 'source_mesh', format: 'stl' },
+};
+
+export class MeshyProvider {
+  private readonly env: MeshyEnv;
+  private readonly slug: string;
+  private readonly fetchImpl: FetchLike;
+  private readonly downloadImpl: DownloadLike;
+  private readonly rateGuard: RateGuard;
+  private readonly sleep: (ms: number) => Promise<void>;
+  // One client per provider: DirectMeshyClient remembers submit→poll paths
+  // on the instance. `new` per call used to drop that map, so rig/animate
+  // polls guessed `/image-to-3d/:id` and returned the static mesh with no
+  // Character_output.glb (`provider_no_output` in ~1s).
+  private cachedGateway: GatewayClient | null = null;
+
+  constructor(deps: MeshyProviderDeps) {
+    this.env = deps.env;
+    this.slug = deps.slug;
+    this.fetchImpl = deps.fetchImpl ?? ((url, init) => fetch(url, init));
+    this.downloadImpl =
+      deps.downloadImpl ?? (async (url) => this.gateway().download(url, { format: urlFormat(url), image: isImageUrl(url) }));
+    this.rateGuard = deps.rateGuard ?? new RateGuard(this.env.rateLimitPerMin);
+    this.sleep = deps.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  }
+
+  async generate(input: MeshyGenerateInput): Promise<ProviderResult> {
+    const result = await this.submitAndDownload(input);
+    // Meshy text is two-stage: preview (white mesh) → refine (texture). Auto-chain
+    // when texturing is wanted (default on; enablePbr=false keeps preview-only).
+    // Persist BOTH task ids: manual refine needs preview; rig/audit use result.
+    if (input.mode === 'text' && input.enablePbr !== false) {
+      const previewTaskId = result.sourceJobId;
+      if (!previewTaskId) {
+        throw Object.assign(new Error('meshy text preview returned no task id'), { code: 'provider_no_task' });
+      }
+      const refined = await this.submitAndDownload({
+        mode: 'refine',
+        previewTaskId,
+        texturePrompt: resolveTexturePrompt(input),
+        enablePbr: input.enablePbr ?? true,
+        // Texture / lighting knobs belong on refine, not white-mesh preview.
+        params: pickRefineParams(input.params),
+      });
+      return {
+        ...refined,
+        mode: 'text',
+        prompt: input.prompt ?? null,
+        meshyTaskRefs: {
+          previewTaskId,
+          resultTaskId: refined.sourceJobId,
+        },
+      };
+    }
+    if (input.mode === 'text' || input.mode === 'image' || input.mode === 'views' || input.mode === 'refine') {
+      return {
+        ...result,
+        meshyTaskRefs: {
+          previewTaskId: input.mode === 'text' ? result.sourceJobId : input.mode === 'refine' ? (input.previewTaskId ?? null) : null,
+          resultTaskId: result.sourceJobId,
+        },
+      };
+    }
+    return result;
+  }
+
+  private async submitAndDownload(input: MeshyGenerateInput): Promise<ProviderResult> {
+    const model = MODE_TO_MODEL[input.mode];
+    const payload = { model, ...this.buildPayload(input) };
+
+    this.rateGuard.check();
+
+    const submitResp = await this.post(payload);
+    const sourceJobId = submitTaskId(submitResp);
+    if (!sourceJobId) {
+      throw Object.assign(new Error('meshy submit returned no task id'), { code: 'provider_no_task' });
+    }
+    await audit(this.slug, { ts: new Date().toISOString(), provider: 'meshy', mode: input.mode, event: 'submit', sourceJobId });
+
+    const urls = await this.poll(input.mode, sourceJobId);
+    const files = await this.downloadFiles(urls);
+    if (!files.some((f) => f.role === 'source_mesh' && f.format === 'glb')) {
+      throw Object.assign(new Error('meshy succeeded without a GLB mesh'), { code: 'provider_no_output' });
+    }
+
+    return {
+      provider: 'meshy',
+      mode: input.mode as GenerationMode,
+      providerMode: 'real',
+      sourceJobId,
+      prompt: input.prompt ?? null,
+      files,
+    };
+  }
+
+  // Auto-rig via gateway (model=meshy-3d-auto-rigging). The rig response data[]
+  // carries the rigged Character_output plus free walk/run clips (withSkin and
+  // _armature variants per format). extractRigUrls classifies by filename so
+  // Character_output.glb (full mesh+skeleton) is picked, not the last
+  // _armature.glb (skeleton-only proxy); the walk/run _withSkin clips are
+  // returned as basicAnimations. The gateway does NOT return rig_type or
+  // expires_at — those were Meshy direct API fields only.
+  async rig(input: MeshyRigInput): Promise<MeshyRigResult> {
+    if (!input.inputTaskId && !input.modelUrl) {
+      throw Object.assign(new Error('meshy rig needs inputTaskId or modelUrl'), {
+        code: 'invalid_rig_input',
+      });
+    }
+    const payload: Record<string, unknown> = { model: MESHY_GATEWAY_MODELS.autoRig };
+    if (input.inputTaskId) payload.input_task_id = input.inputTaskId;
+    if (input.modelUrl) payload.model_url = input.modelUrl;
+    if (input.heightMeters !== undefined) payload.height_meters = input.heightMeters;
+
+    this.rateGuard.check();
+    const submit = await this.post(payload);
+    const taskId = submitTaskId(submit);
+    if (!taskId) {
+      throw Object.assign(new Error('meshy rig submit returned no task id'), { code: 'provider_no_task' });
+    }
+    await audit(this.slug, { ts: new Date().toISOString(), provider: 'meshy', mode: 'image', event: 'submit', sourceJobId: taskId, detail: 'rig' });
+
+    const resp = await this.pollTask(taskId, 'rig');
+    const urls = extractRigUrls(resp);
+    if (!urls.glb) {
+      throw Object.assign(new Error('meshy rig returned no rigged glb (Character_output.glb)'), { code: 'provider_no_output' });
+    }
+    const basicAnimations: MeshyBasicAnimation[] = [];
+    if (urls.walkGlb) {
+      basicAnimations.push({
+        category: 'walking',
+        glb: await this.downloadImpl(urls.walkGlb),
+        fbx: urls.walkFbx ? await this.downloadImpl(urls.walkFbx) : null,
+      });
+    }
+    if (urls.runGlb) {
+      basicAnimations.push({
+        category: 'running',
+        glb: await this.downloadImpl(urls.runGlb),
+        fbx: urls.runFbx ? await this.downloadImpl(urls.runFbx) : null,
+      });
+    }
+    return {
+      sourceJobId: taskId,
+      // Gateway does NOT return rig_type / expires_at — set null.
+      rigType: null,
+      expiresAt: null,
+      glb: await this.downloadImpl(urls.glb),
+      fbx: urls.fbx ? await this.downloadImpl(urls.fbx) : null,
+      basicAnimations,
+    };
+  }
+
+  // Apply one motion via gateway (model=meshy-3d-animation).
+  // Gateway has no expires_at — treat remote "rig gone/expired/invalid" as
+  // rig_expired so callers can decide whether to re-rig (PLAN §8-Q3).
+  async animate(input: MeshyAnimateInput): Promise<MeshyAnimateResult> {
+    this.rateGuard.check();
+    try {
+      const submit = await this.post({
+        model: MESHY_GATEWAY_MODELS.animation,
+        rig_task_id: input.rigTaskId,
+        action_id: input.actionId,
+      });
+      const taskId = submitTaskId(submit);
+      if (!taskId) {
+        throw Object.assign(new Error('meshy animate submit returned no task id'), { code: 'provider_no_task' });
+      }
+      await audit(this.slug, { ts: new Date().toISOString(), provider: 'meshy', mode: 'image', event: 'submit', sourceJobId: taskId, detail: `animate:${input.actionId}` });
+
+      const resp = await this.pollTask(taskId, 'animate');
+      const urls = extractGatewayUrls(resp);
+      if (!urls.glb) {
+        throw Object.assign(new Error('meshy animate returned no animation glb'), { code: 'provider_no_output' });
+      }
+      return {
+        sourceJobId: taskId,
+        expiresAt: null,
+        glb: await this.downloadImpl(urls.glb),
+        fbx: urls.fbx ? await this.downloadImpl(urls.fbx) : null,
+      };
+    } catch (error) {
+      throw mapAnimateRigExpiry(error);
+    }
+  }
+
+  // Motion catalog — static, no network call.
+  async listActions(): Promise<MeshyAction[]> {
+    return MESHY_ACTIONS.map((row) => {
+      const [id, name, category, , previewGifRel] = row;
+      return {
+        id,
+        name,
+        category,
+        rigType: null,
+        isFree: false,
+        previewGifUrl: previewGifRel ? `${MESHY_ACTION_BASE}${previewGifRel}` : null,
+      };
+    });
+  }
+
+  // Gateway has no balance endpoint — null means "unknown", not zero credits.
+  async getBalance(): Promise<number | null> {
+    return null;
+  }
+
+  // Flatten data[] urls into downloaded ProviderResultFiles.
+  private async downloadFiles(urls: Record<string, string>): Promise<ProviderResultFile[]> {
+    const files: ProviderResultFile[] = [];
+    const seenRoles = new Set<string>();
+    for (const [key, mapping] of Object.entries(DATA_TYPE_TO_FILE)) {
+      const url = urls[key];
+      if (!url) continue;
+      const roleKey = `${mapping.role}:${mapping.format}`;
+      if (seenRoles.has(roleKey)) continue;
+      seenRoles.add(roleKey);
+      files.push({ role: mapping.role, format: mapping.format, data: await this.downloadImpl(url) });
+    }
+    if (urls.__thumbnail) {
+      files.push({ role: 'preview_image', format: 'png', data: await this.downloadImpl(urls.__thumbnail) });
+    }
+    if (urls.__texture_base_color) {
+      files.push({ role: 'texture', format: 'png', data: await this.downloadImpl(urls.__texture_base_color) });
+    }
+    return files;
+  }
+
+  // Poll for generation tasks (returns flat url map for downloadFiles).
+  private async poll(mode: MeshyMode, taskId: string): Promise<Record<string, string>> {
+    try {
+      const resp = await this.gateway().pollTask(`${GATEWAY_POLL}/${taskId}`, {
+        timeoutMs: this.env.pollTimeoutMs,
+        intervalMs: this.env.pollIntervalMs,
+        label: 'meshy task',
+      });
+      await audit(this.slug, { ts: new Date().toISOString(), provider: 'meshy', mode, event: 'poll_succeeded', sourceJobId: taskId, detail: 'succeeded' });
+      return extractGatewayUrls(resp);
+    } catch (error) {
+      await audit(this.slug, { ts: new Date().toISOString(), provider: 'meshy', mode, event: auditPollEvent(error), sourceJobId: taskId, detail: error instanceof Error ? error.message : undefined });
+      throw error;
+    }
+  }
+
+  // Poll for rig/animate tasks (returns the full response for custom extraction).
+  private async pollTask(taskId: string, label: string): Promise<Record<string, unknown>> {
+    try {
+      const resp = await this.gateway().pollTask(`${GATEWAY_POLL}/${taskId}`, {
+        timeoutMs: this.env.pollTimeoutMs,
+        intervalMs: this.env.pollIntervalMs,
+        label: `meshy ${label}`,
+      });
+      await audit(this.slug, { ts: new Date().toISOString(), provider: 'meshy', mode: 'image', event: 'poll_succeeded', sourceJobId: taskId, detail: label });
+      return resp;
+    } catch (error) {
+      await audit(this.slug, { ts: new Date().toISOString(), provider: 'meshy', mode: 'image', event: auditPollEvent(error), sourceJobId: taskId, detail: `${label}:${error instanceof Error ? error.message : ''}` });
+      throw error;
+    }
+  }
+
+  private buildPayload(input: MeshyGenerateInput): Record<string, unknown> {
+    const polycount =
+      input.targetPolycount !== undefined ? clampTargetPolycount(input.targetPolycount) : undefined;
+
+    if (input.mode === 'text') {
+      const payload: Record<string, unknown> = { mode: 'preview', prompt: input.prompt ?? '' };
+      if (input.aiModel) payload.ai_model = input.aiModel;
+      if (polycount) {
+        payload.should_remesh = true;
+        payload.target_polycount = polycount;
+      }
+      // Preview is white-mesh only — strip refine/texture-only knobs.
+      this.applyProviderParams(payload, omitKeys(input.params, REFINE_TEXTURE_KEYS));
+      return payload;
+    }
+    if (input.mode === 'refine') {
+      const payload: Record<string, unknown> = { mode: 'refine', preview_task_id: input.previewTaskId };
+      if (input.texturePrompt) payload.texture_prompt = input.texturePrompt;
+      if (typeof input.enablePbr === 'boolean') payload.enable_pbr = input.enablePbr;
+      this.applyProviderParams(payload, pickRefineParams(input.params));
+      return payload;
+    }
+    if (input.mode === 'image') {
+      const payload: Record<string, unknown> = { image_url: input.imageUrl ?? '' };
+      this.applyMeshOptions(payload, input, polycount);
+      this.applyProviderParams(payload, input.params);
+      sanitizeSmartTopologyPayload(payload);
+      return payload;
+    }
+    // views
+    const payload: Record<string, unknown> = { image_urls: (input.imageUrls ?? []).slice(0, 4) };
+    this.applyMeshOptions(payload, input, polycount);
+    this.applyProviderParams(payload, input.params);
+    sanitizeSmartTopologyPayload(payload);
+    return payload;
+  }
+
+  private applyProviderParams(
+    payload: Record<string, unknown>,
+    params: Record<string, string | number | boolean> | undefined,
+  ): void {
+    if (!params) return;
+    Object.assign(payload, params);
+    if (payload.topology !== undefined && payload.should_remesh === undefined) {
+      payload.should_remesh = true;
+    }
+  }
+
+  private applyMeshOptions(
+    payload: Record<string, unknown>,
+    input: MeshyGenerateInput,
+    polycount: number | undefined,
+  ): void {
+    if (input.aiModel) payload.ai_model = input.aiModel;
+    if (typeof input.enablePbr === 'boolean') payload.enable_pbr = input.enablePbr;
+    else payload.enable_pbr = true;
+    // LiteLLM gateway does not inherit Meshy API defaults — explicit should_texture
+    // avoids white-mesh-only output on image/views (character turnaround path).
+    if (typeof input.shouldTexture === 'boolean') payload.should_texture = input.shouldTexture;
+    else payload.should_texture = true;
+    if (polycount) payload.target_polycount = polycount;
+  }
+
+  private async post(body: unknown): Promise<Record<string, unknown>> {
+    return this.gateway().post(GATEWAY_SUBMIT, body);
+  }
+
+  private async get(path: string): Promise<Record<string, unknown>> {
+    return this.gateway().get(path);
+  }
+
+  private gateway(): GatewayClient {
+    if (!this.cachedGateway) {
+      this.cachedGateway =
+        this.env.transport === 'direct'
+          ? new DirectMeshyClient({ env: this.env, fetchImpl: this.fetchImpl, sleep: this.sleep })
+          : new GatewayClient({ env: this.env, fetchImpl: this.fetchImpl, sleep: this.sleep });
+    }
+    return this.cachedGateway;
+  }
+}
+
+function submitTaskId(resp: Record<string, unknown>): string | null {
+  return (resp.id as string) ?? null;
+}
+
+function auditPollEvent(error: unknown): 'poll_failed' | 'poll_timeout' {
+  return (error as { code?: unknown } | null)?.code === 'provider_timeout' ? 'poll_timeout' : 'poll_failed';
+}
+
+/** Map gateway animate failures that mean the rig task is gone → rig_expired. */
+export function mapAnimateRigExpiry(error: unknown): Error {
+  if ((error as { code?: unknown } | null)?.code === 'rig_expired') return error as Error;
+  const msg = error instanceof Error ? error.message : String(error);
+  const code = (error as { code?: unknown } | null)?.code;
+  const looksExpired =
+    /expir|rig[_\s-]?task.*(invalid|not\s*found|gone|missing)|invalid.*rig|task.*(not\s*found|gone)/i.test(msg) ||
+    (code === 'provider_bad_request' && /rig/i.test(msg)) ||
+    (code === 'provider_not_enabled' && /rig/i.test(msg));
+  if (looksExpired) {
+    return Object.assign(new Error(`Meshy rig task expired or invalid; re-run gen3d:auto-rig or pass autoReRig:true (${msg})`), {
+      code: 'rig_expired',
+      cause: error,
+    });
+  }
+  return error instanceof Error ? error : Object.assign(new Error(msg), { code: 'provider_failed' });
+}
+
+/** Texture / lighting fields that must not ride on text preview. */
+const REFINE_TEXTURE_KEYS = [
+  'hd_texture',
+  'texture_prompt',
+  'texture_image_url',
+  'remove_lighting',
+] as const;
+
+function resolveTexturePrompt(input: MeshyGenerateInput): string | undefined {
+  const fromParams = input.params?.texture_prompt;
+  if (typeof fromParams === 'string' && fromParams.trim()) return fromParams.trim();
+  return input.prompt;
+}
+
+/** Params that belong on refine / textured stages. */
+export function pickRefineParams(
+  params: Record<string, string | number | boolean> | undefined,
+): Record<string, string | number | boolean> | undefined {
+  if (!params) return undefined;
+  const out: Record<string, string | number | boolean> = {};
+  if (params.hd_texture === true) out.hd_texture = true;
+  if (typeof params.texture_prompt === 'string' && params.texture_prompt.trim()) {
+    out.texture_prompt = params.texture_prompt.trim();
+  }
+  if (typeof params.texture_image_url === 'string' && params.texture_image_url.trim()) {
+    out.texture_image_url = params.texture_image_url.trim();
+  }
+  if (typeof params.remove_lighting === 'boolean') out.remove_lighting = params.remove_lighting;
+  return Object.keys(out).length > 0 ? out : undefined;
+}
+
+function omitKeys(
+  params: Record<string, string | number | boolean> | undefined,
+  keys: readonly string[],
+): Record<string, string | number | boolean> | undefined {
+  if (!params) return undefined;
+  const drop = new Set(keys);
+  const out: Record<string, string | number | boolean> = {};
+  for (const [k, v] of Object.entries(params)) {
+    if (!drop.has(k)) out[k] = v;
+  }
+  return Object.keys(out).length > 0 ? out : undefined;
+}
+
+/**
+ * Smart Topology ignores remesh/topology; coerce standard Meshy-5/6 ids to t2
+ * so a mismatched advanced-param combo still hits the documented t2 path.
+ */
+export function sanitizeSmartTopologyPayload(payload: Record<string, unknown>): void {
+  if (payload.model_type !== 'smart-topology') return;
+  const ai = payload.ai_model;
+  if (ai === undefined || ai === 'meshy-5' || ai === 'meshy-6' || ai === 'latest') {
+    payload.ai_model = 'meshy-t2';
+  }
+  delete payload.should_remesh;
+  delete payload.topology;
+  delete payload.decimation_mode;
+}
+
+function urlFormat(url: string): string | undefined {
+  const path = url.split('?', 1)[0] ?? url;
+  return path.split('.').pop()?.toLowerCase();
+}
+
+function isImageUrl(url: string): boolean {
+  return /\.(png|jpe?g|webp|gif)(?:\?|$)/i.test(url);
+}