@pedrofariasx/qwenproxy 1.1.0 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pedrofariasx/qwenproxy",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "Local OpenAI-compatible proxy API that routes requests to Qwen (chat.qwen.ai) via Playwright browser automation.",
   "main": "index.js",
   "scripts": {

package/src/routes/upload.ts

@@ -137,6 +137,10 @@ async function uploadToOSS(
     endpoint,
   } = stsData;
 
+  if (process.env.TEST_MOCK_PLAYWRIGHT) {
+    return stsData.file_url.split("?")[0];
+  }
+
   const OSS = (await import("ali-oss")).default;
   const client = new OSS({
     region,
@@ -608,9 +612,40 @@ export async function processImagesForQwen(
       let fileId = "";
 
       if (mediaUrl.startsWith("http://") || mediaUrl.startsWith("https://")) {
-        fileUrl = mediaUrl;
-        filename = mediaUrl.split("/").pop()?.split("?")[0] || "file.bin";
-        fileId = uuidv4();
+        try {
+          const downloadRes = await fetch(mediaUrl);
+          if (!downloadRes.ok) {
+            console.error(`[Upload] Failed to download media: ${downloadRes.status} ${mediaUrl}`);
+            continue;
+          }
+          const buffer = Buffer.from(await downloadRes.arrayBuffer());
+          fileSize = buffer.length;
+          filename = mediaUrl.split("/").pop()?.split("?")[0] || "file.bin";
+          if (!filename.includes(".")) {
+            const mime = downloadRes.headers.get("content-type") || "";
+            const mimeExt: Record<string, string> = {
+              "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif",
+              "image/webp": "webp", "video/mp4": "mp4", "video/webm": "webm",
+              "audio/mpeg": "mp3", "audio/wav": "wav", "audio/ogg": "ogg",
+              "audio/flac": "flac", "audio/mp4": "m4a", "audio/aac": "aac",
+              "application/pdf": "pdf",
+            };
+            const ext = mimeExt[mime] || "bin";
+            filename = `${filename}.${ext}`;
+          }
+          const typeInfo = detectFileType(filename);
+          const stsData = await getSTSToken(
+            filename,
+            fileSize,
+            typeInfo.qwenFileType,
+            headers,
+          );
+          fileUrl = await uploadToOSS(buffer.buffer, stsData, filename);
+          fileId = stsData.file_id;
+        } catch (err: any) {
+          console.error("[Upload] Failed to download/re-upload HTTP media:", err.message);
+          continue;
+        }
       } else if (mediaUrl.startsWith("data:")) {
         try {
           // Detect type from data URI
@@ -631,6 +666,8 @@ export async function processImagesForQwen(
             "image/jpeg": "jpg",
             "image/gif": "gif",
             "image/webp": "webp",
+            "application/pdf": "pdf",
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "xlsx",
           };
           const detectedExt =
             extFromMime[dataMime] ||

package/src/services/qwen.ts

@@ -351,19 +351,28 @@ export async function createQwenStream(
   const chatHeaders = chatEntry.headers;
   const actualParentId: string | null = null;
 
-  // Process pending multimodal uploads using warm pool headers (no extra Playwright roundtrip)
+  // Process pending multimodal uploads — requires full headers with bx-ua/bx-umidtoken
   let resolvedFiles = files || [];
   if (pendingMultimodal && pendingMultimodal.length > 0 && resolvedFiles.length === 0) {
     try {
       const { processImagesForQwen } = await import('../routes/upload.ts');
+      const { headers: fullHeaders } = await getQwenHeaders(false, accountId);
       const uploadHeaders: Record<string, string> = {
-        cookie: chatHeaders['cookie'] || '',
-        'user-agent': chatHeaders['user-agent'] || '',
-        'bx-ua': chatHeaders['bx-ua'] || '',
-        'bx-umidtoken': chatHeaders['bx-umidtoken'] || '',
-        'bx-v': chatHeaders['bx-v'] || '',
+        cookie: fullHeaders['cookie'] || chatHeaders['cookie'] || '',
+        'user-agent': fullHeaders['user-agent'] || chatHeaders['user-agent'] || '',
+        'bx-ua': fullHeaders['bx-ua'] || '',
+        'bx-umidtoken': fullHeaders['bx-umidtoken'] || '',
+        'bx-v': fullHeaders['bx-v'] || chatHeaders['bx-v'] || '',
       };
-      // Process all multimodal parts in parallel
+      if (!uploadHeaders['bx-ua']) {
+        console.warn('[Qwen] Missing bx-ua header for multimodal upload, attempting forced refresh...');
+        const { headers: refreshedHeaders } = await getQwenHeaders(true, accountId);
+        uploadHeaders['cookie'] = refreshedHeaders['cookie'] || uploadHeaders['cookie'];
+        uploadHeaders['user-agent'] = refreshedHeaders['user-agent'] || uploadHeaders['user-agent'];
+        uploadHeaders['bx-ua'] = refreshedHeaders['bx-ua'] || '';
+        uploadHeaders['bx-umidtoken'] = refreshedHeaders['bx-umidtoken'] || '';
+        uploadHeaders['bx-v'] = refreshedHeaders['bx-v'] || uploadHeaders['bx-v'];
+      }
       const results = await Promise.all(
         pendingMultimodal.map(parts => processImagesForQwen(parts, uploadHeaders))
       );

package/src/tests/media/doc1.pdf

@@ -0,0 +1,105 @@
+%PDF-1.4
+%���� ReportLab Generated PDF document http://www.reportlab.com
+1 0 obj
+<<
+/F1 2 0 R /F2 3 0 R /F3 5 0 R /F4 6 0 R
+>>
+endobj
+2 0 obj
+<<
+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font
+>>
+endobj
+3 0 obj
+<<
+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font
+>>
+endobj
+4 0 obj
+<<
+/Contents 11 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 10 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+5 0 obj
+<<
+/BaseFont /Symbol /Name /F3 /Subtype /Type1 /Type /Font
+>>
+endobj
+6 0 obj
+<<
+/BaseFont /Courier /Encoding /WinAnsiEncoding /Name /F4 /Subtype /Type1 /Type /Font
+>>
+endobj
+7 0 obj
+<<
+/Contents 12 0 R /MediaBox [ 0 0 595.2756 841.8898 ] /Parent 10 0 R /Resources <<
+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]
+>> /Rotate 0 /Trans <<
+
+>> 
+  /Type /Page
+>>
+endobj
+8 0 obj
+<<
+/PageMode /UseNone /Pages 10 0 R /Type /Catalog
+>>
+endobj
+9 0 obj
+<<
+/Author (\(anonymous\)) /CreationDate (D:20260604002337+00'00') /Creator (\(unspecified\)) /Keywords () /ModDate (D:20260604002337+00'00') /Producer (ReportLab PDF Library - www.reportlab.com) 
+  /Subject (\(unspecified\)) /Title (\(anonymous\)) /Trapped /False
+>>
+endobj
+10 0 obj
+<<
+/Count 2 /Kids [ 4 0 R 7 0 R ] /Type /Pages
+>>
+endobj
+11 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 756
+>>
+stream
+Gat=h?Z4XP'ZJu*'^(TQr%k1&p5Q2TKXRk%bMuo&EZsi`411VWp??qa64f)RWBuJ1I!pD?4823P^crF&8HF3qdD$D-6c;WR;/_-_#JS0Za-P@Y7MA07d@3h4-G4sI#%>?N>.donEkJYbW?K3UQ5U6KTi';Q!u;^r:=6d_PFc"M]Km>uX5'bmUbYG6cJuni<C,>g6MuB/1V*fqg!Dn^%S[BchDI0<29#;/K_kpOm9`g2VnB:(CLl:gm^h:_.A'Uu?=Je@8Z=R?BpEedrC?Qdet.**:0J6V5"A9+ant;?kbQH_;\&Dg>HF[=MDSK#.0]a6E#\1#ngZr:"c&qZn;L90'oG.[IT#T[R!H3A./7!GaFQ;=0%;L(b<-&.Za`^bjXK_b-(&7LLOEtIido^-k?*\`=)YD&9L%#rmCOaO:Q!_9%j"uih[eb2V"ZfYi_ng>,X%]@\J\$(eKQAo(18'/_jZH;_pW&6Zl,\HKGr3W7QXEUi['B@Ga$OBc=bc9X'GHp#;SkEV/'YgDiO1hVTFtA4KlYE<EU7+4)j&>/F$<&>\Q#FG3[SG@i!Xs*EgJ+Rj(CC7jOL'[FMU_?!^gLQ.fgA_;T<^V[^?YSP\S,\\goB1NFTX"m+@'p9F!YT`%AKd>>A(Vkm7:HYd('.`5>cjFSEC6>F'*k/4O/S8Joc9e'R8Q;Zh!S+3F<5*=Goam]8I40"\2EjN2hQ20i7\pDeo(lp172]g1X4oNhIf#I^6ND,CYU\+\'c)QK~>endstream
+endobj
+12 0 obj
+<<
+/Filter [ /ASCII85Decode /FlateDecode ] /Length 619
+>>
+stream
+Gat=(9lHLd&A@sBN&\?_+^RhJpdV?^:$X-$bcU,<M^K"b_V2Up.>m]R4(*XY9>Mf=6=P]D_<a9$9^#s14im[V@Q;&3^g)[;J9d.[nH(nnUYUa6Dk72QmP-'p_LN3IQo#8[)%i0WF9KgDaorR1<3BF=N?a]NV?R]4."lAM:k+#j2oHeRNE`u#l`fD1X;)r*PK%Yo"K'nM#oB7R=.hJ@+CkeXT#"[-QlB^l:G8WLEVJmhhf]JHiF\!nFEK@*fp8-"QU$@m=2S`i9?`_priLoBUEks*=hg/S5rB?P6QTS_=_`Pk;!V*CmrM9<od<IQ3;O-X<ot</AJF]O35PU\SA(ii_m!@6_j2ASfNYNk%N=:@caIrR?WT2<YiM(2I,4,)o,Y;AB^.tKpj4S'RYV*l?L%p!9Wh5p1U)$Zmcd;0O34c-:<1@5W2]FlSh)/k=u35+Fk95rDJ2"/pjth'lPY0*0.+`*DC/+DUC)\;?M@C%F+;hOmZA$,/,=1U_/EG@;QY:5Ihj#jGU)id[-T]PF[Br$V%bWH6>.=(Jled@2iksrQS?ZC<,Mb?mklR6(5qYNR2[\;0t!/.-5dOfB(Q1e\c#kX9j;nrE*Z*YmO=+o!WC&Bao~>endstream
+endobj
+xref
+0 13
+0000000000 65535 f 
+0000000073 00000 n 
+0000000134 00000 n 
+0000000241 00000 n 
+0000000353 00000 n 
+0000000558 00000 n 
+0000000635 00000 n 
+0000000740 00000 n 
+0000000945 00000 n 
+0000001014 00000 n 
+0000001297 00000 n 
+0000001363 00000 n 
+0000002210 00000 n 
+trailer
+<<
+/ID 
+[<eb349826a1153a222f9fd87f711c0d31><eb349826a1153a222f9fd87f711c0d31>]
+% ReportLab generated PDF document -- digest (http://www.reportlab.com)
+
+/Info 9 0 R
+/Root 8 0 R
+/Size 13
+>>
+startxref
+2920
+%%EOF

package/src/tests/multimodal.test.ts

@@ -0,0 +1,146 @@
+import test from 'node:test';
+import assert from 'node:assert';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import net from 'node:net';
+import { serve } from '@hono/node-server';
+import { app } from '../api/server.js';
+import { initPlaywright, closePlaywright } from '../services/playwright.ts';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const mediaDir = path.join(__dirname, 'media');
+
+function isPortAvailable(port: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    const server = net.createServer();
+    server.once('error', () => resolve(false));
+    server.once('listening', () => {
+      server.close(() => resolve(true));
+    });
+    server.listen(port);
+  });
+}
+
+async function getFreePort(startPort: number): Promise<number> {
+  let port = startPort;
+  while (true) {
+    const available = await isPortAvailable(port);
+    if (available) return port;
+    port++;
+  }
+}
+
+function fileToDataUri(filePath: string): string {
+  const buffer = fs.readFileSync(filePath);
+  const ext = path.extname(filePath).slice(1).toLowerCase();
+  const mimeMap: Record<string, string> = {
+    png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg',
+    mp4: 'video/mp4', mp3: 'audio/mpeg',
+    pdf: 'application/pdf',
+    xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+  };
+  return `data:${mimeMap[ext] || 'application/octet-stream'};base64,${buffer.toString('base64')}`;
+}
+
+async function sendMultimodalRequest(
+  port: number,
+  prompt: string,
+  urlType: string,
+  dataUri: string,
+): Promise<{ content: string; reasoning: string }> {
+  const contentPart: any = { type: urlType };
+  if (urlType === 'image_url') contentPart.image_url = { url: dataUri };
+  else if (urlType === 'video_url') contentPart.video_url = { url: dataUri };
+  else if (urlType === 'audio_url') contentPart.audio_url = { url: dataUri };
+  else contentPart.file_url = { url: dataUri };
+
+  const response = await fetch(`http://localhost:${port}/v1/chat/completions`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({
+      model: 'qwen3.6-plus',
+      messages: [{ role: 'user', content: [
+        { type: 'text', text: prompt },
+        contentPart,
+      ]}],
+      stream: true
+    })
+  });
+
+  assert.strictEqual(response.status, 200, `Expected 200, got ${response.status}`);
+
+  const reader = response.body!.getReader();
+  const decoder = new TextDecoder();
+  let content = '';
+  let reasoning = '';
+  let buffer = '';
+
+  while (true) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    buffer += decoder.decode(value, { stream: true });
+    const lines = buffer.split('\n');
+    buffer = lines.pop() || '';
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (!trimmed || !trimmed.startsWith('data: ')) continue;
+      const dataStr = trimmed.slice(6);
+      if (dataStr === '[DONE]') continue;
+      try {
+        const chunk = JSON.parse(dataStr);
+        const delta = chunk.choices?.[0]?.delta;
+        if (delta?.content) content += delta.content;
+        if (delta?.reasoning_content) reasoning += delta.reasoning_content;
+      } catch {}
+    }
+  }
+
+  return { content, reasoning };
+}
+
+test('Multimodal: all media files with real Qwen responses', { skip: process.env.CI ? 'Requires real accounts - skipped in CI' : false }, async () => {
+  const port = await getFreePort(3200);
+  const server = serve({ fetch: app.fetch, port });
+  console.log(`[MultimodalTest] Server started on port ${port}`);
+
+  await initPlaywright(true);
+
+  try {
+    const scenarios = [
+      { file: 'farias.png', urlType: 'image_url', prompt: 'Descreva essa imagem em detalhes', requireContent: true },
+      { file: 'video.mp4', urlType: 'video_url', prompt: 'Descreva o conteúdo deste vídeo', requireContent: true },
+      { file: 'audio.mp3', urlType: 'audio_url', prompt: 'Transcreva e descreva o que é dito neste áudio', requireContent: true },
+      { file: 'doc1.pdf', urlType: 'file_url', prompt: 'Resuma o conteúdo deste documento PDF', requireContent: false },
+      { file: 'doc2.xlsx', urlType: 'file_url', prompt: 'Analise os dados desta planilha e descreva o que contém', requireContent: false },
+    ];
+
+    for (const scenario of scenarios) {
+      const filePath = path.join(mediaDir, scenario.file);
+      if (!fs.existsSync(filePath)) {
+        console.log(`[MultimodalTest] SKIP ${scenario.file} - not found`);
+        continue;
+      }
+
+      const dataUri = fileToDataUri(filePath);
+      console.log(`[MultimodalTest] Sending ${scenario.file} (${(fs.statSync(filePath).size / 1024).toFixed(1)}KB)...`);
+
+      const { content, reasoning } = await sendMultimodalRequest(port, scenario.prompt, scenario.urlType, dataUri);
+
+      console.log(`[MultimodalTest] ${scenario.file} => ${content.length} chars`);
+      if (content) console.log(`  Content: ${content.substring(0, 300)}`);
+      if (reasoning) console.log(`  Reasoning: ${reasoning.substring(0, 150)}...`);
+
+      if (scenario.requireContent) {
+        assert.ok(content.length > 10, `${scenario.file}: expected meaningful response, got ${content.length} chars`);
+      } else if (content.length === 0) {
+        console.log(`[MultimodalTest] WARN: ${scenario.file} returned empty response (Qwen may not support this file type via ${scenario.urlType})`);
+      }
+    }
+  } finally {
+    await closePlaywright();
+    server.close();
+    console.log('[MultimodalTest] Done.');
+  }
+});