channel-worker 2.5.13 → 2.5.15

package/lib/command-poller.js

@@ -180,18 +180,26 @@ class CommandPoller {
     let conn;
     try {
       conn = await this._connectNstProfileByName(profileName);
-      const products = await scraper.scrapeOffers(conn.browser, {
-        sort_type: payload.sort_type ?? 2,        // 2 = commission desc
+      const res = await scraper.scrapeOffers(conn.browser, {
+        category: payload.category || 'women_clothes',   // Tier-1 default: quần áo nữ
+        match_id: payload.match_id || null,
+        sort_types: payload.sort_types || [2, 3],         // commission + sales, merged
+        pages: payload.pages ?? 2,
         page_limit: payload.page_limit ?? 20,
-        pages: payload.pages ?? 3,
-        list_type: payload.list_type ?? 0,
+        delay_ms: payload.delay_ms ?? 2500,               // human-like pacing (anti-bot)
+        filters: payload.filters || {},
       });
-      const filtered = payload.category_group
-        ? products.filter(p => p.category_group === payload.category_group)
-        : products;
-      console.log(`[shopee] scraped ${products.length} offers (${filtered.length} after filter) from ${profileName}`);
-      await this.api.upsertAffiliateProducts(filtered, { user_id: command.user_id });
-      await this.api.updateCommand(command._id, { status: 'done', result: { count: filtered.length, total: products.length } });
+      if (res.status === 'needs_verify') {
+        // Anti-bot bounce — DON'T fail/retry into more captchas. Report cleanly
+        // so the dashboard can prompt the operator to re-verify via VNC.
+        console.warn(`[shopee] needs_verify: ${res.reason}`);
+        if (res.products?.length) await this.api.upsertAffiliateProducts(res.products, { user_id: command.user_id });
+        await this.api.updateCommand(command._id, { status: 'done', result: { needs_verify: true, reason: res.reason, count: res.products?.length || 0 } });
+        return;
+      }
+      console.log(`[shopee] scraped ${res.raw_count} raw → ${res.products.length} after filters from ${profileName}`);
+      await this.api.upsertAffiliateProducts(res.products, { user_id: command.user_id });
+      await this.api.updateCommand(command._id, { status: 'done', result: { count: res.products.length, raw: res.raw_count } });
     } catch (err) {
       console.error(`[shopee] scrape failed: ${err.message}`);
       await this.api.updateCommand(command._id, { status: 'failed', error: String(err.message || err).slice(0, 500) });

package/lib/shopee-scraper.js

@@ -120,33 +120,115 @@ function parseOfferRow(row) {
   };
 }
 
-// In-page fetch the affiliate offer list. sort_type: 1=relevance, 2=commission,
-// 3=sales (Shopee's internal codes). Loops `pages` times, page_limit per page.
-async function scrapeOffers(browser, { sort_type = 2, page_limit = 20, pages = 1, list_type = 0, timeoutMs = 45000 } = {}) {
+// Affiliate offer-page category tabs → match_id (read from rc-tabs node keys,
+// confirmed live). Tier-1 fashion focus first; the offer page has NO dedicated
+// Shoes/Bags/Accessories tab → those come later via keyword search.
+const AFFILIATE_CATEGORIES = {
+  women_clothes: { match_id: 100017, label: 'Quần áo nữ', group: 'fashion' },
+  beauty:        { match_id: 100630, label: 'Làm đẹp',    group: 'beauty' },
+  home_living:   { match_id: 100636, label: 'Nhà cửa',    group: 'other' },
+  grocery:       { match_id: 100629, label: 'Tạp hoá',    group: 'other' },
+};
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+// True when the session got bounced to the anti-bot captcha (scene=crawler_item)
+// or logged out — in either case the offer API returns 404/empty and we must
+// STOP (not keep hammering) and ask the operator to re-verify via VNC.
+function isBlockedState(pageUrl, res) {
+  if (/\/verify\/captcha|\/buyer\/login|is_from_login/.test(pageUrl || '')) return true;
+  if (res && (res.status === 403 || res.status === 404)) return true;
+  return false;
+}
+
+// Apply the Flow-1 hard filters (criteria) to parsed rows. Defaults are lenient;
+// the controller passes the agreed thresholds (commission/price/sold/rating/imgs).
+function applyFilters(rows, f = {}) {
+  return rows.filter((p) => {
+    if (f.category_groups?.length && !f.category_groups.includes(p.category_group)) return false;
+    if (f.min_commission != null && (p.commission_rate || 0) < f.min_commission) return false;
+    if (f.price_min != null && (p.price || 0) < f.price_min) return false;
+    if (f.price_max != null && (p.price || 0) > f.price_max) return false;
+    if (f.min_sold != null && (p.sold_count || 0) < f.min_sold) return false;
+    if (f.min_rating != null && p.rating != null && p.rating < f.min_rating) return false;
+    if (f.min_images != null && (p.images?.length || 0) < f.min_images) return false;
+    return true;
+  });
+}
+
+// In-page fetch the affiliate offer list — anti-bot aware. Pulls one or more
+// sort_types (2=commission, 3=sales), paginates gently with a human-like delay,
+// merges + dedups by item_id, then applies hard filters. Returns
+//   { status: 'ok'|'needs_verify', products: [...], raw_count }
+// Never throws on a blocked session — returns status:'needs_verify' so the
+// worker reports it cleanly instead of failing + retrying into more captchas.
+async function scrapeOffers(browser, {
+  category = null,        // key in AFFILIATE_CATEGORIES (e.g. 'women_clothes')
+  match_id = null,        // explicit category id override
+  sort_types = [2],      // [2]=commission; pass [2,3] to merge commission+sales
+  pages = 2,
+  page_limit = 20,
+  delay_ms = 2500,       // human-like pause between calls (anti-bot)
+  filters = {},
+  timeoutMs = 45000,
+  // legacy single-sort param (back-compat with the first build)
+  sort_type = null,
+} = {}) {
+  if (sort_type != null) sort_types = [sort_type];
+  const cat = category && AFFILIATE_CATEGORIES[category] ? AFFILIATE_CATEGORIES[category] : null;
+  const mid = match_id || cat?.match_id || null;
+  // Default the category_groups filter to the category's own group (so a
+  // Women-Clothes scrape keeps only fashion rows even though the tab mixes in
+  // a few accessories), unless the caller overrides.
+  if (cat && !filters.category_groups) filters = { ...filters, category_groups: [cat.group] };
+
   const pagesOpen = await browser.pages();
   let page = pagesOpen.find((p) => p.url().includes('affiliate.shopee.vn'));
   let opened = false;
   if (!page) { page = await browser.newPage(); opened = true; }
   try {
-    if (!page.url().includes('affiliate.shopee.vn')) {
-      await page.goto('https://affiliate.shopee.vn/offer/product_offer', { waitUntil: 'networkidle2', timeout: timeoutMs });
+    if (!page.url().includes('affiliate.shopee.vn/offer')) {
+      await page.goto('https://affiliate.shopee.vn/offer/product_offer', { waitUntil: 'networkidle2', timeout: timeoutMs }).catch(() => {});
+      await sleep(1500);
+    }
+    if (isBlockedState(page.url(), null)) {
+      return { status: 'needs_verify', products: [], raw_count: 0, reason: 'session at captcha/login — re-verify via VNC' };
     }
-    const all = [];
-    for (let i = 0; i < pages; i++) {
-      const offset = i * page_limit;
-      const res = await page.evaluate(async (q) => {
-        const url = `/api/v3/offer/product/list?list_type=${q.list_type}&sort_type=${q.sort_type}&page_offset=${q.offset}&page_limit=${q.page_limit}&client_type=1`;
-        const r = await fetch(url, { credentials: 'include' });
-        return { status: r.status, json: await r.json().catch(() => null) };
-      }, { list_type, sort_type, offset, page_limit });
-      if (res.status !== 200 || res.json?.code !== 0) {
-        throw new Error(`offer list failed (http ${res.status}, code ${res.json?.code}) — session may be expired`);
+
+    const byId = new Map();
+    let rawCount = 0;
+    for (const st of sort_types) {
+      for (let i = 0; i < pages; i++) {
+        const offset = i * page_limit;
+        const res = await page.evaluate(async (q) => {
+          const cat = q.mid ? `&list_type=3&match_type=2&match_id=${q.mid}` : '&list_type=0';
+          const url = `/api/v3/offer/product/list?sort_type=${q.st}&page_offset=${q.offset}&page_limit=${q.page_limit}&client_type=1${cat}`;
+          try {
+            const r = await fetch(url, { credentials: 'include' });
+            return { status: r.status, json: await r.json().catch(() => null) };
+          } catch (e) { return { status: 0, error: String(e.message) }; }
+        }, { st, offset, page_limit, mid });
+
+        if (isBlockedState(page.url(), res) || res.json?.code === 90309999) {
+          return { status: 'needs_verify', products: [...byId.values()], raw_count: rawCount, reason: `blocked mid-scrape (http ${res.status}, code ${res.json?.code})` };
+        }
+        if (res.status !== 200 || res.json?.code !== 0) {
+          // soft stop — treat as end-of-data for this sort, move on
+          break;
+        }
+        const list = res.json?.data?.list || [];
+        rawCount += list.length;
+        for (const row of list.map(parseOfferRow)) {
+          if (row.item_id && !byId.has(row.item_id)) byId.set(row.item_id, row);
+        }
+        if (list.length < page_limit) break; // ran out
+        await sleep(delay_ms); // pace between page fetches
       }
-      const list = res.json?.data?.list || [];
-      all.push(...list.map(parseOfferRow).filter((p) => p.item_id));
-      if (list.length < page_limit) break; // ran out
+      await sleep(delay_ms); // pace between sort passes
     }
-    return all;
+
+    const products = applyFilters([...byId.values()], filters);
+    return { status: 'ok', products, raw_count: rawCount };
   } finally {
     if (opened) await page.close().catch(() => {});
   }
@@ -162,4 +244,4 @@ function parseProductUrl(url = '') {
   return null;
 }
 
-module.exports = { ingestProduct, scrapeOffers, parseProductUrl, parseOfferRow, parsePdpItem, categoryGroup, pctToNumber };
+module.exports = { ingestProduct, scrapeOffers, parseProductUrl, parseOfferRow, parsePdpItem, categoryGroup, pctToNumber, AFFILIATE_CATEGORIES, applyFilters };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.5.13",
+  "version": "2.5.15",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {

package/scripts/upload_facebook.js

@@ -180,6 +180,54 @@ async function dismissBsOnboarding(page, log) {
   }
 }
 
+// ── Large-file upload bypass ───────────────────────────────────────────────
+// Playwright caps file transfers at 50MB when the browser is reached over
+// connectOverCDP (our Nstbrowser case):
+//   "Cannot transfer files larger than 50Mb to a browser not connected to the
+//    server"
+// Reup videos are routinely 60–200MB, so the native setInputFiles()/
+// fileChooser.setFiles() throws and the upload never starts. Bypass via the raw
+// CDP command DOM.setFileInputFiles: the browser reads the path off its OWN
+// local disk (worker + Nstbrowser are co-located on the same machine), so no
+// bytes cross the wire and there is no size cap. Unlike an in-page fetch() this
+// is also immune to facebook.com's strict connect-src CSP. We tag the exact
+// <input> with a unique attribute so CDP resolves its nodeId unambiguously
+// (the page carries several file inputs).
+async function cdpSetInputFiles(page, inputHandle, filePath, log, tag) {
+  const ATTR = 'data-cm-bigfile';
+  let cdp;
+  try {
+    await inputHandle.evaluate((el, a) => el.setAttribute(a, '1'), ATTR);
+    cdp = await page.context().newCDPSession(page);
+    const { root } = await cdp.send('DOM.getDocument', { depth: 0 });
+    const { nodeId } = await cdp.send('DOM.querySelector', {
+      nodeId: root.nodeId,
+      selector: `input[${ATTR}="1"]`,
+    });
+    if (!nodeId) throw new Error('CDP querySelector: tagged input not found');
+    await cdp.send('DOM.setFileInputFiles', { files: [filePath], nodeId });
+    log('info', `[${tag}] file set via CDP DOM.setFileInputFiles (>50MB bypass)`);
+  } finally {
+    await inputHandle.evaluate((el, a) => el.removeAttribute(a), ATTR).catch(() => {});
+    if (cdp) await cdp.detach().catch(() => {});
+  }
+}
+
+// Set a video file on a file <input>, transparently bypassing the CDP 50MB cap.
+// Tries Playwright's native path first (fine for <50MB / thumbnails), then
+// falls back to the CDP command only on the size error.
+async function setVideoFile(page, inputHandle, filePath, log, tag = 'fb-pw') {
+  if (!inputHandle) throw new Error('setVideoFile: no input handle');
+  try {
+    await inputHandle.setInputFiles(filePath); // native — works for <50MB
+    return;
+  } catch (e) {
+    if (!/larger than 50\s?mb|not connected to the server/i.test(String(e.message || ''))) throw e;
+    log('info', `[${tag}] native setInputFiles hit the 50MB CDP cap — switching to DOM.setFileInputFiles…`);
+  }
+  await cdpSetInputFiles(page, inputHandle, filePath, log, tag);
+}
+
 async function run({ page, payload, log }) {
   const {
     video_url, title, description = '', tags = [],
@@ -187,7 +235,7 @@ async function run({ page, payload, log }) {
   } = payload || {};
   if (!video_url) throw new Error('No video_url provided');
 
-  log('info', '[fb-pw] selectors version=2026.06.05a-pagewall-no-double-post');
+  log('info', '[fb-pw] selectors version=2026.06.14a-bigfile-cdp');
 
   page.on('dialog', (d) => { d.accept().catch(() => {}); });
 
@@ -389,7 +437,7 @@ async function run({ page, payload, log }) {
           page.waitForEvent('filechooser', { timeout: 8000 }),
           btn.click({ timeout: 3000 }),
         ]);
-        await chooser.setFiles(videoPath);
+        await setVideoFile(page, chooser.element(), videoPath, log);
         videoSet = true;
         log('info', `[fb-pw] video file set via modal-scoped "${(await btn.innerText().catch(() => '')).slice(0, 30)}" button`);
       } catch (e) {
@@ -402,7 +450,7 @@ async function run({ page, payload, log }) {
       const fi = reelsDialog.locator("input[type='file']").last();
       if (await fi.count().catch(() => 0) > 0) {
         try {
-          await fi.setInputFiles(videoPath);
+          await setVideoFile(page, await fi.elementHandle(), videoPath, log);
           videoSet = true;
           log('info', '[fb-pw] video file set via modal-scoped fallback input[type=file]');
         } catch (e) { log('info', `[fb-pw] modal fallback input setInputFiles failed: ${e.message.slice(0, 80)}`); }
@@ -1326,9 +1374,12 @@ async function run({ page, payload, log }) {
         // output. (Observed on reel 1506614811005729: step 1 → click Tiếp
         // → publish, the thumb-edit pill never appeared.)
         if (thumbPath && !customThumbDone) {
-          await dumpInventory(page, log, `thumb-step-missed-${step + 1}`);
-          await dumpFailure(page, `thumb-step-missed-${step + 1}`, log);
-          throw new Error(`FB publish: thumbnail_url provided but the "Chỉnh sửa hình thu nhỏ" overlay was never detected before publish (step ${step + 1}). Refusing to ship without custom thumb. Inspect dump screenshots in Temp\\cm-worker-pw\\.`);
+          // Custom thumb couldn't be applied (FB didn't render the "Chỉnh sửa
+          // hình thu nhỏ" overlay). DON'T refuse to publish — the strict
+          // failure shipped false "failed" results (the reel still published)
+          // and double-posts on retry. Publish with FB's auto-thumbnail; the
+          // custom artwork is a nice-to-have, not worth a hard failure.
+          log('warn', `[fb-pw] custom thumb skipped — "Chỉnh sửa hình thu nhỏ" overlay never appeared (step ${step + 1}); publishing with FB auto-thumb`);
         }
         log('info', `[fb-pw] click publish "${pub.verb}" via "${pub.sel}" (step ${step + 1})`);
         // Snapshot the captured-IDs list RIGHT BEFORE the publish click. The