channel-worker 2.5.11 → 2.5.13

package/lib/api-client.js

@@ -68,7 +68,7 @@ class ApiClient {
   async getNextCommand(workerId) {
     // Daemon-handled types. `_pw` variants route to the Playwright pipeline
     // (lib/playwright-runner → scripts/<base>.js) instead of the extension.
-    const workerTypes = 'launch_profile,close_profile,launch_veo3_profile,set_profile_proxy,save_file,set_thumbnail,set_tags,set_file_input,click_and_upload,type_text,verify_logins,update_extension,sync_youtube_stats,restart_worker,upload_youtube_pw,upload_tiktok_pw,upload_facebook_pw';
+    const workerTypes = 'launch_profile,close_profile,launch_veo3_profile,set_profile_proxy,save_file,set_thumbnail,set_tags,set_file_input,click_and_upload,type_text,verify_logins,update_extension,sync_youtube_stats,restart_worker,upload_youtube_pw,upload_tiktok_pw,upload_facebook_pw,scrape_affiliate_products,ingest_shopee_product';
     return this.request('GET', `/workers/commands?worker_id=${workerId}&types=${encodeURIComponent(workerTypes)}`);
   }
 
@@ -84,6 +84,16 @@ class ApiClient {
     return this.request('POST', `/extension/commands/${commandId}/result`, { status, result: result || {}, error: error || null });
   }
 
+  // Shopee scraper callbacks — worker ships parsed products, API upserts into
+  // the global AffiliateProduct kho (dedup by shop_id+item_id).
+  async upsertAffiliateProducts(products, meta = {}) {
+    return this.request('POST', '/products/worker-upsert', { products, ...meta });
+  }
+  // Single PDP ingest result (Flow 2). idea_id links it back to the fashion idea.
+  async ingestShopeeResult(product, meta = {}) {
+    return this.request('POST', '/products/worker-ingest', { product, ...meta });
+  }
+
   // Return the calling daemon's own Worker doc — primarily for reading
   // parallel_limit (the per-daemon scene-generation concurrency cap that
   // replaced the legacy global flowkit_max_concurrent setting).

package/lib/command-poller.js

@@ -104,6 +104,12 @@ class CommandPoller {
         case 'set_profile_proxy':
           await this.handleSetProfileProxy(command);
           break;
+        case 'scrape_affiliate_products':
+          await this.handleScrapeAffiliateProducts(command);
+          break;
+        case 'ingest_shopee_product':
+          await this.handleIngestShopeeProduct(command);
+          break;
         default:
           // Playwright-based pipeline: any command whose type ends in '_pw'
           // is routed to scripts/<base>.js (BrowserClaw-style automation
@@ -129,6 +135,98 @@ class CommandPoller {
   // for upload + page-management tasks). Command type 'upload_youtube_pw' maps
   // to scripts/upload_youtube.js, etc. The script handles its own profile
   // launch via NST + CDP attach; the daemon orchestrates + ships the result.
+  // Lazy-init the NST manager (shared with the _pw + launch handlers).
+  async _ensureNst(command) {
+    if (this.nst) return true;
+    try {
+      const apiKey = await this.api.getSetting('nst_api_key');
+      if (apiKey) { const NstManager = require('./nst-manager'); this.nst = new NstManager(apiKey); }
+    } catch {}
+    if (!this.nst) {
+      await this.api.updateCommand(command._id, { status: 'failed', error: 'NST API key not configured' });
+      return false;
+    }
+    return true;
+  }
+
+  // Ensure a profile (by name) is running and return a puppeteer-core browser
+  // attached over CDP via its local debug port. Worker runs ON the same box as
+  // Nstbrowser, so localhost:<remoteDebuggingPort> is reachable directly.
+  async _connectNstProfileByName(name) {
+    const puppeteer = require('puppeteer-core');
+    const profileId = await this.nst.findProfile(name);
+    if (!profileId) throw new Error(`NST profile "${name}" not found`);
+    let running = (await this.nst.getRunningBrowsers()).find(b => b.profileId === profileId);
+    if (!running) {
+      await this.nst.launchProfile(profileId);
+      // brief wait for the debug port to come up
+      for (let i = 0; i < 10 && !running; i++) {
+        await new Promise(r => setTimeout(r, 1500));
+        running = (await this.nst.getRunningBrowsers()).find(b => b.profileId === profileId);
+      }
+    }
+    const port = running?.remoteDebuggingPort;
+    if (!port) throw new Error(`No debug port for profile "${name}" (launch failed?)`);
+    const browser = await puppeteer.connect({ browserURL: `http://127.0.0.1:${port}`, defaultViewport: null });
+    return { browser, disconnect: () => browser.disconnect() };
+  }
+
+  // Flow 1 — quét affiliate offer list (sort theo hoa hồng) → upsert vào kho.
+  async handleScrapeAffiliateProducts(command) {
+    const payload = command.payload || {};
+    if (!(await this._ensureNst(command))) return;
+    const scraper = require('./shopee-scraper');
+    const profileName = payload.profile_name || (await this.api.getSetting('shopee_affiliate_profile').catch(() => null)) || 'Shopee1';
+    let conn;
+    try {
+      conn = await this._connectNstProfileByName(profileName);
+      const products = await scraper.scrapeOffers(conn.browser, {
+        sort_type: payload.sort_type ?? 2,        // 2 = commission desc
+        page_limit: payload.page_limit ?? 20,
+        pages: payload.pages ?? 3,
+        list_type: payload.list_type ?? 0,
+      });
+      const filtered = payload.category_group
+        ? products.filter(p => p.category_group === payload.category_group)
+        : products;
+      console.log(`[shopee] scraped ${products.length} offers (${filtered.length} after filter) from ${profileName}`);
+      await this.api.upsertAffiliateProducts(filtered, { user_id: command.user_id });
+      await this.api.updateCommand(command._id, { status: 'done', result: { count: filtered.length, total: products.length } });
+    } catch (err) {
+      console.error(`[shopee] scrape failed: ${err.message}`);
+      await this.api.updateCommand(command._id, { status: 'failed', error: String(err.message || err).slice(0, 500) });
+    } finally {
+      if (conn) conn.disconnect();
+    }
+  }
+
+  // Flow 2 — ingest 1 sản phẩm từ link bất kỳ (bắt PDP get_pc) → full product.
+  async handleIngestShopeeProduct(command) {
+    const payload = command.payload || {};
+    if (!(await this._ensureNst(command))) return;
+    const scraper = require('./shopee-scraper');
+    const profileName = payload.profile_name || (await this.api.getSetting('shopee_shopping_profile').catch(() => null)) || 'Shopee1';
+    let ids = (payload.shop_id && payload.item_id) ? { shop_id: payload.shop_id, item_id: payload.item_id } : null;
+    if (!ids && payload.url) ids = scraper.parseProductUrl(payload.url);
+    if (!ids) {
+      await this.api.updateCommand(command._id, { status: 'failed', error: 'need {shop_id,item_id} or a parseable {url}' });
+      return;
+    }
+    let conn;
+    try {
+      conn = await this._connectNstProfileByName(profileName);
+      const product = await scraper.ingestProduct(conn.browser, ids);
+      console.log(`[shopee] ingested: ${product.name} (${product.images.length} imgs)`);
+      await this.api.ingestShopeeResult(product, { user_id: command.user_id, idea_id: payload.idea_id || null });
+      await this.api.updateCommand(command._id, { status: 'done', result: { name: product.name, images: product.images.length } });
+    } catch (err) {
+      console.error(`[shopee] ingest failed: ${err.message}`);
+      await this.api.updateCommand(command._id, { status: 'failed', error: String(err.message || err).slice(0, 500) });
+    } finally {
+      if (conn) conn.disconnect();
+    }
+  }
+
   async handlePlaywrightCommand(command) {
     const { runPlaywrightScript } = require('./playwright-runner');
     const payload = command.payload || {};

package/lib/shopee-scraper.js

@@ -0,0 +1,165 @@
+// Shopee scraper — runs against a logged-in Nstbrowser profile via CDP.
+//
+// Two jobs, both proven against the live "Shopee1" profile on relabs03:
+//   1. ingestProduct  — open a PDP, capture the api/v4/pdp/get_pc response the
+//      page itself fires (signed headers we can't forge by hand) → full product.
+//   2. scrapeOffers   — in-page fetch of affiliate.shopee.vn offer list (only
+//      needs the session cookie) → products + commission %, sorted/paginated.
+//
+// Pattern mirrors the cookie-bridge XHS capture: don't replay the API by hand,
+// let the real page sign it and grab the response at the network layer.
+
+const IMG_BASE = 'https://down-vn.img.susercontent.com/file/';
+const imgUrl = (h) => (h && !/^https?:/.test(h) ? IMG_BASE + h : h);
+
+// Map a Shopee breadcrumb / fe_category path → our coarse bucket. Keyword match
+// is intentionally loose (VN + EN) so new sub-categories still land somewhere.
+function categoryGroup(text = '') {
+  const t = text.toLowerCase();
+  if (/giày|dép|sandal|boot|sneaker|shoe/.test(t)) return 'shoes';
+  if (/son|kem|skincare|makeup|mỹ phẩm|làm đẹp|beauty|nước hoa|dưỡng/.test(t)) return 'beauty';
+  if (/túi|ví|kính|mũ|nón|trang sức|phụ kiện|đồng hồ|thắt lưng|accessor/.test(t)) return 'accessory';
+  if (/thời trang|áo|quần|váy|đầm|đồ|set|fashion|apparel|clothing|đồ ngủ|đồ lót/.test(t)) return 'fashion';
+  return 'other';
+}
+
+// "13%" / "13.5%" / 0.13 → 13 (numeric percent). Tolerant of both encodings.
+function pctToNumber(v) {
+  if (v == null) return 0;
+  if (typeof v === 'number') return v <= 1 ? +(v * 100).toFixed(2) : +v.toFixed(2);
+  const s = String(v).replace('%', '').trim();
+  const n = parseFloat(s);
+  if (!isFinite(n)) return 0;
+  return s.includes('.') && n < 1 ? +(n * 100).toFixed(2) : +n.toFixed(2);
+}
+
+function parsePdpItem(item) {
+  const cat = (item.categories || item.fe_categories || []).map((c) => c.display_name).filter(Boolean).join(' > ');
+  const cover = imgUrl(item.image);
+  const variants = (item.tier_variations || []).map((t) => ({
+    name: t.name,
+    options: t.options || [],
+    images: (t.images || []).map(imgUrl),
+  }));
+  // Gallery = cover + all variant images (deduped) — the keyframe source pool.
+  const gallery = [cover, ...variants.flatMap((v) => v.images)].filter(Boolean);
+  return {
+    shop_id: String(item.shop_id),
+    item_id: String(item.item_id),
+    name: (item.title || item.name || '').trim(),
+    description: (item.description || '').trim(),
+    brand: item.brand || '',
+    category: cat,
+    category_group: categoryGroup(cat + ' ' + (item.title || '')),
+    price: (item.price || 0) / 100000,
+    price_before_discount: (item.price_before_discount || 0) / 100000,
+    sold_count: item.historical_sold ?? item.sold ?? null,
+    rating: item.item_rating?.rating_star ?? null,
+    cover_image: cover,
+    images: [...new Set(gallery)],
+    variants,
+    attributes: (item.attributes || []).map((a) => `${a.name}: ${a.value}`).filter((s) => s !== ': '),
+    size_chart_image: imgUrl(item.size_chart),
+  };
+}
+
+// Open the PDP and resolve with the get_pc payload the page fires. We listen at
+// the response layer because a hand-rolled fetch gets error 90309999 (no sig).
+async function ingestProduct(browser, { shop_id, item_id, timeoutMs = 60000 } = {}) {
+  if (!shop_id || !item_id) throw new Error('ingestProduct needs shop_id + item_id');
+  const page = await browser.newPage();
+  try {
+    const captured = new Promise((resolve) => {
+      page.on('response', async (res) => {
+        if (!res.url().includes('/api/v4/pdp/get_pc')) return;
+        try { const j = await res.json(); if (j?.data?.item) resolve(j.data.item); } catch { /* keep waiting */ }
+      });
+      setTimeout(() => resolve(null), timeoutMs);
+    });
+    await page.goto(`https://shopee.vn/product/${shop_id}/${item_id}`, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
+    const item = await captured;
+    if (!item) throw new Error('PDP get_pc not captured (login expired or product gone)');
+    const parsed = parsePdpItem(item);
+    parsed.product_url = `https://shopee.vn/product/${shop_id}/${item_id}`;
+    return parsed;
+  } finally {
+    await page.close().catch(() => {});
+  }
+}
+
+// Parse one row of /api/v3/offer/product/list. Price fields come back as
+// STRINGS in the *100000 scale (e.g. "10000100000" = 100,001 VND), same as PDP.
+function parseOfferRow(row) {
+  const card = row.batch_item_for_item_card_full || row.item_card_displayed_asset || {};
+  const shopId = String(card.shopid || card.shop_id || '');
+  const itemId = String(row.item_id || card.itemid || card.item_id || '');
+  const rate = pctToNumber(row.default_commission_rate ?? row.max_commission_rate ?? row.commission_rate);
+  const sellerRate = pctToNumber(row.seller_commission_rate);
+  const price = (Number(card.price || card.price_min) || 0) / 100000;
+  const priceBefore = (Number(card.price_before_discount || card.price_min_before_discount) || 0) / 100000;
+  const cover = card.image ? imgUrl(card.image) : '';
+  const gallery = (card.images || []).map(imgUrl).filter(Boolean);
+  const name = (card.name || '').trim();
+  return {
+    shop_id: shopId,
+    item_id: itemId,
+    name,
+    category_group: categoryGroup(name),
+    product_url: row.product_link || (shopId && itemId ? `https://shopee.vn/product/${shopId}/${itemId}` : ''),
+    affiliate_link: row.long_link || row.offer_link || '',
+    commission_rate: rate,
+    commission_seller_rate: sellerRate,
+    commission_value: price && rate ? Math.round(price * rate / 100) : null,
+    price,
+    price_before_discount: priceBefore,
+    sold_count: card.historical_sold ?? card.sold ?? null,
+    rating: card.item_rating?.rating_star ? +card.item_rating.rating_star.toFixed(2) : null,
+    shop_name: card.shop_name || '',
+    cover_image: cover,
+    images: gallery.length ? [...new Set([cover, ...gallery])].filter(Boolean) : (cover ? [cover] : []),
+  };
+}
+
+// In-page fetch the affiliate offer list. sort_type: 1=relevance, 2=commission,
+// 3=sales (Shopee's internal codes). Loops `pages` times, page_limit per page.
+async function scrapeOffers(browser, { sort_type = 2, page_limit = 20, pages = 1, list_type = 0, timeoutMs = 45000 } = {}) {
+  const pagesOpen = await browser.pages();
+  let page = pagesOpen.find((p) => p.url().includes('affiliate.shopee.vn'));
+  let opened = false;
+  if (!page) { page = await browser.newPage(); opened = true; }
+  try {
+    if (!page.url().includes('affiliate.shopee.vn')) {
+      await page.goto('https://affiliate.shopee.vn/offer/product_offer', { waitUntil: 'networkidle2', timeout: timeoutMs });
+    }
+    const all = [];
+    for (let i = 0; i < pages; i++) {
+      const offset = i * page_limit;
+      const res = await page.evaluate(async (q) => {
+        const url = `/api/v3/offer/product/list?list_type=${q.list_type}&sort_type=${q.sort_type}&page_offset=${q.offset}&page_limit=${q.page_limit}&client_type=1`;
+        const r = await fetch(url, { credentials: 'include' });
+        return { status: r.status, json: await r.json().catch(() => null) };
+      }, { list_type, sort_type, offset, page_limit });
+      if (res.status !== 200 || res.json?.code !== 0) {
+        throw new Error(`offer list failed (http ${res.status}, code ${res.json?.code}) — session may be expired`);
+      }
+      const list = res.json?.data?.list || [];
+      all.push(...list.map(parseOfferRow).filter((p) => p.item_id));
+      if (list.length < page_limit) break; // ran out
+    }
+    return all;
+  } finally {
+    if (opened) await page.close().catch(() => {});
+  }
+}
+
+// Parse a Shopee product URL → { shop_id, item_id }. Accepts
+// /product/<shop>/<item> and i.<shop>.<item> forms.
+function parseProductUrl(url = '') {
+  let m = url.match(/\/product\/(\d+)\/(\d+)/);
+  if (m) return { shop_id: m[1], item_id: m[2] };
+  m = url.match(/i\.(\d+)\.(\d+)/);
+  if (m) return { shop_id: m[1], item_id: m[2] };
+  return null;
+}
+
+module.exports = { ingestProduct, scrapeOffers, parseProductUrl, parseOfferRow, parsePdpItem, categoryGroup, pctToNumber };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "channel-worker",
-  "version": "2.5.11",
+  "version": "2.5.13",
   "description": "Channel Manager worker daemon — runs on remote machines to execute video pipeline jobs",
   "main": "lib/daemon.js",
   "bin": {

package/scripts/upload_facebook.js

@@ -187,7 +187,7 @@ async function run({ page, payload, log }) {
   } = payload || {};
   if (!video_url) throw new Error('No video_url provided');
 
-  log('info', '[fb-pw] selectors version=2026.06.01s-pagewall-thumb-60s-retry');
+  log('info', '[fb-pw] selectors version=2026.06.05a-pagewall-no-double-post');
 
   page.on('dialog', (d) => { d.accept().catch(() => {}); });
 
@@ -820,12 +820,18 @@ async function run({ page, payload, log }) {
       // attach a custom thumbnail in the Page-wall composer.
       if (!customThumbDone && thumbPath && /^https?:\/\/(www\.)?facebook\.com\/?($|\?|#)/.test(page.url())) {
         const editBtn = await page.evaluate(() => {
-          // Look for the thumbnail-edit button inside ANY visible composer
-          // dialog. The overlay is a small button on the video preview with:
-          //   - aria-label containing "hình thu nhỏ" / "thumbnail"
-          //   - OR text "Chỉnh sửa" with size < ~120px wide (overlay vs full
-          //     "Chỉnh sửa ảnh bìa" page cover button which is much wider)
-          const dlgs = document.querySelectorAll("[role='dialog']");
+          // Look for the thumbnail-edit button inside the REEL COMPOSER dialog
+          // only — scanning all dialogs risked matching a button on the
+          // page-wall photo composer (which would post a separate "Tin dạng
+          // ảnh" alongside the Reel). The Reel composer dialog contains the
+          // header text "Tạo thước phim" / "Create reel".
+          const allDlgs = document.querySelectorAll("[role='dialog']");
+          const dlgs = [];
+          for (const d of allDlgs) {
+            const txt = (d.innerText || '').slice(0, 400);
+            if (/Tạo thước phim|Create reel|Create a reel/i.test(txt)) dlgs.push(d);
+          }
+          if (dlgs.length === 0) return null;
           for (const dlg of dlgs) {
             const r = dlg.getBoundingClientRect();
             if (r.width < 8 || r.height < 8) continue;
@@ -1001,9 +1007,15 @@ async function run({ page, payload, log }) {
       }
 
       // Thumbnail step handling (BS composer legacy).
-      const onThumbStep = await firstVisible(page.locator(
+      // SAFETY GATE — only run on business.facebook.com. On the Page-wall flow
+      // (facebook.com root), the page-wide setInputFiles selectors below would
+      // match the regular "Tạo bài viết" photo composer's hidden file input
+      // and accidentally publish a separate "Tin dạng ảnh" post alongside the
+      // Reel (observed bug: 2 posts at same minute — one image-only, one Reel).
+      const isBSComposer = /^https?:\/\/business\.facebook\.com/.test(page.url());
+      const onThumbStep = isBSComposer ? await firstVisible(page.locator(
         "[aria-label='Hình thu nhỏ tạo tự động 1'], [aria-label*='Auto-generated thumbnail'], div[role='button']:has-text('Tải hình ảnh lên'), div[role='button']:has-text('Upload image')"
-      ), 3);
+      ), 3) : null;
       if (onThumbStep && !customThumbDone) {
         if (thumbPath) {
           log('info', '[fb-pw] thumbnail step — uploading custom thumb…');

package/scripts/upload_youtube.js

@@ -515,8 +515,41 @@ async function run({ page, payload, log }) {
     await jitter(page, 800);
 
     // 5) The KEY step — set the video file.
-    log('info', `[yt-pw] setInputFiles(video)…`);
-    await page.locator(S.fileInput).setInputFiles(videoPath);
+    //
+    // Playwright's setInputFiles() over connectOverCDP caps at 50MB. Real
+    // shorts are 100–300MB. Bypass via in-browser fetch: the browser pulls
+    // the file directly from our media server (CORS-open on redrop) and we
+    // assign the resulting Blob to the input through DataTransfer. The CDP
+    // channel never carries the bytes. Falls back to Playwright's native
+    // setInputFiles if the browser fetch fails (small files, missing URL).
+    log('info', `[yt-pw] setInputFiles(video) via in-browser fetch (bypass CDP 50MB cap)…`);
+    let injected = false;
+    if (video_url) {
+      try {
+        injected = await page.evaluate(async ({ url, selector }) => {
+          const r = await fetch(url, { credentials: 'omit' });
+          if (!r.ok) throw new Error(`fetch ${r.status}`);
+          const blob = await r.blob();
+          if (!blob || !blob.size) throw new Error('empty blob');
+          const fname = (url.split('?')[0].split('/').pop()) || 'video.mp4';
+          const file = new File([blob], fname, { type: blob.type || 'video/mp4' });
+          const inputs = Array.from(document.querySelectorAll(selector));
+          const input = inputs.find((el) => /video/i.test(el.accept || '')) || inputs[0];
+          if (!input) throw new Error('input not found');
+          const dt = new DataTransfer();
+          dt.items.add(file);
+          input.files = dt.files;
+          input.dispatchEvent(new Event('change', { bubbles: true }));
+          return true;
+        }, { url: video_url, selector: S.fileInput });
+        log('info', `[yt-pw] in-browser fetch upload OK`);
+      } catch (e) {
+        log('warn', `[yt-pw] in-browser fetch upload failed: ${String(e.message || e).slice(0, 160)} — falling back to Playwright setInputFiles`);
+      }
+    }
+    if (!injected) {
+      await page.locator(S.fileInput).setInputFiles(videoPath);
+    }
     log('info', `[yt-pw] file set; waiting for the metadata form to appear…`);
 
     // 6) Wait for the Title field to be ready (Studio renders it as the file