@iducky/media-agent 1.1.0

package/src/media_agent/platforms/douyin/douyin_hotspot_v2.py

@@ -0,0 +1,454 @@
+"""Legacy operation, executed only through main()."""
+
+def main():
+    #!/usr/bin/env python3
+    """Douyin热点数据完整采集 v2 - 独立容器分页 + 趋势修复"""
+    from media_agent.runtime.paths import runtime_home
+    import json, time, sys, os, csv
+    from pathlib import Path
+    from datetime import datetime, timezone, timedelta
+    from media_agent.runtime.browser import launch_persistent_context
+
+    BASE = runtime_home()
+    ALIAS = 'douyin_creator_personal_test'
+    PROFILE_DIR = BASE / 'profiles' / ALIAS
+    BROWSER_DATA = str(PROFILE_DIR / 'browser_data')
+    LOCK_FILE = BASE / 'locks' / f'{ALIAS}.lock'
+    ts = datetime.now().strftime("%Y%m%d_%H%M%S")
+    TASK_ID = f'hotspot_v2_{ALIAS}_{ts}'
+    OUT_DIR = BASE / 'tasks' / 'douyin_index' / ts
+    OUT_DIR.mkdir(parents=True, exist_ok=True)
+
+    # ===== Lock =====
+    LOCK_FILE.parent.mkdir(parents=True, exist_ok=True)
+    if LOCK_FILE.exists():
+        try:
+            d = json.loads(LOCK_FILE.read_text()); pid = d.get('pid')
+            if pid:
+                try: os.kill(pid, 0); print('ERROR|Lock active', flush=True); sys.exit(1)
+                except OSError: LOCK_FILE.unlink()
+        except: LOCK_FILE.unlink()
+    LOCK_FILE.write_text(json.dumps({'task_id':TASK_ID,'pid':os.getpid()}, indent=2))
+    print(f'LOCK|{TASK_ID}', flush=True)
+
+    # ===== Launch =====
+    cfg = json.loads((PROFILE_DIR / 'config.json').read_text())
+    ctx = launch_persistent_context(user_data_dir=BROWSER_DATA, headless=False,
+        stealth_args=True, viewport=cfg['fingerprint']['viewport'],
+        locale=cfg['fingerprint']['locale'], timezone=cfg['fingerprint']['timezone'],
+        humanize=False, geoip=False)
+    page = ctx.pages[0] if ctx.pages else ctx.new_page()
+
+    # ===== Navigate =====
+    page.goto('https://creator.douyin.com/', wait_until='load', timeout=60000)
+    time.sleep(5)
+    create_center = page.locator('text=创作中心').first
+    if create_center and create_center.is_visible(): create_center.click(); time.sleep(2)
+    hot_index = page.locator('text=抖音指数').first
+    if hot_index and hot_index.is_visible(): hot_index.click(); time.sleep(5)
+    try:
+        confirm = page.locator('text=确认').first
+        if confirm and confirm.is_visible(timeout=3000): confirm.click(); time.sleep(3)
+    except: pass
+
+    collected_at = datetime.now(timezone(timedelta(hours=8))).isoformat()
+    source_url = page.url
+    page_title = page.title()
+    print(f'URL|{source_url}', flush=True)
+
+    # ===== Stage 1: Inspect pagination structure =====
+    print('=== STAGE 1: INSPECT ===', flush=True)
+    inspect = page.evaluate('''() => {
+        const result = {};
+        // Count titles
+        const titles = document.querySelectorAll('*');
+        let rtCount = 0, rsCount = 0;
+        for (const el of titles) {
+            if (el.textContent.trim() === '抖音实时热点') rtCount++;
+            if (el.textContent.trim() === '抖音飙升热点') rsCount++;
+        }
+        result.title_rt_count = rtCount;
+        result.title_rs_count = rsCount;
+
+        // Find pagination areas
+        const paginations = document.querySelectorAll('[class*="pagination"], [class*="page-list"], [class*="pager"]');
+        result.pagination_count = paginations.length;
+        result.paginations = [];
+        for (const p of paginations) {
+            const rect = p.getBoundingClientRect();
+            result.paginations.push({
+                x: Math.round(rect.x), y: Math.round(rect.y),
+                w: Math.round(rect.width), h: Math.round(rect.height),
+                visible: rect.width > 0 && rect.height > 0,
+                text: p.textContent.trim().substring(0, 50)
+            });
+        }
+
+        // Find rising title and its ancestor containers
+        const rsTitle = Array.from(document.querySelectorAll('*')).find(el => el.textContent.trim() === '抖音飙升热点');
+        if (rsTitle) {
+            result.rs_title_found = true;
+            let parent = rsTitle.parentElement;
+            for (let i = 0; i < 10 && parent; i++) {
+                const hasTable = parent.querySelector('table') || parent.querySelector('[role="table"]');
+                const hasPagination = parent.querySelector('[class*="pagination"], [class*="page-list"], [class*="pager"]');
+                const hasLi = parent.querySelectorAll('li').length;
+                const rect = parent.getBoundingClientRect();
+                if (hasTable || hasPagination || hasLi > 3) {
+                    result.rs_ancestor = {
+                        level: i,
+                        tag: parent.tagName,
+                        className: parent.className?.substring(0, 80),
+                        hasTable: !!hasTable,
+                        hasPagination: !!hasPagination,
+                        liCount: hasLi,
+                        x: Math.round(rect.x), y: Math.round(rect.y),
+                        w: Math.round(rect.width)
+                    };
+                    if (hasPagination) break;
+                }
+                parent = parent.parentElement;
+            }
+        }
+
+        return result;
+    }''')
+    print(f'INSPECT|{json.dumps(inspect, ensure_ascii=False)}', flush=True)
+
+    # ===== Stage 2: Extract with per-container pagination =====
+    print('=== STAGE 2: EXTRACT ===', flush=True)
+
+    def parse_index(text):
+        t = text.strip()
+        if '万' in t:
+            return round(float(t.replace('万', '').strip()) * 10000), t
+        try: return round(float(t.replace(',', ''))), t
+        except: return 0, t
+
+    def parse_trend_dom(page):
+        """Parse trend from DOM using JS to inspect icon/SVG/class."""
+        return page.evaluate('''() => {
+            const result = { realtime: [], rising: [] };
+            // Find all trend cells
+            const allCells = document.querySelectorAll('td');
+            for (const td of allCells) {
+                const text = td.textContent.trim();
+                if (text === '--' || text === '-') {
+                    // Determine which table this belongs to
+                    const parent = td.closest('table, [role="table"], [class*="table"]');
+                    const parentText = parent ? parent.textContent.substring(0, 50) : '';
+                    const listType = parentText.includes('飙升') ? 'rising' : 'realtime';
+                    const row = td.closest('tr');
+                    const rankCell = row ? row.querySelector('td:first-child, [class*="rank"]') : null;
+                    const rank = rankCell ? parseInt(rankCell.textContent.trim()) : 0;
+                    if (rank > 0) {
+                        result[listType].push({rank, trend: 'flat'});
+                    }
+                }
+                // Check for up/down arrow
+                const svg = td.querySelector('svg');
+                if (svg) {
+                    const svgClass = svg.getAttribute('class') || '';
+                    const svgStyle = svg.getAttribute('style') || '';
+                    const parent = td.closest('table, [role="table"], [class*="table"]');
+                    const parentText = parent ? parent.textContent.substring(0, 50) : '';
+                    const listType = parentText.includes('飙升') ? 'rising' : 'realtime';
+                    const row = td.closest('tr');
+                    const rankCell = row ? row.querySelector('td:first-child, [class*="rank"]') : null;
+                    const rank = rankCell ? parseInt(rankCell.textContent.trim()) : 0;
+                    if (rank > 0) {
+                        if (svgClass.includes('up') || svgClass.includes('rise') || svgStyle.includes('deg(0)') || svgStyle.includes('deg(180)')) {
+                            result[listType].push({rank, trend: 'up'});
+                        } else if (svgClass.includes('down') || svgClass.includes('fall') || svgStyle.includes('rotate(180)')) {
+                            result[listType].push({rank, trend: 'down'});
+                        }
+                    }
+                }
+                // Check color
+                const color = window.getComputedStyle(td).color;
+                if (color === 'rgb(255, 0, 0)' || color === 'rgb(229, 28, 35)') {
+                    const parent = td.closest('table, [role="table"], [class*="table"]');
+                    const parentText = parent ? parent.textContent.substring(0, 50) : '';
+                    const listType = parentText.includes('飙升') ? 'rising' : 'realtime';
+                    const row = td.closest('tr');
+                    const rankCell = row ? row.querySelector('td:first-child, [class*="rank"]') : null;
+                    const rank = rankCell ? parseInt(rankCell.textContent.trim()) : 0;
+                    if (rank > 0) result[listType].push({rank, trend: 'up'});
+                }
+            }
+            return result;
+        }''')
+
+    def extract_page_data(page):
+        """Extract items from current page for both tables."""
+        return page.evaluate('''() => {
+            const result = { realtime: [], rising: [] };
+            const all = document.body.innerText;
+
+            function parseSection(text) {
+                const lines = text.split('\\n').map(l => l.trim()).filter(l => l);
+                const items = [];
+                let i = 0;
+                while (i < lines.length) {
+                    const line = lines[i];
+                    if (['排名','热点名称','热点指数','热点指数变化','共30条记录','1','2','3','抖音实时热点','抖音飙升热点'].includes(line)) {
+                        i++; continue;
+                    }
+                    let rank = 0, name = '', idxText = '';
+                    if (/^\\d{1,2}$/.test(line)) {
+                        rank = parseInt(line); i++;
+                        if (i >= lines.length) break;
+                        name = lines[i]; i++;
+                        if (i >= lines.length) break;
+                        idxText = lines[i]; i++;
+                        // Skip trend line
+                        if (i < lines.length && /^[▲▼↑↓\\-]{1,2}$/.test(lines[i])) i++;
+                    } else {
+                        rank = items.length + 1;
+                        name = line; i++;
+                        if (i >= lines.length) break;
+                        idxText = lines[i]; i++;
+                        if (i < lines.length && /^[▲▼↑↓\\-]{1,2}$/.test(lines[i])) i++;
+                    }
+                    if (rank > 0 && name && idxText) {
+                        items.push({rank, name, idxText});
+                    }
+                }
+                return items;
+            }
+
+            const rtIdx = all.indexOf('抖音实时热点');
+            const rsIdx = all.indexOf('抖音飙升热点');
+            if (rtIdx > -1) {
+                const end = rsIdx > -1 ? rsIdx : all.length;
+                result.realtime = parseSection(all.substring(rtIdx, end));
+            }
+            if (rsIdx > -1) {
+                result.rising = parseSection(all.substring(rsIdx));
+            }
+            return result;
+        }''')
+
+    def click_pagination(page, page_num, side='left'):
+        """Click pagination by side. side='left' for realtime, 'right' for rising."""
+        return page.evaluate('''(args) => {
+            const num = args[0], side = args[1];
+            const lis = Array.from(document.querySelectorAll('li'));
+            const pageLis = lis.filter(li => {
+                const t = li.textContent.trim();
+                return t === String(num) && li.offsetParent !== null && li.offsetParent.offsetParent !== null;
+            });
+            if (pageLis.length === 0) return 'not_found';
+            if (side === 'right') {
+                pageLis.sort((a, b) => b.getBoundingClientRect().x - a.getBoundingClientRect().x);
+            } else {
+                pageLis.sort((a, b) => a.getBoundingClientRect().x - b.getBoundingClientRect().x);
+            }
+            pageLis[0].click();
+            return 'clicked_' + side;
+        }''', [page_num, side])
+
+    def wait_for_page_change(page, prev_first_item, timeout=10):
+        """Wait until the first item in the table changes."""
+        for i in range(timeout * 2):
+            time.sleep(0.5)
+            data = extract_page_data(page)
+            curr_first = data['rising'][0]['name'] if data['rising'] else ''
+            if curr_first and curr_first != prev_first_item:
+                return True
+        return False
+
+    # Collect all pages - separate pagination for each table
+    all_rt = []
+    all_rs = []
+
+    # First collect realtime (pages 1-3)
+    for pg in [1, 2, 3]:
+        print(f'=== RT PAGE {pg} ===', flush=True)
+        if pg > 1:
+            result = click_pagination(page, pg, 'left')
+            print(f'CLICK_RT|{pg}|{result}', flush=True)
+            time.sleep(5)
+        data = extract_page_data(page)
+        print(f'RT_PAGE_{pg}|{len(data["realtime"])} items|first={data["realtime"][0]["name"][:40] if data["realtime"] else "none"}', flush=True)
+        for item in data['realtime']:
+            idx_num, idx_disp = parse_index(item['idxText'])
+            all_rt.append({
+                'list_type': 'realtime', 'rank': item['rank'],
+                'hot_name': item['name'], 'hot_index': idx_num,
+                'hot_index_display': idx_disp, 'trend': 'unknown',
+                'collected_at': collected_at, 'source_url': source_url
+            })
+
+    # Go back to page 1 for rising
+    click_pagination(page, 1, 'left')
+    time.sleep(3)
+
+    # Then collect rising (pages 1-3)
+    for pg in [1, 2, 3]:
+        print(f'=== RS PAGE {pg} ===', flush=True)
+        if pg > 1:
+            result = click_pagination(page, pg, 'right')
+            print(f'CLICK_RS|{pg}|{result}', flush=True)
+            time.sleep(5)
+        data = extract_page_data(page)
+        print(f'RS_PAGE_{pg}|{len(data["rising"])} items|first={data["rising"][0]["name"][:40] if data["rising"] else "none"}', flush=True)
+        for item in data['rising']:
+            idx_num, idx_disp = parse_index(item['idxText'])
+            all_rs.append({
+                'list_type': 'rising', 'rank': item['rank'],
+                'hot_name': item['name'], 'hot_index': idx_num,
+                'hot_index_display': idx_disp, 'trend': 'unknown',
+                'collected_at': collected_at, 'source_url': source_url
+            })
+
+    # ===== Stage 3: Trend detection =====
+    print('=== STAGE 3: TREND ===', flush=True)
+    trends = parse_trend_dom(page)
+    print(f'TREND_DETECTED|RT={len(trends["realtime"])}|RS={len(trends["rising"])}', flush=True)
+
+    # Apply trends
+    for item in all_rt:
+        for t in trends['realtime']:
+            if t['rank'] == item['rank']:
+                item['trend'] = t['trend']
+                break
+    for item in all_rs:
+        for t in trends['rising']:
+            if t['rank'] == item['rank']:
+                item['trend'] = t['trend']
+                break
+
+    # ===== Validation =====
+    errors = []
+    # Deduplicate
+    seen_rt = set()
+    all_rt_dedup = []
+    for r in all_rt:
+        if r['rank'] not in seen_rt:
+            seen_rt.add(r['rank']); all_rt_dedup.append(r)
+    seen_rs = set()
+    all_rs_dedup = []
+    for r in all_rs:
+        if r['rank'] not in seen_rs:
+            seen_rs.add(r['rank']); all_rs_dedup.append(r)
+
+    all_rt = all_rt_dedup; all_rs = all_rs_dedup
+    all_rt.sort(key=lambda x: x['rank']); all_rs.sort(key=lambda x: x['rank'])
+
+    truncated = sum(1 for r in all_rt + all_rs if r['hot_name'].endswith('...') or r['hot_name'].endswith('…'))
+    unknown_trend = sum(1 for r in all_rt + all_rs if r['trend'] == 'unknown')
+
+    if len(all_rt) < 30: errors.append(f'realtime: {len(all_rt)}/30')
+    if len(all_rs) < 30: errors.append(f'rising: {len(all_rs)}/30')
+    for lst, label in [(all_rt,'realtime'),(all_rs,'rising')]:
+        ranks = set()
+        for r in lst:
+            if r['rank'] in ranks: errors.append(f'{label} dup rank {r["rank"]}')
+            ranks.add(r['rank'])
+            if not r['hot_name']: errors.append(f'{label} rank {r["rank"]} empty name')
+        for rk in range(1, 31):
+            if rk not in ranks: errors.append(f'{label} missing rank {rk}')
+
+    completeness = 'complete' if len(all_rt) >= 30 and len(all_rs) >= 30 else 'partial'
+
+    print(f'TOTAL|RT={len(all_rt)}|RS={len(all_rs)}|TRUNC={truncated}|UNKNOWN={unknown_trend}', flush=True)
+    print(f'COMPLETENESS|{completeness}', flush=True)
+
+    # ===== Save =====
+    output = {
+        'source': 'douyin_creator_center', 'page': 'douyin_index',
+        'profile_id': ALIAS, 'collected_at': collected_at,
+        'source_url': source_url,
+        'lists': {'realtime': all_rt, 'rising': all_rs},
+        'errors': errors
+    }
+
+    json_path = OUT_DIR / 'douyin_hotspots.json'
+    json_path.write_text(json.dumps(output, indent=2, ensure_ascii=False))
+
+    csv_path = OUT_DIR / 'douyin_hotspots.csv'
+    with open(csv_path, 'w', newline='', encoding='utf-8') as f:
+        w = csv.writer(f)
+        w.writerow(['list_type','rank','hot_name','hot_index','hot_index_display','trend','collected_at','source_url'])
+        for lt in [all_rt, all_rs]:
+            for r in lt:
+                w.writerow([r['list_type'],r['rank'],r['hot_name'],r['hot_index'],r['hot_index_display'],r['trend'],r['collected_at'],r['source_url']])
+
+    csv_lines = sum(1 for _ in open(csv_path)) - 1
+    if csv_lines != len(all_rt) + len(all_rs):
+        errors.append(f'CSV({csv_lines}) != JSON({len(all_rt)+len(all_rs)})')
+
+    meta = {
+        'profile_id': ALIAS, 'page_title': page_title, 'source_url': source_url,
+        'collected_at': collected_at,
+        'displayed_realtime_total': 30, 'displayed_rising_total': 30,
+        'collected_realtime_count': len(all_rt), 'collected_rising_count': len(all_rs),
+        'truncated_name_count': truncated, 'unknown_trend_count': unknown_trend,
+        'completeness': completeness, 'errors': errors,
+        'json_path': str(json_path), 'csv_path': str(csv_path),
+        'screenshot_path': str(OUT_DIR / 'douyin_hotspots.png'),
+        'report_path': str(OUT_DIR / 'douyin_hotspots_report.md')
+    }
+    meta_path = OUT_DIR / 'douyin_hotspots.meta.json'
+    meta_path.write_text(json.dumps(meta, indent=2, ensure_ascii=False))
+
+    # Screenshot
+    ss = OUT_DIR / 'douyin_hotspots.png'
+    page.evaluate("""() => {
+        const imgs = document.querySelectorAll('img');
+        for (const img of imgs) {
+            if (img.width < 100 && img.height < 100) { img.style.filter = 'blur(8px)'; img.style.opacity = '0.3'; }
+        }
+    }""")
+    time.sleep(0.5)
+    page.screenshot(path=str(ss))
+
+    # Report
+    report = [
+        f'# 抖音指数与实时热点日报｜{datetime.now(timezone(timedelta(hours=8))).strftime("%Y-%m-%d %H:%M")}',
+        '', '## 采集信息',
+        f'- 采集时间: {collected_at}', f'- 数据来源: {source_url}',
+        f'- Profile: {ALIAS}', f'- 实时热点: {len(all_rt)} 条', f'- 飙升热点: {len(all_rs)} 条',
+        f'- 完整性: {completeness}', f'- 截断标题: {truncated} 条', f'- 未知趋势: {unknown_trend} 条',
+        '', '## 抖音实时热点',
+        '| 排名 | 热点名称 | 热点指数 | 指数数值 | 趋势 |',
+        '|------|---------|---------|---------|------|',
+    ]
+    for r in all_rt:
+        report.append(f'| {r["rank"]} | {r["hot_name"]} | {r["hot_index_display"]} | {r["hot_index"]} | {r["trend"]} |')
+    report.extend([
+        '', '## 抖音飙升热点',
+        '| 排名 | 热点名称 | 热点指数 | 指数数值 | 趋势 |',
+        '|------|---------|---------|---------|------|',
+    ])
+    for r in all_rs:
+        report.append(f'| {r["rank"]} | {r["hot_name"]} | {r["hot_index_display"]} | {r["hot_index"]} | {r["trend"]} |')
+    report.extend([
+        '', '## 数据质量',
+        f'- 截断标题: {truncated}', f'- 未知趋势: {unknown_trend}',
+        f'- 异常: {errors if errors else "无"}',
+        '', '## 文件',
+        f'- JSON: {json_path}', f'- CSV: {csv_path}',
+        f'- Meta: {meta_path}', f'- 截图: {ss}',
+    ])
+    report_path = OUT_DIR / 'douyin_hotspots_report.md'
+    report_path.write_text('\n'.join(report))
+
+    print(f'JSON|{json_path}', flush=True)
+    print(f'CSV|{csv_path}', flush=True)
+    print(f'META|{meta_path}', flush=True)
+    print(f'SS|{ss}', flush=True)
+    print(f'REPORT|{report_path}', flush=True)
+
+    if completeness == 'complete' and not errors:
+        print('FINAL|READY_FOR_FEISHU', flush=True)
+    else:
+        print('FINAL|PARTIAL', flush=True)
+
+    ctx.close()
+    LOCK_FILE.unlink(missing_ok=True)
+    print('DONE', flush=True)
+
+if __name__ == '__main__':
+    main()