@jackwener/opencli 1.6.1 → 1.6.2

package/src/clis/xiaoe/catalog.yaml

@@ -0,0 +1,129 @@
+site: xiaoe
+name: catalog
+description: 小鹅通课程目录（支持普通课程、专栏、大专栏）
+domain: h5.xet.citv.cn
+strategy: cookie
+
+args:
+  url:
+    type: str
+    required: true
+    positional: true
+    description: 课程页面 URL
+
+pipeline:
+  - navigate: ${{ args.url }}
+
+  - wait: 8
+
+  - evaluate: |
+      (async () => {
+        var el = document.querySelector('#app');
+        var store = (el && el.__vue__) ? el.__vue__.$store : null;
+        if (!store) return [];
+        var coreInfo = store.state.coreInfo || {};
+        var resourceType = coreInfo.resource_type || 0;
+        var origin = window.location.origin;
+        var courseName = coreInfo.resource_name || '';
+
+        function typeLabel(t) {
+          return {1:'图文',2:'直播',3:'音频',4:'视频',6:'专栏',8:'大专栏'}[Number(t)] || String(t||'');
+        }
+        function buildUrl(item) {
+          var u = item.jump_url || item.h5_url || item.url || '';
+          return (u && !u.startsWith('http')) ? origin + u : u;
+        }
+        function clickTab(name) {
+          var tabs = document.querySelectorAll('span, div');
+          for (var i = 0; i < tabs.length; i++) {
+            if (tabs[i].children.length === 0 && tabs[i].textContent.trim() === name) {
+              tabs[i].click(); return;
+            }
+          }
+        }
+
+        clickTab('目录');
+        await new Promise(function(r) { setTimeout(r, 2000); });
+
+        // ===== 专栏 / 大专栏 =====
+        if (resourceType === 6 || resourceType === 8) {
+          await new Promise(function(r) { setTimeout(r, 1000); });
+          var listData = [];
+          var walkList = function(vm, depth) {
+            if (!vm || depth > 6 || listData.length > 0) return;
+            var d = vm.$data || {};
+            var keys = ['columnList', 'SingleItemList', 'chapterChildren'];
+            for (var ki = 0; ki < keys.length; ki++) {
+              var arr = d[keys[ki]];
+              if (arr && Array.isArray(arr) && arr.length > 0 && arr[0].resource_id) {
+                for (var j = 0; j < arr.length; j++) {
+                  var item = arr[j];
+                  if (!item.resource_id || !/^[pvlai]_/.test(item.resource_id)) continue;
+                  listData.push({
+                    ch: 1, chapter: courseName, no: j + 1,
+                    title: item.resource_title || item.title || item.chapter_title || '',
+                    type: typeLabel(item.resource_type || item.chapter_type),
+                    resource_id: item.resource_id,
+                    url: buildUrl(item),
+                    status: item.finished_state === 1 ? '已完成' : (item.resource_count ? item.resource_count + '节' : ''),
+                  });
+                }
+                return;
+              }
+            }
+            if (vm.$children) {
+              for (var c = 0; c < vm.$children.length; c++) walkList(vm.$children[c], depth + 1);
+            }
+          };
+          walkList(el.__vue__, 0);
+          return listData;
+        }
+
+        // ===== 普通课程 =====
+        var chapters = document.querySelectorAll('.chapter_box');
+        for (var ci = 0; ci < chapters.length; ci++) {
+          var vue = chapters[ci].__vue__;
+          if (vue && typeof vue.getSecitonList === 'function' && (!vue.isShowSecitonsList || !vue.chapterChildren.length)) {
+            if (vue.isShowSecitonsList) vue.isShowSecitonsList = false;
+            try { vue.getSecitonList(); } catch(e) {}
+            await new Promise(function(r) { setTimeout(r, 1500); });
+          }
+        }
+        await new Promise(function(r) { setTimeout(r, 3000); });
+
+        var result = [];
+        chapters = document.querySelectorAll('.chapter_box');
+        for (var cj = 0; cj < chapters.length; cj++) {
+          var v = chapters[cj].__vue__;
+          if (!v) continue;
+          var chTitle = (v.chapterItem && v.chapterItem.chapter_title) || '';
+          var children = v.chapterChildren || [];
+          for (var ck = 0; ck < children.length; ck++) {
+            var child = children[ck];
+            var resId = child.resource_id || child.chapter_id || '';
+            var chType = child.chapter_type || child.resource_type || 0;
+            var urlPath = {1:'/v1/course/text/',2:'/v2/course/alive/',3:'/v1/course/audio/',4:'/v1/course/video/'}[Number(chType)];
+            result.push({
+              ch: cj + 1, chapter: chTitle, no: ck + 1,
+              title: child.chapter_title || child.resource_title || '',
+              type: typeLabel(chType),
+              resource_id: resId,
+              url: urlPath ? origin + urlPath + resId + '?type=2' : '',
+              status: child.is_finish === 1 ? '已完成' : (child.learn_progress > 0 ? child.learn_progress + '%' : '未学'),
+            });
+          }
+        }
+        return result;
+      })()
+
+  - map:
+      ch: ${{ item.ch }}
+      chapter: ${{ item.chapter }}
+      no: ${{ item.no }}
+      title: ${{ item.title }}
+      type: ${{ item.type }}
+      resource_id: ${{ item.resource_id }}
+      url: ${{ item.url }}
+      status: ${{ item.status }}
+
+columns: [ch, chapter, no, title, type, resource_id, status]

package/src/clis/xiaoe/content.yaml

@@ -0,0 +1,43 @@
+site: xiaoe
+name: content
+description: 提取小鹅通图文页面内容为文本
+domain: h5.xet.citv.cn
+strategy: cookie
+
+args:
+  url:
+    type: str
+    required: true
+    positional: true
+    description: 页面 URL
+
+pipeline:
+  - navigate: ${{ args.url }}
+
+  - wait: 6
+
+  - evaluate: |
+      (() => {
+        var selectors = ['.rich-text-wrap','.content-wrap','.article-content','.text-content',
+          '.course-detail','.detail-content','[class*="richtext"]','[class*="rich-text"]','.ql-editor'];
+        var content = '';
+        for (var i = 0; i < selectors.length; i++) {
+          var el = document.querySelector(selectors[i]);
+          if (el && el.innerText.trim().length > 50) { content = el.innerText.trim(); break; }
+        }
+        if (!content) content = (document.querySelector('main') || document.querySelector('#app') || document.body).innerText.trim();
+
+        var images = [];
+        document.querySelectorAll('img').forEach(function(img) {
+          if (img.src && !img.src.startsWith('data:') && img.src.includes('xiaoe')) images.push(img.src);
+        });
+        return [{
+          title: document.title,
+          content: content,
+          content_length: content.length,
+          image_count: images.length,
+          images: JSON.stringify(images.slice(0, 20)),
+        }];
+      })()
+
+columns: [title, content_length, image_count]

package/src/clis/xiaoe/courses.yaml

@@ -0,0 +1,73 @@
+site: xiaoe
+name: courses
+description: 列出已购小鹅通课程（含 URL 和店铺名）
+domain: study.xiaoe-tech.com
+strategy: cookie
+
+pipeline:
+  - navigate: https://study.xiaoe-tech.com/
+
+  - wait: 8
+
+  - evaluate: |
+      (async () => {
+        // 切换到「内容」tab
+        var tabs = document.querySelectorAll('span, div');
+        for (var i = 0; i < tabs.length; i++) {
+          if (tabs[i].children.length === 0 && tabs[i].textContent.trim() === '内容') {
+            tabs[i].click();
+            break;
+          }
+        }
+        await new Promise(function(r) { setTimeout(r, 2000); });
+
+        // 匹配课程卡片标题与 Vue 数据
+        function matchEntry(title, vm, depth) {
+          if (!vm || depth > 5) return null;
+          var d = vm.$data || {};
+          for (var k in d) {
+            if (!Array.isArray(d[k])) continue;
+            for (var j = 0; j < d[k].length; j++) {
+              var e = d[k][j];
+              if (!e || typeof e !== 'object') continue;
+              var t = e.title || e.resource_name || '';
+              if (t && title.includes(t.substring(0, 10))) return e;
+            }
+          }
+          return vm.$parent ? matchEntry(title, vm.$parent, depth + 1) : null;
+        }
+
+        // 构造课程 URL
+        function buildUrl(entry) {
+          if (entry.h5_url) return entry.h5_url;
+          if (entry.url) return entry.url;
+          if (entry.app_id && entry.resource_id) {
+            var base = 'https://' + entry.app_id + '.h5.xet.citv.cn';
+            if (entry.resource_type === 6) return base + '/v1/course/column/' + entry.resource_id + '?type=3';
+            return base + '/p/course/ecourse/' + entry.resource_id;
+          }
+          return '';
+        }
+
+        var cards = document.querySelectorAll('.course-card-list');
+        var results = [];
+        for (var c = 0; c < cards.length; c++) {
+          var titleEl = cards[c].querySelector('.card-title-box');
+          var title = titleEl ? titleEl.textContent.trim() : '';
+          if (!title) continue;
+          var entry = matchEntry(title, cards[c].__vue__, 0);
+          results.push({
+            title: title,
+            shop: entry ? (entry.shop_name || entry.app_name || '') : '',
+            url: entry ? buildUrl(entry) : '',
+          });
+        }
+        return results;
+      })()
+
+  - map:
+      title: ${{ item.title }}
+      shop: ${{ item.shop }}
+      url: ${{ item.url }}
+
+columns: [title, shop, url]

package/src/clis/xiaoe/detail.yaml

@@ -0,0 +1,39 @@
+site: xiaoe
+name: detail
+description: 小鹅通课程详情（名称、价格、学员数、店铺）
+domain: h5.xet.citv.cn
+strategy: cookie
+
+args:
+  url:
+    type: str
+    required: true
+    positional: true
+    description: 课程页面 URL
+
+pipeline:
+  - navigate: ${{ args.url }}
+
+  - wait: 5
+
+  - evaluate: |
+      (() => {
+        var vm = (document.querySelector('#app') || {}).__vue__;
+        if (!vm || !vm.$store) return [];
+        var core = vm.$store.state.coreInfo || {};
+        var goods = vm.$store.state.goodsInfo || {};
+        var shop = ((vm.$store.state.compositeInfo || {}).shop_conf) || {};
+        return [{
+          name: core.resource_name || '',
+          resource_id: core.resource_id || '',
+          resource_type: core.resource_type || '',
+          cover: core.resource_img || '',
+          user_count: core.user_count || 0,
+          price: goods.price ? (goods.price / 100).toFixed(2) : '0',
+          original_price: goods.line_price ? (goods.line_price / 100).toFixed(2) : '0',
+          is_free: goods.is_free || 0,
+          shop_name: shop.shop_name || '',
+        }];
+      })()
+
+columns: [name, price, original_price, user_count, shop_name]

package/src/clis/xiaoe/play-url.yaml

@@ -0,0 +1,124 @@
+site: xiaoe
+name: play-url
+description: 小鹅通视频/音频/直播回放 M3U8 播放地址
+domain: h5.xet.citv.cn
+strategy: cookie
+
+args:
+  url:
+    type: str
+    required: true
+    positional: true
+    description: 小节页面 URL
+
+pipeline:
+  - navigate: ${{ args.url }}
+
+  - wait: 2
+
+  - evaluate: |
+      (async () => {
+        var pageUrl = window.location.href;
+        var origin = window.location.origin;
+        var resourceId = (pageUrl.match(/[val]_[a-f0-9]+/) || [])[0] || '';
+        var productId = (pageUrl.match(/product_id=([^&]+)/) || [])[1] || '';
+        var appId = (origin.match(/(app[a-z0-9]+)\./) || [])[1] || '';
+        var isLive = resourceId.startsWith('l_') || pageUrl.includes('/alive/');
+        var m3u8Url = '', method = '', title = document.title, duration = 0;
+
+        // 深度搜索 Vue 组件树找 M3U8
+        function searchVueM3u8() {
+          var el = document.querySelector('#app');
+          if (!el || !el.__vue__) return '';
+          var walk = function(vm, d) {
+            if (!vm || d > 10) return '';
+            var data = vm.$data || {};
+            for (var k in data) {
+              if (k[0] === '_' || k[0] === '$') continue;
+              var v = data[k];
+              if (typeof v === 'string' && v.includes('.m3u8')) return v;
+              if (typeof v === 'object' && v) {
+                try {
+                  var s = JSON.stringify(v);
+                  var m = s.match(/https?:[^"]*\.m3u8[^"]*/);
+                  if (m) return m[0].replace(/\\\//g, '/');
+                } catch(e) {}
+              }
+            }
+            if (vm.$children) {
+              for (var c = 0; c < vm.$children.length; c++) {
+                var f = walk(vm.$children[c], d + 1);
+                if (f) return f;
+              }
+            }
+            return '';
+          };
+          return walk(el.__vue__, 0);
+        }
+
+        // ===== 视频课: detail_info → getPlayUrl =====
+        if (!isLive && resourceId.startsWith('v_')) {
+          try {
+            var detailRes = await fetch(origin + '/xe.course.business.video.detail_info.get/2.0.0', {
+              method: 'POST', credentials: 'include',
+              headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+              body: new URLSearchParams({
+                'bizData[resource_id]': resourceId,
+                'bizData[product_id]': productId || resourceId,
+                'bizData[opr_sys]': 'MacIntel',
+              }),
+            });
+            var detail = await detailRes.json();
+            var vi = (detail.data || {}).video_info || {};
+            title = vi.file_name || title;
+            duration = vi.video_length || 0;
+            if (vi.play_sign) {
+              var userId = (document.cookie.match(/ctx_user_id=([^;]+)/) || [])[1] || window.__user_id || '';
+              var playRes = await fetch(origin + '/xe.material-center.play/getPlayUrl', {
+                method: 'POST', credentials: 'include',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                  org_app_id: appId, app_id: vi.material_app_id || appId,
+                  user_id: userId, play_sign: [vi.play_sign],
+                  play_line: 'A', opr_sys: 'MacIntel',
+                }),
+              });
+              var playData = await playRes.json();
+              if (playData.code === 0 && playData.data) {
+                var m = JSON.stringify(playData.data).match(/https?:[^"]*\.m3u8[^"]*/);
+                if (m) { m3u8Url = m[0].replace(/\\u0026/g, '&').replace(/\\\//g, '/'); method = 'api_direct'; }
+              }
+            }
+          } catch(e) {}
+        }
+
+        // ===== 兜底: Performance API + Vue 搜索轮询 =====
+        if (!m3u8Url) {
+          for (var attempt = 0; attempt < 30; attempt++) {
+            var entries = performance.getEntriesByType('resource');
+            for (var i = 0; i < entries.length; i++) {
+              if (entries[i].name.includes('.m3u8')) { m3u8Url = entries[i].name; method = 'perf_api'; break; }
+            }
+            if (!m3u8Url) { m3u8Url = searchVueM3u8(); if (m3u8Url) method = 'vue_search'; }
+            if (m3u8Url) break;
+            await new Promise(function(r) { setTimeout(r, 500); });
+          }
+        }
+
+        if (!duration) {
+          var vid = document.querySelector('video'), aud = document.querySelector('audio');
+          if (vid && vid.duration && !isNaN(vid.duration)) duration = Math.round(vid.duration);
+          if (aud && aud.duration && !isNaN(aud.duration)) duration = Math.round(aud.duration);
+        }
+
+        return [{ title: title, resource_id: resourceId, m3u8_url: m3u8Url, duration_sec: duration, method: method }];
+      })()
+
+  - map:
+      title: ${{ item.title }}
+      resource_id: ${{ item.resource_id }}
+      m3u8_url: ${{ item.m3u8_url }}
+      duration_sec: ${{ item.duration_sec }}
+      method: ${{ item.method }}
+
+columns: [title, resource_id, m3u8_url, duration_sec, method]

package/src/clis/xiaohongshu/comments.test.ts

@@ -15,8 +15,6 @@ function createPageMock(evaluateResult: any): IPage {
     getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
     wait: vi.fn().mockResolvedValue(undefined),
     tabs: vi.fn().mockResolvedValue([]),
-    closeTab: vi.fn().mockResolvedValue(undefined),
-    newTab: vi.fn().mockResolvedValue(undefined),
     selectTab: vi.fn().mockResolvedValue(undefined),
     networkRequests: vi.fn().mockResolvedValue([]),
     consoleMessages: vi.fn().mockResolvedValue([]),

package/src/clis/xiaohongshu/creator-note-detail.test.ts

@@ -22,8 +22,6 @@ function createPageMock(evaluateResult: any): IPage {
     getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
     wait: vi.fn().mockResolvedValue(undefined),
     tabs: vi.fn().mockResolvedValue([]),
-    closeTab: vi.fn().mockResolvedValue(undefined),
-    newTab: vi.fn().mockResolvedValue(undefined),
     selectTab: vi.fn().mockResolvedValue(undefined),
     networkRequests: vi.fn().mockResolvedValue([]),
     consoleMessages: vi.fn().mockResolvedValue([]),

package/src/clis/xiaohongshu/creator-notes.test.ts

@@ -26,8 +26,6 @@ function createPageMock(evaluateResult: any, interceptedRequests: any[] = []): I
     getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
     wait: vi.fn().mockResolvedValue(undefined),
     tabs: vi.fn().mockResolvedValue([]),
-    closeTab: vi.fn().mockResolvedValue(undefined),
-    newTab: vi.fn().mockResolvedValue(undefined),
     selectTab: vi.fn().mockResolvedValue(undefined),
     networkRequests: vi.fn().mockResolvedValue([]),
     consoleMessages: vi.fn().mockResolvedValue([]),

package/src/clis/xiaohongshu/download.test.ts

@@ -29,8 +29,6 @@ function createPageMock(evaluateResult: any): IPage {
     getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
     wait: vi.fn().mockResolvedValue(undefined),
     tabs: vi.fn().mockResolvedValue([]),
-    closeTab: vi.fn().mockResolvedValue(undefined),
-    newTab: vi.fn().mockResolvedValue(undefined),
     selectTab: vi.fn().mockResolvedValue(undefined),
     networkRequests: vi.fn().mockResolvedValue([]),
     consoleMessages: vi.fn().mockResolvedValue([]),

package/src/clis/xiaohongshu/note.test.ts

@@ -16,8 +16,6 @@ function createPageMock(evaluateResult: any): IPage {
     getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
     wait: vi.fn().mockResolvedValue(undefined),
     tabs: vi.fn().mockResolvedValue([]),
-    closeTab: vi.fn().mockResolvedValue(undefined),
-    newTab: vi.fn().mockResolvedValue(undefined),
     selectTab: vi.fn().mockResolvedValue(undefined),
     networkRequests: vi.fn().mockResolvedValue([]),
     consoleMessages: vi.fn().mockResolvedValue([]),

package/src/clis/xiaohongshu/publish.test.ts

@@ -25,8 +25,6 @@ function createPageMock(evaluateResults: any[], overrides: Partial<IPage> = {}):
     getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
     wait: vi.fn().mockResolvedValue(undefined),
     tabs: vi.fn().mockResolvedValue([]),
-    closeTab: vi.fn().mockResolvedValue(undefined),
-    newTab: vi.fn().mockResolvedValue(undefined),
     selectTab: vi.fn().mockResolvedValue(undefined),
     networkRequests: vi.fn().mockResolvedValue([]),
     consoleMessages: vi.fn().mockResolvedValue([]),

package/src/clis/xiaohongshu/search.test.ts

@@ -20,8 +20,6 @@ function createPageMock(evaluateResults: any[]): IPage {
     getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
     wait: vi.fn().mockResolvedValue(undefined),
     tabs: vi.fn().mockResolvedValue([]),
-    closeTab: vi.fn().mockResolvedValue(undefined),
-    newTab: vi.fn().mockResolvedValue(undefined),
     selectTab: vi.fn().mockResolvedValue(undefined),
     networkRequests: vi.fn().mockResolvedValue([]),
     consoleMessages: vi.fn().mockResolvedValue([]),
@@ -41,8 +39,8 @@ describe('xiaohongshu search', () => {
     expect(cmd?.func).toBeTypeOf('function');
 
     const page = createPageMock([
-      // First evaluate: early login-wall check (returns true)
-      true,
+      // First evaluate: MutationObserver wait (login wall detected)
+      'login_wall',
     ]);
 
     await expect(cmd!.func!(page, { query: '特斯拉', limit: 5 })).rejects.toThrow(
@@ -63,21 +61,18 @@ describe('xiaohongshu search', () => {
       'https://www.xiaohongshu.com/user/profile/635a9c720000000018028b40?xsec_token=user-token&xsec_source=pc_search';
 
     const page = createPageMock([
-      // First evaluate: early login-wall check (returns false → no wall)
-      false,
-      // Second evaluate: main DOM extraction
-      {
-        loginWall: false,
-        results: [
-          {
-            title: '某鱼买FSD被坑了4万',
-            author: '随风',
-            likes: '261',
-            url: detailUrl,
-            author_url: authorUrl,
-          },
-        ],
-      },
+      // First evaluate: MutationObserver wait (content appeared)
+      'content',
+      // Second evaluate: main DOM extraction (returns array directly)
+      [
+        {
+          title: '某鱼买FSD被坑了4万',
+          author: '随风',
+          likes: '261',
+          url: detailUrl,
+          author_url: authorUrl,
+        },
+      ],
     ]);
 
     const result = await cmd!.func!(page, { query: '特斯拉', limit: 1 });
@@ -103,35 +98,32 @@ describe('xiaohongshu search', () => {
     expect(cmd?.func).toBeTypeOf('function');
 
     const page = createPageMock([
-      // First evaluate: early login-wall check (returns false → no wall)
-      false,
-      // Second evaluate: main DOM extraction
-      {
-        loginWall: false,
-        results: [
-          {
-            title: 'Result A',
-            author: 'UserA',
-            likes: '10',
-            url: 'https://www.xiaohongshu.com/search_result/aaa',
-            author_url: '',
-          },
-          {
-            title: '',
-            author: 'UserB',
-            likes: '5',
-            url: 'https://www.xiaohongshu.com/search_result/bbb',
-            author_url: '',
-          },
-          {
-            title: 'Result C',
-            author: 'UserC',
-            likes: '3',
-            url: 'https://www.xiaohongshu.com/search_result/ccc',
-            author_url: '',
-          },
-        ],
-      },
+      // First evaluate: MutationObserver wait (content appeared)
+      'content',
+      // Second evaluate: main DOM extraction (returns array directly)
+      [
+        {
+          title: 'Result A',
+          author: 'UserA',
+          likes: '10',
+          url: 'https://www.xiaohongshu.com/search_result/aaa',
+          author_url: '',
+        },
+        {
+          title: '',
+          author: 'UserB',
+          likes: '5',
+          url: 'https://www.xiaohongshu.com/search_result/bbb',
+          author_url: '',
+        },
+        {
+          title: 'Result C',
+          author: 'UserC',
+          likes: '3',
+          url: 'https://www.xiaohongshu.com/search_result/ccc',
+          author_url: '',
+        },
+      ],
     ]);
 
     const result = (await cmd!.func!(page, { query: '测试', limit: 1 })) as any[];
@@ -140,6 +132,25 @@ describe('xiaohongshu search', () => {
     expect(result).toHaveLength(1);
     expect(result[0]).toMatchObject({ rank: 1, title: 'Result A' });
   });
+
+  it('waits for content via MutationObserver before extracting', async () => {
+    const cmd = getRegistry().get('xiaohongshu/search');
+    expect(cmd?.func).toBeTypeOf('function');
+
+    const page = createPageMock([
+      // First evaluate: MutationObserver wait (content appeared)
+      'content',
+      // Second evaluate: extraction (returns empty array)
+      [],
+    ]);
+
+    const result = (await cmd!.func!(page, { query: '测试等待', limit: 5 })) as any[];
+    expect(result).toHaveLength(0);
+    // Only one navigation, no retry
+    expect(page.goto).toHaveBeenCalledTimes(1);
+    // Two evaluate calls: wait + extraction
+    expect(page.evaluate).toHaveBeenCalledTimes(2);
+  });
 });
 
 describe('noteIdToDate (ObjectID timestamp parsing)', () => {