@tikomni/skills 1.0.5 → 1.0.7

package/skills/social-media-crawl/tests/test_api_only_routing.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import json
+import sys
+import unittest
+from pathlib import Path
+
+SKILL_ROOT = Path(__file__).resolve().parents[1]
+if str(SKILL_ROOT) not in sys.path:
+    sys.path.insert(0, str(SKILL_ROOT))
+
+from scripts.core.call_tikomni_api import call_endpoint
+from scripts.core.resolve_api_endpoint import resolve_endpoint
+
+
+class ApiOnlyRoutingTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.operations = json.loads(
+            (SKILL_ROOT / "references" / "api-catalog" / "operations.json").read_text(encoding="utf-8")
+        )
+
+    def test_error_report_paths_are_not_in_catalog(self) -> None:
+        paths = {str(operation["path"]) for operation in self.operations}
+        wrong_paths = {
+            "/api/u1/v1/health",
+            "/api/u1/v1/douyin/hot_search_list",
+            "/api/u1/v1/Douyin/douyin_hot_search_list",
+            "/api/u1/v1/wechat/article",
+            "/api/u1/v1/wechat/fetch_article",
+            "/api/u1/v1/account/info",
+            "/api/u1/v1/quote/list",
+            "/api/u1/v1/demo/wechat/article",
+        }
+        self.assertTrue(wrong_paths.isdisjoint(paths))
+        self.assertIn("/api/u1/v1/demo/douyin/web/fetch_one_video", paths)
+
+    def test_douyin_hot_search_resolves_real_endpoint(self) -> None:
+        resolved = resolve_endpoint(platform="douyin", capability="hot_search")
+        self.assertTrue(resolved["ok"])
+        self.assertEqual(
+            resolved["recommended"]["endpoint_id"],
+            "douyin.hot_search.app_v3.fetch_hot_search_list",
+        )
+        self.assertEqual(
+            resolved["recommended"]["path"],
+            "/api/u1/v1/douyin/app/v3/fetch_hot_search_list",
+        )
+        self.assertTrue(resolved["alternatives"])
+
+    def test_wechat_mp_and_channels_are_split(self) -> None:
+        mp = resolve_endpoint(platform="wechat_mp", capability="article_detail")
+        self.assertTrue(mp["ok"])
+        self.assertEqual(mp["platform"], "wechat_mp")
+        self.assertEqual(mp["recommended"]["path"], "/api/u1/v1/wechat_mp/web/fetch_mp_article_detail_json")
+
+        channels = resolve_endpoint(platform="wechat_channels", capability="video_detail")
+        self.assertTrue(channels["ok"])
+        self.assertEqual(channels["platform"], "wechat_channels")
+        self.assertEqual(channels["recommended"]["path"], "/api/u1/v1/wechat_channels/fetch_video_detail")
+
+    def test_ambiguous_wechat_returns_suggestions_not_guess(self) -> None:
+        resolved = resolve_endpoint(platform="wechat", capability="article_detail")
+        self.assertFalse(resolved["ok"])
+        self.assertEqual(resolved["code"], "ENDPOINT_NOT_RESOLVED")
+        self.assertIn({"platform": "wechat_mp", "capability": "article_detail"}, resolved["suggestions"])
+
+    def test_caller_rejects_arbitrary_routing_params(self) -> None:
+        result = call_endpoint(
+            endpoint_id="douyin.hot_search.app_v3.fetch_hot_search_list",
+            params={"path": "/api/u1/v1/douyin/hot_search_list"},
+            dry_run=True,
+        )
+        self.assertFalse(result["ok"])
+        self.assertEqual(result["code"], "RESERVED_PARAM_KEY")
+
+        result = call_endpoint(
+            endpoint_id="douyin.hot_search.app_v3.fetch_hot_search_list",
+            params={"method": "POST"},
+            dry_run=True,
+        )
+        self.assertFalse(result["ok"])
+        self.assertEqual(result["code"], "RESERVED_PARAM_KEY")
+
+    def test_caller_allows_declared_path_payload_field(self) -> None:
+        result = call_endpoint(
+            endpoint_id="xiaohongshu.search.web.sign",
+            params={
+                "path": "/api/sns/web/v1/search/notes",
+                "cookie": "web_session=test",
+                "data": {"keyword": "test"},
+            },
+            dry_run=True,
+        )
+        self.assertTrue(result["ok"])
+        self.assertEqual(result["request"]["path"], "/api/u1/v1/xiaohongshu/web/sign")
+        self.assertEqual(result["request"]["body"]["path"], "/api/sns/web/v1/search/notes")
+
+    def test_caller_rejects_unknown_endpoint_id(self) -> None:
+        result = call_endpoint(
+            endpoint_id="douyin.hot_search.app_v3.not_real",
+            params={},
+            dry_run=True,
+        )
+        self.assertFalse(result["ok"])
+        self.assertEqual(result["code"], "UNKNOWN_ENDPOINT_ID")
+
+    def test_u2_u3_stay_in_same_catalog(self) -> None:
+        endpoint_ids = {str(operation["endpoint_id"]) for operation in self.operations}
+        self.assertIn("u2.transcription.service.transcription", endpoint_ids)
+        self.assertIn("u3.media_upload.service.uploads", endpoint_ids)
+
+    def test_plain_english_intent_does_not_match_single_letter_x_alias(self) -> None:
+        resolved = resolve_endpoint(intent="extract comments from this post")
+        self.assertFalse(resolved["ok"])
+        self.assertEqual(resolved["capability"], "comments")
+        self.assertEqual(resolved["platform"], "")
+
+    def test_search_operations_are_not_classified_as_comments(self) -> None:
+        by_id = {str(operation["endpoint_id"]): operation for operation in self.operations}
+        for endpoint_id in {
+            "xiaohongshu.search.app_v2.search_notes",
+            "xiaohongshu.search.app.search_notes",
+            "xiaohongshu.search.web.search_notes",
+            "xiaohongshu.search.web.search_notes_v3",
+        }:
+            self.assertIn(endpoint_id, by_id)
+            self.assertEqual(by_id[endpoint_id]["capability"], "search")

package/skills/social-media-crawl/tests/test_fixed_pipeline_fallback.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 import os
 import sys
 import unittest
@@ -14,10 +15,59 @@ if str(SKILL_ROOT) not in sys.path:
 
 from scripts.core.asr_pipeline import run_u2_asr_batch_with_timeout_retry
 from scripts.pipelines import homepage_collectors
+from scripts.pipelines import run_douyin_single_work
 from scripts.pipelines import run_xiaohongshu_single_work
 
 
 class FixedPipelineFallbackTest(unittest.TestCase):
+    def test_douyin_single_fetch_attempt_trace_is_json_serializable(self) -> None:
+        def fake_call_json_api(**_: object) -> dict:
+            return {
+                "ok": True,
+                "status_code": 200,
+                "request_id": "req-dy-single",
+                "error_reason": None,
+                "data": {"aweme_detail": {"aweme_id": "123"}},
+            }
+
+        with patch.object(run_douyin_single_work, "call_json_api", side_effect=fake_call_json_api):
+            response = run_douyin_single_work._u1_fetch_one_video(
+                base_url="https://api.tikomni.com",
+                token="test-token",
+                share_url="https://v.douyin.com/test-single/",
+                app_timeout_ms=1000,
+                web_timeout_ms=1000,
+            )
+
+        json.dumps(response, ensure_ascii=False)
+        self.assertNotIn("response", response["_attempts"][0])
+
+    def test_xhs_single_fetch_attempt_trace_is_json_serializable(self) -> None:
+        def fake_call_json_api(**_: object) -> dict:
+            return {
+                "ok": True,
+                "status_code": 200,
+                "request_id": "req-xhs-single",
+                "error_reason": None,
+                "data": {
+                    "title": "test title",
+                    "desc": "test content",
+                    "video": {"master_url": "https://example.com/video.mp4"},
+                },
+            }
+
+        with patch.object(run_xiaohongshu_single_work, "call_json_api", side_effect=fake_call_json_api):
+            response = run_xiaohongshu_single_work._fetch_note_info(
+                base_url="https://api.tikomni.com",
+                token="test-token",
+                timeout_ms=1000,
+                source_input={"share_text": "https://xhslink.com/test-single", "note_id": "note123"},
+                progress=None,
+            )
+
+        json.dumps(response, ensure_ascii=False)
+        self.assertNotIn("response", response["_attempts"][0])
+
     def test_xhs_single_route_plan_respects_version_priority_and_cookie_gate(self) -> None:
         source_input = {
             "share_text": (

package/skills/social-media-crawl/references/guides/generic-mcp-objects.md

@@ -1,40 +0,0 @@
-# Generic MCP Objects Guide
-
-The following objects do not freeze a fine-grained schema in the first release:
-
-- Comment threads
-- Search results
-- Ranking pages
-- Livestream rooms
-- Product pages
-
-In addition, every platform and object combination that does not match a fixed pipeline falls under this guide, for example:
-
-- A single X/Twitter post
-- An X/Twitter thread
-- An X/Twitter long-form post
-- An X/Twitter creator homepage
-- The top N comments from a comment section
-
-## Rules
-
-- Route these objects through the generic MCP workflow inside this skill.
-- The platform is not limited to the Douyin and Xiaohongshu cases covered by fixed pipelines. If the platform is discoverable in the MCP catalog, try this workflow first.
-- Detect the object first, then use `catalog.search` and `endpoint.describe` to choose the smallest toolchain.
-- Do not jump to browser/CDP only because the platform is not Douyin or Xiaohongshu.
-- Use browser/CDP only when the generic MCP path is unavailable, or when the task explicitly requires page-level interaction that the API cannot satisfy. Explain the reason in the output.
-- The output must satisfy the unified envelope.
-- No card write is required in the first release.
-- Do not fabricate fields only to satisfy schema completeness.
-
-## Minimum Deliverable
-
-- `object_type`
-- `platform`
-- `input`
-- `normalized`
-- `request_id`
-- `completeness`
-- `missing_fields`
-- `error_reason`
-- `extract_trace`

package/skills/social-media-crawl/references/mcp-usage-contract.md

@@ -1,40 +0,0 @@
-# MCP Usage Contract
-
-## Scope
-
-- This contract applies to every social-media object that does not match a fixed pipeline, not only Douyin and Xiaohongshu.
-- The currently supported platforms include Douyin, Xiaohongshu, Kuaishou, Bilibili, Weibo, TikTok, YouTube, Instagram, Threads, Twitter/X, Reddit, LinkedIn, WeChat Channels, Official Accounts, Toutiao, Xigua, Zhihu, Lemon8, and Pipixia.
-- Typical objects include X/Twitter posts, threads, long-form posts, creator homepages, comment sections, search results, ranking pages, livestream rooms, and product pages.
-- Fixed pipelines are frozen only for Douyin and Xiaohongshu single-work and creator-home cases. All other supported platform and object combinations should use the generic MCP path defined here.
-
-## Fixed Inputs
-
-- MCP URL: `https://mcp.tikomni.com/mcp`
-- Auth: `Authorization: Bearer <TIKOMNI_API_KEY>`
-- Do not repeat the API key inside tool parameters.
-
-## Required Tool Order
-
-1. Detect the platform and object type.
-2. Decide whether a fixed pipeline matches.
-3. If a fixed pipeline matches, run the fixed script directly and do not enter the generic MCP path.
-4. If no fixed pipeline matches:
-   - `tools/list`
-   - `catalog.search`
-   - `endpoint.describe`
-   - `api.call`
-5. If video text is required:
-   - `u2.submit`
-   - `u2.query`
-   - Enter the U3 fallback path if the task is still `pending` after 60 seconds.
-6. Use browser/CDP only when the generic MCP path is unavailable or clearly insufficient. Do not skip step 4 and jump straight to browser/CDP.
-
-## Output Rules
-
-- Keep factual fields separate from derived metadata.
-- The result must include `request_id`.
-- The result must include `completeness`.
-- The result must include `missing_fields`.
-- The result must include `error_reason`.
-- The result must include `extract_trace`.
-- If the flow ends in browser/CDP fallback, `extract_trace` must also include the earlier MCP attempts and the fallback reason.

package/skills/social-media-crawl/scripts/core/mcp_dispatch.py

@@ -1,161 +0,0 @@
-#!/usr/bin/env python3
-"""Minimal MCP HTTP client for social-media-crawl."""
-
-from __future__ import annotations
-
-import json
-import urllib.request
-from dataclasses import dataclass
-from typing import Any, Dict, Optional
-
-
-def _parse_response_body(body: bytes) -> Any:
-    text = body.decode("utf-8", errors="replace").strip()
-    if not text:
-        return {}
-    if text.startswith("data:"):
-        payloads = []
-        for line in text.splitlines():
-            if not line.startswith("data:"):
-                continue
-            payload = line.split("data:", 1)[1].strip()
-            if payload:
-                payloads.append(payload)
-        if payloads:
-            return json.loads(payloads[-1])
-    return json.loads(text)
-
-
-@dataclass
-class McpResponse:
-    ok: bool
-    status_code: int
-    data: Any
-    session_id: Optional[str]
-    error_reason: Optional[str] = None
-
-
-class McpHttpClient:
-    def __init__(self, *, url: str, api_key: str, timeout_ms: int = 60000) -> None:
-        self.url = url.rstrip("/")
-        self.api_key = api_key
-        self.timeout_ms = timeout_ms
-        self.session_id: Optional[str] = None
-        self._next_id = 1
-
-    def _headers(self) -> Dict[str, str]:
-        headers = {
-            "Authorization": f"Bearer {self.api_key}",
-            "Content-Type": "application/json",
-            "Accept": "application/json, text/event-stream",
-            "User-Agent": "OpenClaw-SocialMediaCrawl/0.1",
-            "X-Client-Name": "social-media-crawl",
-            "X-Client-Version": "0.1.0",
-        }
-        if self.session_id:
-            headers["mcp-session-id"] = self.session_id
-        return headers
-
-    def _request(self, payload: Dict[str, Any]) -> McpResponse:
-        req = urllib.request.Request(
-            self.url,
-            data=json.dumps(payload).encode("utf-8"),
-            headers=self._headers(),
-            method="POST",
-        )
-        try:
-            with urllib.request.urlopen(req, timeout=max(self.timeout_ms / 1000.0, 1.0)) as response:
-                body = response.read()
-                data = _parse_response_body(body)
-                session_id = response.headers.get("mcp-session-id") or response.headers.get("Mcp-Session-Id")
-                if session_id:
-                    self.session_id = session_id
-                return McpResponse(ok=True, status_code=response.getcode(), data=data, session_id=self.session_id)
-        except urllib.error.HTTPError as error:
-            body = error.read()
-            try:
-                data = _parse_response_body(body)
-            except Exception:
-                data = {"raw": body.decode("utf-8", errors="replace")}
-            return McpResponse(
-                ok=False,
-                status_code=error.code,
-                data=data,
-                session_id=self.session_id,
-                error_reason=f"http_error:{error.code}",
-            )
-        except Exception as error:
-            return McpResponse(
-                ok=False,
-                status_code=0,
-                data={},
-                session_id=self.session_id,
-                error_reason=f"request_failed:{type(error).__name__}:{error}",
-            )
-
-    def _rpc(self, method: str, params: Optional[Dict[str, Any]] = None, notification: bool = False) -> McpResponse:
-        payload: Dict[str, Any] = {"jsonrpc": "2.0", "method": method}
-        if not notification:
-            payload["id"] = self._next_id
-            self._next_id += 1
-        if params is not None:
-            payload["params"] = params
-        return self._request(payload)
-
-    def initialize(self) -> McpResponse:
-        response = self._rpc(
-            "initialize",
-            {
-                "protocolVersion": "2025-03-26",
-                "capabilities": {},
-                "clientInfo": {"name": "social-media-crawl", "version": "0.1.0"},
-            },
-        )
-        if response.ok:
-            self._rpc("notifications/initialized", notification=True)
-        return response
-
-    def tools_list(self) -> McpResponse:
-        if not self.session_id:
-            init = self.initialize()
-            if not init.ok:
-                return init
-        return self._rpc("tools/list")
-
-    def tool_call(self, name: str, arguments: Optional[Dict[str, Any]] = None) -> McpResponse:
-        if not self.session_id:
-            init = self.initialize()
-            if not init.ok:
-                return init
-        return self._rpc("tools/call", {"name": name, "arguments": arguments or {}})
-
-    def catalog_search(self, query: str) -> McpResponse:
-        return self.tool_call("catalog.search", {"query": query})
-
-    def endpoint_describe(self, method: str, path: str) -> McpResponse:
-        return self.tool_call("endpoint.describe", {"method": method, "path": path})
-
-    def api_call(
-        self,
-        method: str,
-        path: str,
-        query: Optional[Dict[str, Any]] = None,
-        headers: Optional[Dict[str, Any]] = None,
-        body: Optional[Any] = None,
-    ) -> McpResponse:
-        return self.tool_call(
-            "api.call",
-            {
-                "method": method,
-                "path": path,
-                "query": query or {},
-                "headers": headers or {},
-                "body": {} if body is None else body,
-            },
-        )
-
-    def u2_submit(self, file_url: str) -> McpResponse:
-        return self.tool_call("u2.submit", {"file_url": file_url})
-
-    def u2_query(self, task_id: str) -> McpResponse:
-        return self.tool_call("u2.query", {"task_id": task_id})