@tikomni/skills 0.1.3 → 0.1.4

package/skills/social-media-crawl/references/contracts/work-fact-card-fields.md

@@ -0,0 +1,48 @@
+# Work Fact Card Fields
+
+## Required Fields
+
+- `platform`
+- `platform_work_id`
+- `platform_author_id`
+- `author_handle`
+- `author`
+- `title`
+- `caption_raw`
+- `work_modality`
+- `published_date`
+- `cover_image`
+- `source_url`
+- `share_url`
+- `primary_text`
+- `primary_text_source`
+- `request_id`
+- `completeness`
+- `missing_fields`
+- `error_reason`
+- `extract_trace`
+
+## Optional Fields
+
+- `digg_count`
+- `comment_count`
+- `collect_count`
+- `share_count`
+- `play_count`
+- `video_download_url`
+- `asr_raw`
+- `asr_clean`
+- `subtitle_raw`
+
+## Field Rules
+
+- `author` 是展示名，不是对象。
+- `primary_text` 为当前任务最适合阅读和索引的主文本。
+- 视频优先顺序：
+  - `subtitle_raw`
+  - `asr_clean`
+  - `caption_raw`
+- 文本作品优先顺序：
+  - `caption_raw`
+- 不允许出现分析字段。
+

package/skills/social-media-crawl/references/guides/generic-mcp-objects.md

@@ -0,0 +1,30 @@
+# Generic MCP Objects Guide
+
+以下对象在首期不冻结细粒度 schema：
+
+- 评论线程
+- 搜索结果
+- 榜单
+- 直播间
+- 商品页
+
+## Rules
+
+- 这些对象统一走 MCP 通用工作流。
+- agent 先做对象识别，再用 `catalog.search` 和 `endpoint.describe` 选最小工具链。
+- 输出必须满足统一 envelope。
+- 不要求首期落卡。
+- 不允许为了 schema 完整性去编造字段。
+
+## Minimum Deliverable
+
+- `object_type`
+- `platform`
+- `input`
+- `normalized`
+- `request_id`
+- `completeness`
+- `missing_fields`
+- `error_reason`
+- `extract_trace`
+

package/skills/social-media-crawl/references/mcp-usage-contract.md

@@ -0,0 +1,30 @@
+# MCP Usage Contract
+
+## Fixed Inputs
+
+- MCP URL: `https://mcp.tikomni.com/mcp`
+- Auth: `Authorization: Bearer <TIKOMNI_API_KEY>`
+- 不在工具参数中重复传 API key
+
+## Required Tool Order
+
+1. `tools/list`
+2. 识别是否命中固定 pipeline
+3. 若不命中固定 pipeline：
+   - `catalog.search`
+   - `endpoint.describe`
+   - `api.call`
+4. 若需要视频文本：
+   - `u2.submit`
+   - `u2.query`
+   - 超过 60 秒仍 `pending` 时进入 U3 fallback
+
+## Output Rules
+
+- 事实字段与派生元数据分离
+- 结果必须包含 `request_id`
+- 结果必须包含 `completeness`
+- 结果必须包含 `missing_fields`
+- 结果必须包含 `error_reason`
+- 结果必须包含 `extract_trace`
+

package/skills/social-media-crawl/references/pipelines/douyin-creator-home.md

@@ -0,0 +1,7 @@
+# Douyin Creator Home
+
+- 输入：主页链接或分享短链
+- 输出：`creator_profile.json`、`work_collection.json`、多张 `work_fact_card`
+- 实现脚本：`scripts/pipelines/run_douyin_creator_home.py`
+- 必测样例：计划冻结样例中的抖音主页链接
+

package/skills/social-media-crawl/references/pipelines/douyin-single-work.md

@@ -0,0 +1,7 @@
+# Douyin Single Work
+
+- 输入：作品链接或分享短链
+- 输出：统一 `work_fact_card`
+- 实现脚本：`scripts/pipelines/run_douyin_single_work.py`
+- 必测样例：计划冻结样例中的抖音作品链接
+

package/skills/social-media-crawl/references/pipelines/xiaohongshu-creator-home.md

@@ -0,0 +1,7 @@
+# Xiaohongshu Creator Home
+
+- 输入：主页链接或分享短链
+- 输出：`creator_profile.json`、`work_collection.json`、多张 `work_fact_card`
+- 实现脚本：`scripts/pipelines/run_xiaohongshu_creator_home.py`
+- 必测样例：计划冻结样例中的小红书主页链接
+

package/skills/social-media-crawl/references/pipelines/xiaohongshu-single-work.md

@@ -0,0 +1,7 @@
+# Xiaohongshu Single Work
+
+- 输入：作品链接或分享短链
+- 输出：统一 `work_fact_card`
+- 实现脚本：`scripts/pipelines/run_xiaohongshu_single_work.py`
+- 必测样例：计划冻结样例中的小红书作品链接
+

package/skills/social-media-crawl/references/schemas/creator-profile.schema.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "social-media-crawl/creator-profile.schema.json",
+  "type": "object",
+  "required": [
+    "platform",
+    "platform_author_id",
+    "author_handle",
+    "nickname",
+    "request_id",
+    "extract_trace"
+  ],
+  "properties": {
+    "platform": { "type": "string" },
+    "platform_author_id": { "type": "string" },
+    "author_handle": { "type": "string" },
+    "nickname": { "type": "string" },
+    "signature": { "type": "string" },
+    "avatar_url": { "type": "string" },
+    "fans_count": { "type": "integer" },
+    "liked_count": { "type": "integer" },
+    "collected_count": { "type": "integer" },
+    "works_count": { "type": "integer" },
+    "verified": { "type": "boolean" },
+    "request_id": { "type": ["string", "null"] },
+    "extract_trace": {
+      "type": "array",
+      "items": { "type": "object" }
+    }
+  },
+  "additionalProperties": true
+}
+

package/skills/social-media-crawl/references/schemas/output-envelope.schema.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "social-media-crawl/output-envelope.schema.json",
+  "type": "object",
+  "required": [
+    "object_type",
+    "platform",
+    "input",
+    "normalized",
+    "completeness",
+    "missing_fields",
+    "extract_trace",
+    "request_id"
+  ],
+  "properties": {
+    "object_type": { "type": "string" },
+    "platform": { "type": "string" },
+    "input": {},
+    "normalized": {},
+    "completeness": {
+      "type": "string",
+      "enum": ["complete", "partial", "incomplete"]
+    },
+    "missing_fields": {
+      "type": "array",
+      "items": { "type": "object" }
+    },
+    "error_reason": {
+      "type": ["string", "null"]
+    },
+    "extract_trace": {
+      "type": "array",
+      "items": { "type": "object" }
+    },
+    "request_id": {
+      "type": ["string", "null"]
+    }
+  },
+  "additionalProperties": true
+}
+

package/skills/social-media-crawl/references/schemas/work-collection.schema.json

@@ -0,0 +1,29 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "social-media-crawl/work-collection.schema.json",
+  "type": "object",
+  "required": [
+    "platform",
+    "platform_author_id",
+    "count",
+    "items",
+    "request_id",
+    "extract_trace"
+  ],
+  "properties": {
+    "platform": { "type": "string" },
+    "platform_author_id": { "type": "string" },
+    "count": { "type": "integer" },
+    "items": {
+      "type": "array",
+      "items": { "type": "object" }
+    },
+    "request_id": { "type": ["string", "null"] },
+    "extract_trace": {
+      "type": "array",
+      "items": { "type": "object" }
+    }
+  },
+  "additionalProperties": true
+}
+

package/skills/social-media-crawl/references/schemas/work-fact-card.schema.json

@@ -0,0 +1,67 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "social-media-crawl/work-fact-card.schema.json",
+  "type": "object",
+  "required": [
+    "platform",
+    "platform_work_id",
+    "platform_author_id",
+    "author_handle",
+    "author",
+    "title",
+    "caption_raw",
+    "work_modality",
+    "published_date",
+    "source_url",
+    "share_url",
+    "primary_text",
+    "primary_text_source",
+    "request_id",
+    "completeness",
+    "missing_fields",
+    "extract_trace"
+  ],
+  "properties": {
+    "platform": { "type": "string" },
+    "platform_work_id": { "type": "string" },
+    "platform_author_id": { "type": "string" },
+    "author_handle": { "type": "string" },
+    "author": { "type": "string" },
+    "title": { "type": "string" },
+    "caption_raw": { "type": "string" },
+    "subtitle_raw": { "type": "string" },
+    "work_modality": { "type": "string" },
+    "published_date": { "type": "string" },
+    "digg_count": { "type": "integer" },
+    "comment_count": { "type": "integer" },
+    "collect_count": { "type": "integer" },
+    "share_count": { "type": "integer" },
+    "play_count": { "type": "integer" },
+    "cover_image": { "type": "string" },
+    "source_url": { "type": "string" },
+    "share_url": { "type": "string" },
+    "video_download_url": { "type": "string" },
+    "primary_text": { "type": "string" },
+    "primary_text_source": { "type": "string" },
+    "asr_raw": { "type": "string" },
+    "asr_clean": { "type": "string" },
+    "request_id": { "type": ["string", "null"] },
+    "completeness": {
+      "type": "string",
+      "enum": ["complete", "partial", "incomplete"]
+    },
+    "missing_fields": {
+      "type": "array",
+      "items": { "type": "object" }
+    },
+    "error_reason": {
+      "type": ["string", "null"]
+    },
+    "extract_trace": {
+      "type": "array",
+      "items": { "type": "object" }
+    }
+  },
+  "additionalProperties": false
+}
+

package/skills/social-media-crawl/references/service-guides/u2-u3-mandatory-fallback.md

@@ -0,0 +1,21 @@
+# U2 U3 Mandatory Fallback
+
+## Hard Rule
+
+- 视频文本优先尝试平台原生字幕。
+- 无原生字幕时先走 U2。
+- `u2.submit` 成功后，最多轮询 60 秒。
+- 如果 60 秒后仍是 `pending`，必须进入 U3 fallback。
+- U3 成功后继续完成 ASR 链路。
+- 若最终失败，保留事实卡并标记 `completeness=incomplete`。
+
+## Trace Requirement
+
+至少记录：
+
+- U2 submit
+- U2 poll
+- 触发 U3 的原因
+- U3 调用步骤
+- 最终文本来源
+

package/skills/social-media-crawl/scripts/__init__.py

@@ -0,0 +1,2 @@
+"""social-media-crawl scripts package."""
+

package/skills/social-media-crawl/scripts/core/__init__.py

@@ -0,0 +1,2 @@
+"""Core helpers for social-media-crawl."""
+

package/skills/{single-work-analysis/scripts/pipeline/asr → social-media-crawl/scripts/core}/asr_pipeline.py

@@ -19,8 +19,10 @@ from scripts.core.tikomni_common import (
     is_terminal_status,
     normalize_text,
 )
+from scripts.core.u3_fallback import run_u3_public_url_fallback
 
 U2_BATCH_SUBMIT_HARD_LIMIT = 100
+DEFAULT_U2_PENDING_TIMEOUT_SEC = 60
 
 
 def clamp_u2_batch_submit_size(size: int, *, default: int = 50, hard_limit: int = U2_BATCH_SUBMIT_HARD_LIMIT) -> int:
@@ -713,10 +715,12 @@ def poll_u2_task_core(
     max_polls: int,
     require_batch_complete: bool = False,
     expected_total: int = 0,
+    pending_timeout_sec: Optional[int] = None,
     progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     trace = []
     last_request_id = None
+    started_at = time.perf_counter()
 
     last_status = "UNKNOWN"
     last_payload: Any = {}
@@ -758,6 +762,7 @@ def poll_u2_task_core(
         trace.append(
             {
                 "attempt": attempt,
+                "elapsed_ms": int((time.perf_counter() - started_at) * 1000),
                 "status_code": response.get("status_code"),
                 "task_status": status,
                 "platform_task_status": platform_status,
@@ -773,6 +778,7 @@ def poll_u2_task_core(
                 progress_callback(
                     {
                         "attempt": attempt,
+                        "elapsed_ms": int((time.perf_counter() - started_at) * 1000),
                         "task_id": task_id,
                         "task_status": status or "UNKNOWN",
                         "platform_task_status": platform_status or "UNKNOWN",
@@ -810,6 +816,26 @@ def poll_u2_task_core(
         task_complete = status_terminal or platform_terminal
         batch_complete = bool(batch_progress.get("complete")) if require_batch_complete else task_complete
 
+        elapsed_sec = time.perf_counter() - started_at
+        if pending_timeout_sec and elapsed_sec >= max(float(pending_timeout_sec), 1.0):
+            timeout_reason = "u2_batch_pending_timeout" if require_batch_complete else "u2_pending_timeout"
+            return {
+                "ok": False,
+                "task_id": task_id,
+                "task_status": effective_status or last_status or "PENDING",
+                "platform_task_status": platform_status,
+                "pending_count": pending_count,
+                "request_id": last_request_id,
+                "error_reason": timeout_reason,
+                "transcript_text": "",
+                "raw_task": payload,
+                "task_metrics": last_metrics,
+                "batch_results": batch_results,
+                "batch_progress": batch_progress,
+                "batch_complete": bool(batch_complete),
+                "trace": trace,
+            }
+
         if require_batch_complete and not batch_complete:
             if attempt < max_polls:
                 time.sleep(max(poll_interval_sec, 0.2))
@@ -918,6 +944,7 @@ def run_u2_asr_candidates_with_timeout_retry(
     max_polls: int,
     timeout_retry_enabled: bool = True,
     timeout_retry_max_retries: int = 3,
+    pending_timeout_sec: int = DEFAULT_U2_PENDING_TIMEOUT_SEC,
     progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     normalized_candidates = normalize_media_candidates(candidates)
@@ -953,6 +980,7 @@ def run_u2_asr_candidates_with_timeout_retry(
             max_polls=max_polls,
             timeout_retry_enabled=timeout_retry_enabled,
             timeout_retry_max_retries=timeout_retry_max_retries,
+            pending_timeout_sec=pending_timeout_sec,
             progress_callback=progress_callback,
         )
         poll_result = bundle.get("poll_result", {})
@@ -966,6 +994,7 @@ def run_u2_asr_candidates_with_timeout_retry(
             "ok": ok,
             "error_reason": error_reason,
             "task_status": poll_result.get("task_status"),
+            "u3_fallback": bundle.get("u3_fallback", {}),
         })
 
         final_bundle = bundle
@@ -973,6 +1002,9 @@ def run_u2_asr_candidates_with_timeout_retry(
         if ok:
             break
 
+        if (bundle.get("u3_fallback") or {}).get("triggered"):
+            break
+
         if error_reason == "INVALID_SOURCE_URL":
             continue
 
@@ -994,6 +1026,7 @@ def run_u2_asr_batch_with_timeout_retry(
     max_polls: int,
     timeout_retry_enabled: bool = True,
     timeout_retry_max_retries: int = 3,
+    pending_timeout_sec: int = DEFAULT_U2_PENDING_TIMEOUT_SEC,
     progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     normalized_urls = normalize_media_candidates(file_urls)
@@ -1072,6 +1105,7 @@ def run_u2_asr_batch_with_timeout_retry(
                 max_polls=max_polls,
                 require_batch_complete=True,
                 expected_total=len(limited_urls),
+                pending_timeout_sec=pending_timeout_sec,
                 progress_callback=progress_callback,
             )
             poll_duration_ms = int((time.perf_counter() - poll_started_at) * 1000)
@@ -1126,7 +1160,7 @@ def run_u2_asr_batch_with_timeout_retry(
         final_submit_bundle = submit_bundle
         final_poll_result = poll_result
 
-        if poll_result.get("error_reason") in {"u2_poll_timeout", "u2_batch_incomplete_timeout"} and round_index < max_rounds:
+        if poll_result.get("error_reason") in {"u2_poll_timeout", "u2_batch_incomplete_timeout", "u2_batch_pending_timeout"} and round_index < max_rounds:
             timeout_retry_triggered = True
             timeout_retry_result = "retrying"
             continue
@@ -1135,7 +1169,7 @@ def run_u2_asr_batch_with_timeout_retry(
 
     if final_poll_result.get("ok"):
         timeout_retry_result = "retry_succeeded" if timeout_retry_triggered else "not_needed"
-    elif final_poll_result.get("error_reason") in {"u2_poll_timeout", "u2_batch_incomplete_timeout"}:
+    elif final_poll_result.get("error_reason") in {"u2_poll_timeout", "u2_batch_incomplete_timeout", "u2_batch_pending_timeout"}:
         timeout_retry_result = "retry_timeout_exhausted" if timeout_retry_triggered else "timeout_no_retry"
     elif timeout_retry_triggered:
         timeout_retry_result = "retry_failed_non_timeout"
@@ -1221,6 +1255,8 @@ def run_u2_asr_with_timeout_retry(
     max_polls: int,
     timeout_retry_enabled: bool = True,
     timeout_retry_max_retries: int = 3,
+    pending_timeout_sec: int = DEFAULT_U2_PENDING_TIMEOUT_SEC,
+    u3_fallback_enabled: bool = True,
     progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     video_url = normalize_media_url(video_url)
@@ -1237,6 +1273,14 @@ def run_u2_asr_with_timeout_retry(
     }
     timeout_retry_triggered = False
     timeout_retry_result = "not_triggered"
+    u3_fallback_bundle: Dict[str, Any] = {
+        "enabled": bool(u3_fallback_enabled),
+        "triggered": False,
+        "ok": False,
+        "result": "not_triggered",
+        "public_url": "",
+        "trace": [],
+    }
 
     for round_index in range(1, max_rounds + 1):
         submit_started_at = time.perf_counter()
@@ -1264,6 +1308,7 @@ def run_u2_asr_with_timeout_retry(
                 task_id=str(task_id),
                 poll_interval_sec=poll_interval_sec,
                 max_polls=max_polls,
+                pending_timeout_sec=pending_timeout_sec,
                 progress_callback=progress_callback,
             )
             poll_duration_ms = int((time.perf_counter() - poll_started_at) * 1000)
@@ -1305,6 +1350,56 @@ def run_u2_asr_with_timeout_retry(
         final_submit_bundle = submit_bundle
         final_poll_result = poll_result
 
+        if (
+            u3_fallback_enabled
+            and poll_result.get("error_reason") in {"u2_pending_timeout", "u2_poll_timeout"}
+        ):
+            u3_fallback_bundle["triggered"] = True
+            u3_result = run_u3_public_url_fallback(
+                base_url=base_url,
+                token=token,
+                timeout_ms=timeout_ms,
+                source_url=video_url,
+            )
+            u3_fallback_bundle.update(
+                {
+                    "ok": bool(u3_result.get("ok")),
+                    "result": "u3_completed" if u3_result.get("ok") else "u3_failed",
+                    "public_url": u3_result.get("public_url") or "",
+                    "request_id": u3_result.get("request_id"),
+                    "error_reason": u3_result.get("error_reason"),
+                    "trace": u3_result.get("trace", []),
+                }
+            )
+            if u3_result.get("ok") and u3_result.get("public_url"):
+                retry_bundle = run_u2_asr_with_timeout_retry(
+                    base_url=base_url,
+                    token=token,
+                    timeout_ms=timeout_ms,
+                    video_url=str(u3_result.get("public_url")),
+                    submit_max_retries=submit_max_retries,
+                    submit_backoff_ms=submit_backoff_ms,
+                    poll_interval_sec=poll_interval_sec,
+                    max_polls=max_polls,
+                    timeout_retry_enabled=timeout_retry_enabled,
+                    timeout_retry_max_retries=timeout_retry_max_retries,
+                    pending_timeout_sec=pending_timeout_sec,
+                    u3_fallback_enabled=False,
+                    progress_callback=progress_callback,
+                )
+                retry_bundle["u3_fallback"] = u3_fallback_bundle
+                retry_bundle["original_source_url"] = video_url
+                return retry_bundle
+            final_poll_result = {
+                "ok": False,
+                "task_id": poll_result.get("task_id") or task_id,
+                "task_status": poll_result.get("task_status") or "UNKNOWN",
+                "request_id": u3_result.get("request_id") or poll_result.get("request_id"),
+                "error_reason": u3_result.get("error_reason") or "u3_fallback_failed",
+                "trace": list(poll_result.get("trace", [])) + list(u3_result.get("trace", [])),
+            }
+            break
+
         if poll_result.get("error_reason") == "u2_poll_timeout" and round_index < max_rounds:
             timeout_retry_triggered = True
             timeout_retry_result = "retrying"
@@ -1331,6 +1426,7 @@ def run_u2_asr_with_timeout_retry(
             "triggered": timeout_retry_triggered,
             "result": timeout_retry_result,
         },
+        "u3_fallback": u3_fallback_bundle,
         "submit_duration_ms": _safe_int((rounds[-1].get("submit") if rounds else {}).get("duration_ms")),
         "poll_duration_ms": _safe_int((rounds[-1].get("poll") if rounds else {}).get("duration_ms")),
     }

package/skills/social-media-crawl/scripts/core/completeness.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Completeness evaluation helpers."""
+
+from __future__ import annotations
+
+import hashlib
+import time
+from typing import Any, Dict, List, Optional
+
+
+def normalize_missing_fields(items: Any) -> List[Dict[str, str]]:
+    normalized: List[Dict[str, str]] = []
+    seen = set()
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if isinstance(item, dict):
+            field = str(item.get("field") or "").strip()
+            reason = str(item.get("reason") or "").strip() or "missing"
+        else:
+            field = str(item or "").strip()
+            reason = "missing"
+        if not field:
+            continue
+        key = (field, reason)
+        if key in seen:
+            continue
+        seen.add(key)
+        normalized.append({"field": field, "reason": reason})
+    return normalized
+
+
+def ensure_request_id(value: Optional[str], fallback_seed: str = "") -> str:
+    text = str(value or "").strip()
+    if text:
+        return text
+    digest = hashlib.sha1(fallback_seed.encode("utf-8")).hexdigest()[:8] if fallback_seed else "unknown"
+    return f"local-{int(time.time())}-{digest}"
+
+
+def evaluate_work_fact_card(card: Dict[str, Any]) -> Dict[str, Any]:
+    missing = normalize_missing_fields(card.get("missing_fields"))
+    existing_fields = {entry["field"] for entry in missing}
+
+    required_pairs = [
+        ("platform", bool(str(card.get("platform") or "").strip())),
+        ("platform_work_id", bool(str(card.get("platform_work_id") or "").strip())),
+        ("platform_author_id", bool(str(card.get("platform_author_id") or "").strip())),
+        ("author_handle", bool(str(card.get("author_handle") or "").strip())),
+        ("title", bool(str(card.get("title") or "").strip())),
+        ("source_url", bool(str(card.get("source_url") or "").strip())),
+        ("share_url", bool(str(card.get("share_url") or "").strip())),
+    ]
+    for field, ok in required_pairs:
+        if ok or field in existing_fields:
+            continue
+        missing.append({"field": field, "reason": "missing_required"})
+        existing_fields.add(field)
+
+    primary_text = str(card.get("primary_text") or "").strip()
+    if not primary_text and "primary_text" not in existing_fields:
+        missing.append({"field": "primary_text", "reason": "missing_primary_text"})
+
+    required_missing = [entry for entry in missing if entry["reason"] == "missing_required"]
+    if required_missing:
+        completeness = "incomplete"
+    elif primary_text:
+        completeness = "complete"
+    else:
+        completeness = "partial"
+
+    return {
+        "completeness": completeness,
+        "missing_fields": missing,
+    }
+
+
+def evaluate_collection(profile: Dict[str, Any], works: List[Dict[str, Any]]) -> str:
+    if not str(profile.get("platform_author_id") or "").strip():
+        return "incomplete"
+    if works:
+        return "complete"
+    return "partial"