@tikomni/skills 0.1.6 → 0.1.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tikomni/skills",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "TikOmni skill installer CLI for structured social media crawling in Codex, Claude Code, and OpenClaw",
   "license": "MIT",
   "homepage": "https://github.com/mark-ly-wang/TikOmni-Skills#readme",

package/skills/social-media-crawl/SKILL.md

@@ -32,7 +32,8 @@ description: Use this skill when the user asks about social media links, posts,
 - 先看用户给的对象是什么，再选固定 pipeline 或 MCP 工具。
 - 优先返回事实字段，不补充主观分析。
 - 视频文本优先使用平台原生字幕；拿不到再走 ASR。
-- 默认输出以结构化 JSON 为主。
+- 默认输出以结构化 JSON 和Markdown并主动落库。
+- 当固定pipeline时，默认执行抓取并落库（强制）。
 
 ## Workflow
 

package/skills/social-media-crawl/scripts/core/asr_pipeline.py

@@ -1014,6 +1014,159 @@ def run_u2_asr_candidates_with_timeout_retry(
     return final_bundle
 
 
+def run_u3_then_u2_asr_candidates_with_timeout_retry(
+    *,
+    base_url: str,
+    token: str,
+    timeout_ms: int,
+    candidates: List[str],
+    submit_max_retries: int,
+    submit_backoff_ms: int,
+    poll_interval_sec: float,
+    max_polls: int,
+    timeout_retry_enabled: bool = True,
+    timeout_retry_max_retries: int = 3,
+    pending_timeout_sec: int = DEFAULT_U2_PENDING_TIMEOUT_SEC,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
+) -> Dict[str, Any]:
+    normalized_candidates = normalize_media_candidates(candidates)
+    attempts: List[Dict[str, Any]] = []
+
+    final_bundle: Dict[str, Any] = {
+        "submit_bundle": {},
+        "poll_result": {"ok": False, "task_status": "UNKNOWN", "error_reason": "no_candidates"},
+        "rounds": [],
+        "timeout_retry": {
+            "enabled": bool(timeout_retry_enabled),
+            "configured_max_retries": max(0, min(3, int(timeout_retry_max_retries))),
+            "triggered": False,
+            "result": "not_triggered",
+        },
+        "u3_fallback": {
+            "enabled": False,
+            "triggered": False,
+            "ok": False,
+            "result": "not_triggered",
+            "public_url": "",
+            "trace": [],
+        },
+    }
+    chosen_url: Optional[str] = None
+    chosen_public_url: Optional[str] = None
+
+    for index, candidate in enumerate(normalized_candidates, start=1):
+        valid = is_valid_u2_media_candidate(candidate)
+        if not valid:
+            attempts.append(
+                {
+                    "index": index,
+                    "candidate": candidate,
+                    "valid": False,
+                    "result": "skipped_non_media_candidate",
+                }
+            )
+            continue
+
+        u3_result = run_u3_public_url_fallback(
+            base_url=base_url,
+            token=token,
+            timeout_ms=timeout_ms,
+            source_url=candidate,
+        )
+        u3_bundle = {
+            "enabled": True,
+            "triggered": True,
+            "ok": bool(u3_result.get("ok")),
+            "result": "u3_completed" if u3_result.get("ok") else "u3_failed",
+            "public_url": normalize_media_url(u3_result.get("public_url")),
+            "request_id": u3_result.get("request_id"),
+            "error_reason": u3_result.get("error_reason"),
+            "trace": u3_result.get("trace", []),
+        }
+
+        attempts.append(
+            {
+                "index": index,
+                "candidate": candidate,
+                "valid": True,
+                "u3_bridge": u3_bundle,
+            }
+        )
+
+        if not u3_bundle.get("ok") or not u3_bundle.get("public_url"):
+            final_bundle = {
+                "submit_bundle": {},
+                "poll_result": {
+                    "ok": False,
+                    "task_status": "UNKNOWN",
+                    "error_reason": u3_bundle.get("error_reason") or "u3_bridge_failed",
+                    "request_id": u3_bundle.get("request_id"),
+                    "trace": list(u3_bundle.get("trace", [])),
+                },
+                "rounds": [],
+                "timeout_retry": {
+                    "enabled": bool(timeout_retry_enabled),
+                    "configured_max_retries": max(0, min(3, int(timeout_retry_max_retries))),
+                    "triggered": False,
+                    "result": "not_triggered",
+                },
+                "u3_fallback": {
+                    "enabled": False,
+                    "triggered": False,
+                    "ok": False,
+                    "result": "not_triggered",
+                    "public_url": "",
+                    "trace": [],
+                },
+                "u3_bridge": u3_bundle,
+            }
+            continue
+
+        bundle = run_u2_asr_with_timeout_retry(
+            base_url=base_url,
+            token=token,
+            timeout_ms=timeout_ms,
+            video_url=str(u3_bundle.get("public_url")),
+            submit_max_retries=submit_max_retries,
+            submit_backoff_ms=submit_backoff_ms,
+            poll_interval_sec=poll_interval_sec,
+            max_polls=max_polls,
+            timeout_retry_enabled=timeout_retry_enabled,
+            timeout_retry_max_retries=timeout_retry_max_retries,
+            pending_timeout_sec=pending_timeout_sec,
+            u3_fallback_enabled=False,
+            progress_callback=progress_callback,
+        )
+        poll_result = bundle.get("poll_result", {})
+        error_reason = str(poll_result.get("error_reason") or "")
+        ok = bool(poll_result.get("ok"))
+
+        attempts[-1].update(
+            {
+                "ok": ok,
+                "error_reason": error_reason,
+                "task_status": poll_result.get("task_status"),
+                "u2_public_url": u3_bundle.get("public_url"),
+            }
+        )
+
+        final_bundle = dict(bundle)
+        final_bundle["u3_bridge"] = u3_bundle
+        chosen_url = candidate
+        chosen_public_url = str(u3_bundle.get("public_url") or "")
+        if ok:
+            break
+        if error_reason == "INVALID_SOURCE_URL":
+            continue
+        break
+
+    final_bundle["candidate_attempts"] = attempts
+    final_bundle["chosen_candidate"] = chosen_url
+    final_bundle["chosen_public_url"] = chosen_public_url
+    final_bundle["normalized_candidates"] = normalized_candidates
+    return final_bundle
+
+
 def run_u2_asr_batch_with_timeout_retry(
     *,
     base_url: str,

package/skills/social-media-crawl/scripts/pipelines/douyin_metadata.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Shared Douyin metadata extraction helpers."""
+
+from __future__ import annotations
+
+import re
+from typing import Any, Dict, Optional
+
+from scripts.core.tikomni_common import normalize_text
+
+INVALID_AUTHOR_HANDLE_VALUES = {"0", "unknown", "none", "null", "nil", "na", "n/a"}
+MUSIC_TITLE_PATTERN = re.compile(r"^@?.+?(?:创作的原声|作品使用的原声|的原声)$")
+
+
+def _safe_int(value: Any) -> Optional[int]:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return int(value)
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(value)
+
+    text = normalize_text(value)
+    if not text:
+        return None
+    try:
+        return int(float(text.replace(",", "")))
+    except Exception:
+        return None
+
+
+def normalize_douyin_author_handle(*values: Any) -> str:
+    for value in values:
+        text = normalize_text(value)
+        if not text:
+            continue
+        if text.lower() in INVALID_AUTHOR_HANDLE_VALUES:
+            continue
+        return text
+    return ""
+
+
+def looks_like_douyin_music_title(value: Any) -> bool:
+    title = normalize_text(value)
+    if not title:
+        return False
+    return bool(MUSIC_TITLE_PATTERN.match(title))
+
+
+def extract_douyin_caption(item: Dict[str, Any]) -> str:
+    if not isinstance(item, dict):
+        return ""
+    for key in ("desc", "caption", "content", "item_title", "preview_title", "title"):
+        text = normalize_text(item.get(key))
+        if text:
+            return text
+    return ""
+
+
+def title_from_douyin_caption(caption: Any) -> str:
+    text = normalize_text(caption)
+    if not text:
+        return ""
+
+    stripped = re.split(r"\s*#\S+", text, maxsplit=1)[0].strip()
+    if stripped:
+        return stripped
+    return text
+
+
+def extract_douyin_title(item: Dict[str, Any]) -> str:
+    if not isinstance(item, dict):
+        return ""
+
+    # Only read title-like fields from the work object itself.
+    # Nested `music.title` is an audio title, not the work title.
+    caption_title = title_from_douyin_caption(extract_douyin_caption(item))
+    for key in ("item_title", "preview_title", "title"):
+        candidate = normalize_text(item.get(key))
+        if not candidate:
+            continue
+        if looks_like_douyin_music_title(candidate) and caption_title:
+            continue
+        return candidate
+    return caption_title
+
+
+def extract_douyin_author(item: Dict[str, Any]) -> Dict[str, Optional[str]]:
+    author = item.get("author") if isinstance(item.get("author"), dict) else {}
+
+    author_platform_id = (
+        normalize_text(author.get("uid"))
+        or normalize_text(author.get("id"))
+        or normalize_text(item.get("author_user_id"))
+    )
+    author_handle = normalize_douyin_author_handle(
+        author.get("unique_id"),
+        author.get("short_id"),
+        author.get("douyin_id"),
+        author.get("display_id"),
+        author.get("nickname"),
+    )
+    douyin_sec_uid = normalize_text(author.get("sec_uid"))
+    douyin_aweme_author_id = normalize_text(item.get("author_user_id")) or author_platform_id
+
+    return {
+        "author_handle": author_handle or None,
+        "platform_author_id": author_platform_id or None,
+        "author_platform_id": author_platform_id or None,
+        "douyin_sec_uid": douyin_sec_uid or None,
+        "douyin_aweme_author_id": douyin_aweme_author_id or None,
+        "unique_id": normalize_text(author.get("unique_id")) or None,
+        "nickname": normalize_text(author.get("nickname")) or None,
+        "signature": normalize_text(author.get("signature")) or None,
+    }
+
+
+def extract_douyin_metrics(item: Dict[str, Any]) -> Dict[str, Optional[int]]:
+    statistics = item.get("statistics") if isinstance(item.get("statistics"), dict) else {}
+
+    def metric(*keys: str, default: Optional[int] = 0) -> Optional[int]:
+        for key in keys:
+            value = _safe_int(statistics.get(key))
+            if value is not None:
+                return value
+            value = _safe_int(item.get(key))
+            if value is not None:
+                return value
+        return default
+
+    metrics = {
+        "digg_count": metric("digg_count", "like_count"),
+        "comment_count": metric("comment_count"),
+        "collect_count": metric("collect_count"),
+        "share_count": metric("share_count", "forward_count"),
+        "play_count": metric("play_count", "view_count", default=None),
+    }
+
+    play_count = metrics.get("play_count")
+    engagement_floor = max(
+        int(metrics.get("digg_count") or 0),
+        int(metrics.get("comment_count") or 0),
+        int(metrics.get("collect_count") or 0),
+        int(metrics.get("share_count") or 0),
+    )
+    if play_count is not None and int(play_count) <= 0 and engagement_floor > 0:
+        metrics["play_count"] = None
+
+    return metrics