@tikomni/skills 0.1.7 → 0.1.9

package/skills/social-media-crawl/scripts/pipelines/input_contracts.py

@@ -0,0 +1,318 @@
+#!/usr/bin/env python3
+"""Shared input normalization and validation for social-media pipelines."""
+
+from __future__ import annotations
+
+import re
+from typing import Dict, List, Optional
+from urllib.parse import parse_qs, urlparse
+
+from scripts.core.tikomni_common import normalize_text
+
+_HTTP_URL_RE = re.compile(r"https?://[^\s<>'\"，。！？；：）】》]+", re.IGNORECASE)
+_URL_TRAILING_PUNCTUATION = ".,!?;:)]}>'\"，。！？；：）】》、"
+_XHS_NOTE_ID_RE = re.compile(r"^[0-9A-Za-z]{16,32}$")
+_XHS_USER_ID_RE = re.compile(r"^[0-9A-Za-z]{8,32}$")
+_DOUYIN_SEC_UID_RE = re.compile(r"^MS4wLjA[A-Za-z0-9_-]{8,}$")
+
+_DOUYIN_HOST_TOKENS = ("douyin.com", "iesdouyin.com", "v.douyin.com")
+_XHS_HOST_TOKENS = ("xiaohongshu.com", "xhslink.com")
+
+
+def _dedupe_keep_order(items: List[str]) -> List[str]:
+    unique: List[str] = []
+    seen = set()
+    for item in items:
+        if item in seen:
+            continue
+        unique.append(item)
+        seen.add(item)
+    return unique
+
+
+def _strip_url_punctuation(value: str) -> str:
+    return value.rstrip(_URL_TRAILING_PUNCTUATION)
+
+
+def extract_http_urls(text: Optional[str]) -> List[str]:
+    raw = normalize_text(text)
+    if not raw:
+        return []
+    matches = [_strip_url_punctuation(match.group(0)) for match in _HTTP_URL_RE.finditer(raw)]
+    return _dedupe_keep_order([item for item in matches if item])
+
+
+def _first_platform_url(text: Optional[str], host_tokens: tuple[str, ...]) -> Optional[str]:
+    for url in extract_http_urls(text):
+        host = urlparse(url).netloc.lower()
+        if any(token in host for token in host_tokens):
+            return url
+    return None
+
+
+def _text_contains_host(text: Optional[str], host_tokens: tuple[str, ...]) -> bool:
+    lowered = normalize_text(text).lower()
+    return any(token in lowered for token in host_tokens)
+
+
+def looks_like_douyin_sec_uid(value: Optional[str]) -> bool:
+    return bool(_DOUYIN_SEC_UID_RE.fullmatch(normalize_text(value)))
+
+
+def looks_like_xhs_note_id(value: Optional[str]) -> bool:
+    return bool(_XHS_NOTE_ID_RE.fullmatch(normalize_text(value)))
+
+
+def looks_like_xhs_user_id(value: Optional[str]) -> bool:
+    return bool(_XHS_USER_ID_RE.fullmatch(normalize_text(value)))
+
+
+def extract_douyin_share_url(text: Optional[str]) -> Optional[str]:
+    return _first_platform_url(text, _DOUYIN_HOST_TOKENS)
+
+
+def extract_douyin_sec_uid(text: Optional[str]) -> Optional[str]:
+    raw = normalize_text(text)
+    if not raw:
+        return None
+    if looks_like_douyin_sec_uid(raw):
+        return raw
+
+    for url in extract_http_urls(raw):
+        query = parse_qs(urlparse(url).query)
+        for key in ("sec_uid", "sec_user_id"):
+            candidate = normalize_text((query.get(key) or [""])[0])
+            if looks_like_douyin_sec_uid(candidate):
+                return candidate
+
+    match = re.search(r"(?:sec_uid|sec_user_id)=([A-Za-z0-9._-]+)", raw)
+    if match:
+        candidate = normalize_text(match.group(1))
+        if looks_like_douyin_sec_uid(candidate):
+            return candidate
+    return None
+
+
+def extract_xhs_note_id(text: Optional[str]) -> Optional[str]:
+    raw = normalize_text(text)
+    if not raw:
+        return None
+    if looks_like_xhs_note_id(raw):
+        return raw
+
+    for pattern in (
+        r"/explore/([0-9A-Za-z]+)",
+        r"/discovery/item/([0-9A-Za-z]+)",
+        r"note_id=([0-9A-Za-z]+)",
+    ):
+        match = re.search(pattern, raw)
+        if not match:
+            continue
+        candidate = normalize_text(match.group(1))
+        if looks_like_xhs_note_id(candidate):
+            return candidate
+    return None
+
+
+def extract_xhs_user_id(text: Optional[str]) -> Optional[str]:
+    raw = normalize_text(text)
+    if not raw:
+        return None
+    if looks_like_xhs_user_id(raw):
+        return raw
+
+    for pattern in (
+        r"/user/profile/([0-9A-Za-z]+)",
+        r"(?:user_id|userid)=([0-9A-Za-z]+)",
+    ):
+        match = re.search(pattern, raw)
+        if not match:
+            continue
+        candidate = normalize_text(match.group(1))
+        if looks_like_xhs_user_id(candidate):
+            return candidate
+    return None
+
+
+def extract_xhs_note_share_url(text: Optional[str]) -> Optional[str]:
+    for url in extract_http_urls(text):
+        parsed = urlparse(url)
+        host = parsed.netloc.lower()
+        path = parsed.path.lower()
+        query = parsed.query.lower()
+        if "xhslink.com" in host:
+            return url
+        if "xiaohongshu.com" not in host:
+            continue
+        if "/explore/" in path or "/discovery/item/" in path or "note_id=" in query:
+            return url
+    return None
+
+
+def extract_xhs_creator_share_url(text: Optional[str]) -> Optional[str]:
+    for url in extract_http_urls(text):
+        parsed = urlparse(url)
+        host = parsed.netloc.lower()
+        path = parsed.path.lower()
+        query = parsed.query.lower()
+        if "xhslink.com" in host:
+            return url
+        if "xiaohongshu.com" not in host:
+            continue
+        if "/user/profile/" in path or "/user/" in path or "user_id=" in query or "userid=" in query:
+            return url
+    return None
+
+
+def text_has_douyin_short_link(text: Optional[str]) -> bool:
+    return _text_contains_host(text, ("v.douyin.com",))
+
+
+def text_has_xhs_short_link(text: Optional[str]) -> bool:
+    return _text_contains_host(text, ("xhslink.com",))
+
+
+def normalize_douyin_work_input(input_value: Optional[str], share_url: Optional[str]) -> Dict[str, object]:
+    explicit_share = normalize_text(share_url)
+    raw_input = normalize_text(input_value)
+    normalized_share = extract_douyin_share_url(explicit_share) or extract_douyin_share_url(raw_input)
+
+    if normalized_share:
+        return {
+            "share_url": normalized_share,
+            "error_reason": None,
+            "missing_fields": [],
+        }
+
+    if raw_input:
+        if text_has_douyin_short_link(raw_input):
+            return {
+                "share_url": None,
+                "error_reason": "short_link_detected_but_unresolved",
+                "missing_fields": [{"field": "share_url", "reason": "short_link_unresolved"}],
+            }
+        return {
+            "share_url": None,
+            "error_reason": "invalid_share_url",
+            "missing_fields": [{"field": "share_url", "reason": "invalid_input"}],
+        }
+
+    return {
+        "share_url": None,
+        "error_reason": None,
+        "missing_fields": [],
+    }
+
+
+def normalize_xhs_note_input(
+    input_value: Optional[str],
+    share_text: Optional[str],
+    note_id: Optional[str],
+) -> Dict[str, object]:
+    explicit_note_id = normalize_text(note_id)
+    if explicit_note_id and not looks_like_xhs_note_id(explicit_note_id):
+        return {
+            "share_text": None,
+            "note_id": None,
+            "error_reason": "invalid_note_id",
+            "missing_fields": [{"field": "note_id", "reason": "invalid_format"}],
+        }
+
+    explicit_share = normalize_text(share_text)
+    raw_input = normalize_text(input_value)
+
+    normalized_note_id = explicit_note_id or extract_xhs_note_id(explicit_share) or extract_xhs_note_id(raw_input)
+    normalized_share = extract_xhs_note_share_url(explicit_share) or extract_xhs_note_share_url(raw_input)
+
+    if normalized_note_id or normalized_share:
+        return {
+            "share_text": normalized_share or None,
+            "note_id": normalized_note_id or None,
+            "error_reason": None,
+            "missing_fields": [],
+        }
+
+    candidate = explicit_share or raw_input
+    if candidate:
+        if text_has_xhs_short_link(candidate):
+            return {
+                "share_text": None,
+                "note_id": None,
+                "error_reason": "short_link_detected_but_unresolved",
+                "missing_fields": [{"field": "note_id", "reason": "short_link_unresolved"}],
+            }
+        return {
+            "share_text": None,
+            "note_id": None,
+            "error_reason": "invalid_note_id",
+            "missing_fields": [{"field": "note_id", "reason": "invalid_format"}],
+        }
+
+    return {
+        "share_text": None,
+        "note_id": None,
+        "error_reason": None,
+        "missing_fields": [],
+    }
+
+
+def normalize_douyin_creator_input(input_value: Optional[str]) -> Dict[str, object]:
+    raw_input = normalize_text(input_value)
+    normalized_input = extract_douyin_sec_uid(raw_input) or extract_douyin_share_url(raw_input) or raw_input or None
+
+    if extract_douyin_sec_uid(raw_input) or extract_douyin_share_url(raw_input):
+        return {
+            "input_value": normalized_input,
+            "error_reason": None,
+            "missing_fields": [],
+        }
+
+    if raw_input:
+        if text_has_douyin_short_link(raw_input):
+            return {
+                "input_value": None,
+                "error_reason": "short_link_detected_but_unresolved",
+                "missing_fields": [{"field": "platform_author_id", "reason": "short_link_unresolved"}],
+            }
+        return {
+            "input_value": None,
+            "error_reason": "invalid_creator_input",
+            "missing_fields": [{"field": "platform_author_id", "reason": "invalid_format"}],
+        }
+
+    return {
+        "input_value": None,
+        "error_reason": None,
+        "missing_fields": [],
+    }
+
+
+def normalize_xhs_creator_input(input_value: Optional[str]) -> Dict[str, object]:
+    raw_input = normalize_text(input_value)
+    normalized_input = extract_xhs_user_id(raw_input) or extract_xhs_creator_share_url(raw_input) or None
+
+    if normalized_input:
+        return {
+            "input_value": normalized_input,
+            "error_reason": None,
+            "missing_fields": [],
+        }
+
+    if raw_input:
+        if text_has_xhs_short_link(raw_input):
+            return {
+                "input_value": None,
+                "error_reason": "short_link_detected_but_unresolved",
+                "missing_fields": [{"field": "platform_author_id", "reason": "short_link_unresolved"}],
+            }
+        return {
+            "input_value": None,
+            "error_reason": "invalid_creator_input",
+            "missing_fields": [{"field": "platform_author_id", "reason": "invalid_format"}],
+        }
+
+    return {
+        "input_value": None,
+        "error_reason": None,
+        "missing_fields": [],
+    }

package/skills/social-media-crawl/scripts/pipelines/media_url_rules.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Shared media URL classification helpers."""
+
+from __future__ import annotations
+
+from typing import Iterable, List
+
+
+def _is_http_url(url: str) -> bool:
+    lower = (url or "").lower()
+    return lower.startswith("http://") or lower.startswith("https://")
+
+
+def is_probable_image_url(url: str) -> bool:
+    lower = (url or "").lower()
+    if not _is_http_url(lower):
+        return False
+    image_tokens = [
+        ".jpg",
+        ".jpeg",
+        ".png",
+        ".webp",
+        ".gif",
+        "imageview2",
+        "imagemogr2",
+        "redimage",
+        "frame/",
+        "sns-img",
+        "sns-webpic",
+        "notes_pre_post",
+        "/image/",
+        "/img/",
+    ]
+    return any(token in lower for token in image_tokens)
+
+
+def is_probable_audio_url(url: str) -> bool:
+    lower = (url or "").lower()
+    if not _is_http_url(lower):
+        return False
+    audio_tokens = [
+        ".m4a",
+        ".mp3",
+        ".aac",
+        ".wav",
+        ".flac",
+        ".ogg",
+        "/audio/",
+        "sns-audio",
+        "redaudio",
+    ]
+    return any(token in lower for token in audio_tokens)
+
+
+def is_probable_video_url(url: str) -> bool:
+    lower = (url or "").lower()
+    if not _is_http_url(lower):
+        return False
+    if is_probable_image_url(lower) or is_probable_audio_url(lower):
+        return False
+    video_tokens = [
+        ".mp4",
+        ".m3u8",
+        ".mov",
+        ".flv",
+        "/video/",
+        "sns-video",
+        "redvideo",
+        "play",
+        "stream",
+        "master",
+        "vod",
+    ]
+    return any(token in lower for token in video_tokens)
+
+
+def filter_video_urls(urls: Iterable[str]) -> List[str]:
+    unique: List[str] = []
+    seen = set()
+    for raw in urls:
+        url = str(raw or "").strip()
+        if not url or url in seen or not is_probable_video_url(url):
+            continue
+        unique.append(url)
+        seen.add(url)
+    return unique

package/skills/social-media-crawl/scripts/pipelines/platform_adapters.py

@@ -13,6 +13,14 @@ from scripts.pipelines.schema import (
     validate_work_item,
     validate_works_collection,
 )
+from scripts.pipelines.douyin_metadata import (
+    extract_douyin_author,
+    extract_douyin_caption,
+    extract_douyin_metrics,
+    extract_douyin_title,
+    normalize_douyin_author_handle,
+)
+from scripts.pipelines.media_url_rules import is_probable_video_url as is_shared_probable_video_url
 from scripts.core.tikomni_common import deep_find_all, deep_find_first
 from scripts.pipelines.select_low_quality_video_url import select_low_quality_video_url
 
@@ -117,12 +125,7 @@ def _normalize_douyin_tags(value: Any) -> List[str]:
 
 
 def _is_probable_video_url(url: str) -> bool:
-    lower = (url or "").lower()
-    if not (lower.startswith("http://") or lower.startswith("https://")):
-        return False
-    if any(token in lower for token in [".jpg", ".jpeg", ".png", ".webp", "image", "img"]):
-        return False
-    return any(token in lower for token in [".mp4", ".m3u8", ".m4a", "video", "stream", "play"])
+    return is_shared_probable_video_url(url)
 
 
 def _extract_douyin_video_down_url(item: Dict[str, Any]) -> str:
@@ -348,14 +351,21 @@ def adapt_douyin_author_home(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], List[
 
     internal_author_id = _t(_first(profile_data, ["sec_user_id", "sec_uid"], raw.get("resolved_author_id")))
     stable_author_id = _t(_first(profile_data, ["uid", "user_id", "id"]))
-    author_handle = _t(_first(profile_data, ["short_id", "unique_id", "douyin_id", "display_id"]))
+    author_handle = normalize_douyin_author_handle(
+        _first(profile_data, ["unique_id"]),
+        _first(profile_data, ["short_id"]),
+        _first(profile_data, ["douyin_id"]),
+        _first(profile_data, ["display_id"]),
+        _first(profile_data, ["nickname", "name"]),
+    )
+    nickname = _t(_first(profile_data, ["nickname", "name"]))
 
     author_id = internal_author_id or stable_author_id
     profile = build_author_profile(
         platform="douyin",
         platform_author_id=author_id,
         author_handle=author_handle,
-        nickname=_t(_first(profile_data, ["nickname", "name"])),
+        nickname=nickname,
         ip_location=_t(_first(profile_data, ["ip_location", "ip_label", "ipLocation"])),
         fans_count=_i(_first(profile_data, ["follower_count", "fans_count", "mplatform_followers_count"])),
         liked_count=_i(_first(profile_data, ["total_favorited", "liked_count", "favoriting_count"])),
@@ -383,23 +393,27 @@ def adapt_douyin_author_home(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], List[
         if not isinstance(item, dict):
             continue
         aweme_id = _t(_first(item, ["aweme_id", "item_id", "id"]))
-        metrics = {
-            "like": _i(_first(item, ["digg_count", "like_count"], 0)),
-            "comment": _i(_first(item, ["comment_count"], 0)),
-            "collect": _i(_first(item, ["collect_count"], 0)),
-            "share": _i(_first(item, ["share_count"], 0)),
-            "play": _optional_i(_first(item, ["play_count", "view_count"], None)),
-        }
+        author_info = extract_douyin_author(item)
+        metrics = extract_douyin_metrics(item)
         video_down_url = _extract_douyin_video_down_url(item)
         tags = _normalize_douyin_tags(_first(item, ["hashtags", "tags", "text_extra"], []))
+        work_author_handle = normalize_douyin_author_handle(
+            author_info.get("author_handle"),
+            author_handle,
+            nickname,
+        )
+        work_platform_author_id = _t(author_info.get("platform_author_id") or author_id)
+        work_author_platform_id = _t(author_info.get("author_platform_id") or stable_author_id or author_id)
+        work_nickname = _t(author_info.get("nickname") or nickname)
+        work_signature = _t(author_info.get("signature") or profile.get("signature"))
         work = build_work_item(
             platform="douyin",
             platform_work_id=aweme_id,
-            platform_author_id=author_id,
-            author_handle=author_handle,
-            author_platform_id=stable_author_id or author_id,
-            title=_t(_first(item, ["title"])),
-            caption_raw=_t(_first(item, ["desc"])),
+            platform_author_id=work_platform_author_id,
+            author_handle=work_author_handle,
+            author_platform_id=work_author_platform_id,
+            title=extract_douyin_title(item),
+            caption_raw=extract_douyin_caption(item),
             subtitle_raw="",
             subtitle_source="missing",
             publish_time=_t(_first(item, ["create_time", "publish_time"])),
@@ -407,7 +421,12 @@ def adapt_douyin_author_home(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], List[
             content_type="video",
             duration_ms=_i(_first(item, ["duration_ms", "duration"], 0)),
             tags=tags,
-            metrics=metrics,
+            metrics={
+                "digg_count": int(metrics.get("digg_count") or 0),
+                "comment_count": int(metrics.get("comment_count") or 0),
+                "collect_count": int(metrics.get("collect_count") or 0),
+                "share_count": int(metrics.get("share_count") or 0),
+            },
             cover_image=(
                 _extract_first_url(_first(item, ["cover_url"], ""))
                 or _extract_first_url(_first(item, ["cover"], ""))
@@ -420,18 +439,31 @@ def adapt_douyin_author_home(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], List[
             asr_error_reason="",
             asr_source="fallback_none",
             platform_native_refs={
-                "douyin_sec_uid": internal_author_id,
-                "douyin_aweme_author_id": stable_author_id or author_id,
+                "douyin_sec_uid": _t(author_info.get("douyin_sec_uid") or internal_author_id),
+                "douyin_aweme_author_id": _t(author_info.get("douyin_aweme_author_id") or stable_author_id or author_id),
+                "douyin_unique_id": _t(author_info.get("unique_id")),
             },
             raw_ref={"aweme_id": aweme_id, "raw_item": item},
         )
         work.update(
             {
-                "digg_count": metrics["like"],
-                "comment_count": metrics["comment"],
-                "collect_count": metrics["collect"],
-                "share_count": metrics["share"],
-                "play_count": metrics["play"],
+                "author": {
+                    "author_handle": work_author_handle,
+                    "platform_author_id": work_platform_author_id,
+                    "author_platform_id": work_author_platform_id,
+                    "douyin_sec_uid": _t(author_info.get("douyin_sec_uid") or internal_author_id),
+                    "douyin_aweme_author_id": _t(author_info.get("douyin_aweme_author_id") or stable_author_id or author_id),
+                    "unique_id": _t(author_info.get("unique_id")),
+                    "nickname": work_nickname,
+                    "signature": work_signature,
+                },
+                "nickname": work_nickname,
+                "signature": work_signature,
+                "digg_count": int(metrics.get("digg_count") or 0),
+                "comment_count": int(metrics.get("comment_count") or 0),
+                "collect_count": int(metrics.get("collect_count") or 0),
+                "share_count": int(metrics.get("share_count") or 0),
+                "play_count": metrics.get("play_count"),
             }
         )
 
@@ -448,16 +480,18 @@ def adapt_xhs_author_home(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], List[Dic
 
     author_id = _t(_first(profile_data, ["user_id", "userid", "id"], raw.get("resolved_author_id")))
     author_handle = _t(_first(profile_data, ["red_id", "redid", "display_id", "username"]))
+    nickname = _t(_first(profile_data, ["nickname", "name"]))
+    signature = _t(_first(profile_data, ["desc", "signature", "bio", "introduction"]))
     profile = build_author_profile(
         platform="xiaohongshu",
         platform_author_id=author_id,
         author_handle=author_handle,
-        nickname=_t(_first(profile_data, ["nickname", "name"])),
+        nickname=nickname,
         ip_location=_t(_first(profile_data, ["ip_location", "ip_location_desc", "ipLocation"])),
         fans_count=_i(_first(profile_data, ["fans", "fans_count", "follower_count", "followers"])),
         liked_count=_i(_first(profile_data, ["liked_count", "likes", "total_liked", "like_count"])),
         collected_count=_i(_first(profile_data, ["collected_count", "collect_count", "total_collected", "favorite_count"])),
-        signature=_t(_first(profile_data, ["desc", "signature", "bio", "introduction"])),
+        signature=signature,
         avatar_url=_extract_xhs_avatar_url(profile_data),
         works_count=_i(_first(profile_data, ["notes", "note_count", "works_count", "post_count"])),
         verified=bool(_first(profile_data, ["official_verified", "verified"], False)),
@@ -480,6 +514,8 @@ def adapt_xhs_author_home(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], List[Dic
             "share": _i(_first(item, ["share_count"], 0)),
             "play": _optional_i(_first(item, ["view_count", "play_count"], None)),
         }
+        if (metrics["play"] or 0) <= 0 and max(metrics["like"], metrics["comment"], metrics["collect"], metrics["share"]) > 0:
+            metrics["play"] = None
         subtitle_inline = _extract_xhs_subtitle_inline(item)
         subtitle_urls = _extract_xhs_subtitle_urls(item)
         video_down_url = _extract_xhs_video_down_url(item)
@@ -489,6 +525,8 @@ def adapt_xhs_author_home(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], List[Dic
         cover_image = _extract_xhs_cover_image(item)
         source_url = _extract_xhs_source_url(item, note_id)
         share_url = _extract_xhs_share_url(item, note_id)
+        work_nickname = nickname
+        work_signature = signature
 
         work = build_work_item(
             platform="xiaohongshu",
@@ -523,6 +561,15 @@ def adapt_xhs_author_home(raw: Dict[str, Any]) -> Tuple[Dict[str, Any], List[Dic
         )
         work.update(
             {
+                "author": {
+                    "author_handle": author_handle,
+                    "platform_author_id": author_id,
+                    "author_platform_id": author_id,
+                    "nickname": work_nickname,
+                    "signature": work_signature,
+                },
+                "nickname": work_nickname,
+                "signature": work_signature,
                 "digg_count": metrics["like"],
                 "comment_count": metrics["comment"],
                 "collect_count": metrics["collect"],