@tikomni/skills 0.1.7 → 0.1.9

package/skills/social-media-crawl/scripts/pipelines/run_douyin_creator_home.py

@@ -27,10 +27,15 @@ from scripts.core.config_loader import config_get, load_tikomni_config, resolve_
 from scripts.core.progress_report import build_progress_reporter
 from scripts.core.storage_router import resolve_author_directory_name
 from scripts.core.tikomni_common import resolve_runtime, write_json_stdout
+from scripts.pipelines.input_contracts import normalize_douyin_creator_input
+from scripts.pipelines.schema import build_author_profile
 from scripts.pipelines.douyin_creator_home_helpers import collect_and_adapt
 from scripts.pipelines.home_asr import enrich_author_home_asr
 from scripts.writers.write_work_fact_card import build_work_fact_card, persist_output_envelope, write_work_fact_card
 
+DEFAULT_MAX_ITEMS = 200
+MAX_ITEMS_HARD_LIMIT = 200
+
 
 def _write_collection_artifacts(
     *,
@@ -81,11 +86,12 @@ def run_douyin_creator_home(
     *,
     input_value: str,
     config: Dict[str, Any],
-    runtime: Dict[str, Any],
+    runtime: Dict[str, Any] | None,
     max_items: int,
     write_card: bool,
     persist_output: bool,
 ) -> Dict[str, Any]:
+    bounded_max_items = max(1, min(int(max_items), MAX_ITEMS_HARD_LIMIT))
     progress = build_progress_reporter(
         workflow="social-media-crawl",
         platform="douyin",
@@ -94,15 +100,69 @@ def run_douyin_creator_home(
         scope="workflow",
     )
     progress.started(stage="author_home.workflow", message="douyin author_home workflow started")
+    preflight = normalize_douyin_creator_input(input_value)
+    normalized_input_value = str(preflight.get("input_value") or "")
+    if preflight.get("error_reason"):
+        request_id = ensure_request_id(None, fallback_seed=input_value)
+        empty_profile = build_author_profile(platform="douyin", request_id=request_id)
+        extract_trace = [
+            {
+                "step": "input.preflight",
+                "ok": False,
+                "input_kind": "creator_url_or_sec_uid",
+                "normalized_input_value": normalized_input_value or None,
+                "error_reason": preflight.get("error_reason"),
+                "missing_fields": list(preflight.get("missing_fields") or []),
+            }
+        ]
+        envelope = {
+            "object_type": "creator",
+            "platform": "douyin",
+            "input": input_value,
+            "normalized": {
+                "creator_profile": {**empty_profile, "request_id": request_id, "extract_trace": extract_trace},
+                "work_collection": {
+                    "platform": "douyin",
+                    "platform_author_id": "",
+                    "count": 0,
+                    "items": [],
+                    "request_id": request_id,
+                    "extract_trace": extract_trace,
+                },
+            },
+            "completeness": evaluate_collection(empty_profile, []),
+            "missing_fields": normalize_missing_fields(preflight.get("missing_fields")),
+            "error_reason": str(preflight.get("error_reason") or "invalid_creator_input"),
+            "extract_trace": extract_trace,
+            "request_id": request_id,
+            "card_write": {
+                "enabled": bool(write_card),
+                "ok": False,
+                "count": 0,
+                "results": [],
+                "reason": "skipped_invalid_input",
+            },
+            "collection_artifacts": {},
+            "output_persist": {"enabled": False, "skipped": True, "reason": "invalid_input"},
+        }
+        progress.done(
+            stage="author_home.workflow",
+            message="douyin author_home workflow finished",
+            data={"request_id": request_id, "works_count": 0, "error_reason": envelope["error_reason"]},
+        )
+        return envelope
+
+    if runtime is None:
+        raise ValueError("runtime_required_for_valid_input")
 
     raw, profile, works, missing = collect_and_adapt(
-        input_value=input_value,
+        input_value=normalized_input_value or input_value,
         base_url=runtime["base_url"],
         token=runtime["token"],
         timeout_ms=runtime["timeout_ms"],
         page_size=20,
         pages_max=50,
-        max_items=max(1, int(max_items)),
+        max_items=bounded_max_items,
         progress=progress.child(scope="author_home.collect"),
     )
 
@@ -138,7 +198,7 @@ def run_douyin_creator_home(
 
     request_id = ensure_request_id(
         raw.get("request_id") or profile.get("request_id"),
-        fallback_seed=input_value,
+        fallback_seed=normalized_input_value or input_value,
     )
     extract_trace = list(raw.get("extract_trace") or []) + list(asr_bundle.get("trace") or [])
 
@@ -206,7 +266,12 @@ def main() -> None:
     parser.add_argument("--allow-process-env", action="store_true", help="Allow process env overrides")
     parser.add_argument("--base-url", default=None, help="Override Tikomni base URL")
     parser.add_argument("--timeout-ms", type=int, default=None, help="Override timeout in ms")
-    parser.add_argument("--max-items", type=int, default=5, help="Max works to collect from homepage")
+    parser.add_argument(
+        "--max-items",
+        type=int,
+        default=DEFAULT_MAX_ITEMS,
+        help=f"Max works to collect from homepage (default full crawl, capped at {MAX_ITEMS_HARD_LIMIT})",
+    )
     parser.set_defaults(write_card=True, persist_output=True)
     parser.add_argument("--write-card", dest="write_card", action="store_true", help="Write work fact cards")
     parser.add_argument("--no-write-card", dest="write_card", action="store_false", help="Skip card writing")
@@ -215,6 +280,19 @@ def main() -> None:
     args = parser.parse_args()
 
     config, _ = load_tikomni_config(args.config, env_file=args.env_file, allow_process_env=args.allow_process_env)
+    preflight = normalize_douyin_creator_input(args.input)
+    if preflight.get("error_reason"):
+        write_json_stdout(
+            run_douyin_creator_home(
+                input_value=args.input,
+                config=config,
+                runtime=None,
+                max_items=int(args.max_items),
+                write_card=bool(args.write_card),
+                persist_output=bool(args.persist_output),
+            )
+        )
+        return
     runtime = resolve_runtime(
         env_file=args.env_file,
         api_key_env=str(config_get(config, "runtime.auth_env_key", "TIKOMNI_API_KEY")),
@@ -224,7 +302,7 @@ def main() -> None:
     )
     write_json_stdout(
         run_douyin_creator_home(
-            input_value=args.input,
+            input_value=str(preflight.get("input_value") or args.input),
             config=config,
             runtime=runtime,
             max_items=int(args.max_items),

package/skills/social-media-crawl/scripts/pipelines/run_douyin_single_work.py

@@ -31,6 +31,13 @@ from scripts.core.config_loader import config_get, load_tikomni_config
 from scripts.core.extract_pipeline import resolve_trace_error_context
 from scripts.core.progress_report import ProgressReporter, build_progress_reporter
 from scripts.pipelines.douyin_video_type_matrix import normalize_douyin_video_type
+from scripts.pipelines.douyin_metadata import (
+    extract_douyin_author as extract_shared_douyin_author,
+    extract_douyin_caption as extract_shared_douyin_caption,
+    extract_douyin_metrics as extract_shared_douyin_metrics,
+    extract_douyin_title as extract_shared_douyin_title,
+)
+from scripts.pipelines.input_contracts import normalize_douyin_work_input
 from scripts.core.asr_pipeline import derive_asr_clean_text, run_u2_asr_with_timeout_retry
 from scripts.pipelines.select_low_quality_video_url import select_low_quality_video_url
 from scripts.core.tikomni_common import (
@@ -156,14 +163,8 @@ def _normalize_input(
     input_value: Optional[str],
     share_url: Optional[str],
 ) -> Dict[str, Optional[str]]:
-    normalized_share = (share_url or "").strip() or None
-
-    if input_value and not normalized_share:
-        candidate = input_value.strip()
-        if candidate.startswith("http://") or candidate.startswith("https://"):
-            normalized_share = candidate
-
-    return {"share_url": normalized_share}
+    normalized = normalize_douyin_work_input(input_value, share_url)
+    return {"share_url": normalize_text(normalized.get("share_url")) or None}
 
 
 def _extract_aweme_detail(payload: Any) -> Optional[Dict[str, Any]]:
@@ -238,76 +239,19 @@ def _normalize_duration_ms(item: Dict[str, Any]) -> Optional[int]:
 
 
 def _pick_title(item: Dict[str, Any]) -> str:
-    for key in ("item_title", "title", "desc", "preview_title"):
-        value = item.get(key)
-        text = normalize_text(value)
-        if text:
-            return text
-    return ""
+    return extract_shared_douyin_title(item)
 
 
 def _pick_desc(item: Dict[str, Any]) -> str:
-    for key in ("desc", "item_title", "title", "preview_title"):
-        value = item.get(key)
-        text = normalize_text(value)
-        if text:
-            return text
-    return ""
+    return extract_shared_douyin_caption(item)
 
 
 def _extract_author(item: Dict[str, Any]) -> Dict[str, Optional[str]]:
-    author = item.get("author")
-    if not isinstance(author, dict):
-        author = {}
-
-    author_platform_id = normalize_text(author.get("uid")) or normalize_text(author.get("id")) or normalize_text(item.get("author_user_id"))
-    author_handle = normalize_text(author.get("short_id")) or normalize_text(author.get("nickname"))
-    douyin_sec_uid = normalize_text(author.get("sec_uid"))
-    douyin_aweme_author_id = normalize_text(item.get("author_user_id")) or author_platform_id
-
-    return {
-        "author_handle": author_handle or None,
-        "platform_author_id": author_platform_id or None,
-        "author_platform_id": author_platform_id or None,
-        "douyin_sec_uid": douyin_sec_uid or None,
-        "douyin_aweme_author_id": douyin_aweme_author_id or None,
-        "nickname": normalize_text(author.get("nickname")) or None,
-        "signature": normalize_text(author.get("signature")) or None,
-    }
+    return extract_shared_douyin_author(item)
 
 
 def _extract_metrics(item: Dict[str, Any]) -> Dict[str, Optional[int]]:
-    statistics = item.get("statistics")
-    if not isinstance(statistics, dict):
-        statistics = {}
-
-    def metric(*keys: str, default: Optional[int] = 0) -> Optional[int]:
-        for key in keys:
-            value = _safe_int(statistics.get(key))
-            if value is not None:
-                return value
-            value = _safe_int(item.get(key))
-            if value is not None:
-                return value
-        return default
-
-    metrics = {
-        "digg_count": metric("digg_count"),
-        "comment_count": metric("comment_count"),
-        "collect_count": metric("collect_count"),
-        "share_count": metric("share_count", "forward_count"),
-        "play_count": metric("play_count", default=None),
-    }
-    play_count = metrics.get("play_count")
-    engagement_floor = max(
-        int(metrics.get("digg_count") or 0),
-        int(metrics.get("comment_count") or 0),
-        int(metrics.get("collect_count") or 0),
-        int(metrics.get("share_count") or 0),
-    )
-    if play_count is not None and int(play_count) <= 0 and engagement_floor > 0:
-        metrics["play_count"] = None
-    return metrics
+    return extract_shared_douyin_metrics(item)
 
 
 def _extract_platform_work_id(item: Dict[str, Any]) -> Optional[str]:
@@ -633,6 +577,7 @@ def _build_result(
     cover_image: Optional[str] = None,
     asr_source: str = "fallback_none",
     timings: Optional[Dict[str, int]] = None,
+    missing_fields: Optional[List[Dict[str, str]]] = None,
 ) -> Dict[str, Any]:
     summary_block = summarize_content(raw_content, source="douyin:single-video-low-quality")
     insights = list(summary_block.get("insights", []))
@@ -698,13 +643,13 @@ def _build_result(
         "insights": insights,
         "confidence": confidence,
         "error_reason": error_reason,
-        "missing_fields": _build_missing_fields(
+        "missing_fields": list(missing_fields or _build_missing_fields(
             title=title,
             desc=desc,
             platform_work_id=platform_work_id,
             video_down_url=video_down_url,
             author=author,
-        ),
+        )),
         "extract_trace": extract_trace,
         "fallback_trace": fallback_trace,
         "request_id": request_id,
@@ -741,7 +686,8 @@ def run_douyin_single_video(
     workflow_started_at = time.perf_counter()
     timings = _empty_timings()
     parse_started_at = time.perf_counter()
-    source_input = _normalize_input(input_value, share_url)
+    preflight = normalize_douyin_work_input(input_value, share_url)
+    source_input = {"share_url": normalize_text(preflight.get("share_url")) or None}
     timings["url_parse_ms"] = _elapsed_ms(parse_started_at)
     if progress is not None:
         progress.started(
@@ -749,6 +695,66 @@ def run_douyin_single_video(
             message="douyin single_video workflow started",
             data={"analysis_mode": analysis_mode, "write_card": bool(write_card), "persist_output": bool(persist_output)},
         )
+    preflight_trace = [
+        {
+            "step": "input.preflight",
+            "ok": preflight.get("error_reason") is None,
+            "input_kind": "share_url",
+            "normalized_share_url": source_input.get("share_url"),
+            "error_reason": preflight.get("error_reason"),
+            "missing_fields": list(preflight.get("missing_fields") or []),
+        }
+    ]
+    if preflight.get("error_reason"):
+        result = _build_result(
+            source_input=source_input,
+            platform_work_id=None,
+            title="",
+            desc="",
+            duration_ms=None,
+            video_down_url=None,
+            author={"author_handle": None, "author_platform_id": None, "douyin_sec_uid": None, "douyin_aweme_author_id": None, "nickname": None, "signature": None},
+            metrics=_empty_metrics(),
+            tags=[],
+            is_video=False,
+            video_type_reason="invalid_input",
+            raw_content="",
+            confidence="low",
+            error_reason=str(preflight.get("error_reason") or "invalid_share_url"),
+            extract_trace=preflight_trace,
+            fallback_trace=[],
+            request_id=None,
+            u2_task_id=None,
+            u2_task_status="UNKNOWN",
+            u2_gate_reason="invalid_input",
+            analysis_mode=analysis_mode,
+            timings=timings,
+            missing_fields=list(preflight.get("missing_fields") or []),
+        )
+        if write_card:
+            card_started_at = time.perf_counter()
+            result["card_write"] = write_work_fact_card(
+                payload=result,
+                platform="douyin",
+                card_type=card_type,
+                card_root=card_root,
+                content_kind=content_kind,
+                storage_config=storage_config,
+                analysis_mode=analysis_mode,
+                progress=progress.child(scope="card_write") if progress is not None else None,
+            )
+            timings["card_write_ms"] = _elapsed_ms(card_started_at)
+            timings["llm_analysis_ms"] = _safe_int((result.get("card_write") or {}).get("llm_analysis_ms"))
+        timings["total_ms"] = _elapsed_ms(workflow_started_at)
+        result["timings"] = dict(timings)
+        _update_pipeline_status(result)
+        return _finalize_result(
+            result=result,
+            source_input=source_input,
+            platform_work_id=None,
+            storage_config=storage_config,
+            persist_output=persist_output,
+        )
     if not source_input.get("share_url"):
         result = _build_result(
             source_input=source_input,
@@ -765,7 +771,7 @@ def run_douyin_single_video(
             raw_content="",
             confidence="low",
             error_reason="missing_share_url",
-            extract_trace=[],
+            extract_trace=preflight_trace,
             fallback_trace=[],
             request_id=None,
             u2_task_id=None,

package/skills/social-media-crawl/scripts/pipelines/run_xiaohongshu_creator_home.py

@@ -28,9 +28,14 @@ from scripts.core.progress_report import build_progress_reporter
 from scripts.core.storage_router import resolve_author_directory_name
 from scripts.core.tikomni_common import resolve_runtime, write_json_stdout
 from scripts.pipelines.home_asr import enrich_author_home_asr
+from scripts.pipelines.input_contracts import normalize_xhs_creator_input
+from scripts.pipelines.schema import build_author_profile
 from scripts.pipelines.xiaohongshu_creator_home_helpers import collect_and_adapt
 from scripts.writers.write_work_fact_card import build_work_fact_card, persist_output_envelope, write_work_fact_card
 
+DEFAULT_MAX_ITEMS = 200
+MAX_ITEMS_HARD_LIMIT = 200
+
 
 def _write_collection_artifacts(
     *,
@@ -81,11 +86,12 @@ def run_xiaohongshu_creator_home(
     *,
     input_value: str,
     config: Dict[str, Any],
-    runtime: Dict[str, Any],
+    runtime: Dict[str, Any] | None,
     max_items: int,
     write_card: bool,
     persist_output: bool,
 ) -> Dict[str, Any]:
+    bounded_max_items = max(1, min(int(max_items), MAX_ITEMS_HARD_LIMIT))
     progress = build_progress_reporter(
         workflow="social-media-crawl",
         platform="xiaohongshu",
@@ -94,15 +100,69 @@ def run_xiaohongshu_creator_home(
         scope="workflow",
     )
     progress.started(stage="author_home.workflow", message="xiaohongshu author_home workflow started")
+    preflight = normalize_xhs_creator_input(input_value)
+    normalized_input_value = str(preflight.get("input_value") or "")
+    if preflight.get("error_reason"):
+        request_id = ensure_request_id(None, fallback_seed=input_value)
+        empty_profile = build_author_profile(platform="xiaohongshu", request_id=request_id)
+        extract_trace = [
+            {
+                "step": "input.preflight",
+                "ok": False,
+                "input_kind": "creator_url_or_user_id",
+                "normalized_input_value": normalized_input_value or None,
+                "error_reason": preflight.get("error_reason"),
+                "missing_fields": list(preflight.get("missing_fields") or []),
+            }
+        ]
+        envelope = {
+            "object_type": "creator",
+            "platform": "xiaohongshu",
+            "input": input_value,
+            "normalized": {
+                "creator_profile": {**empty_profile, "request_id": request_id, "extract_trace": extract_trace},
+                "work_collection": {
+                    "platform": "xiaohongshu",
+                    "platform_author_id": "",
+                    "count": 0,
+                    "items": [],
+                    "request_id": request_id,
+                    "extract_trace": extract_trace,
+                },
+            },
+            "completeness": evaluate_collection(empty_profile, []),
+            "missing_fields": normalize_missing_fields(preflight.get("missing_fields")),
+            "error_reason": str(preflight.get("error_reason") or "invalid_creator_input"),
+            "extract_trace": extract_trace,
+            "request_id": request_id,
+            "card_write": {
+                "enabled": bool(write_card),
+                "ok": False,
+                "count": 0,
+                "results": [],
+                "reason": "skipped_invalid_input",
+            },
+            "collection_artifacts": {},
+            "output_persist": {"enabled": False, "skipped": True, "reason": "invalid_input"},
+        }
+        progress.done(
+            stage="author_home.workflow",
+            message="xiaohongshu author_home workflow finished",
+            data={"request_id": request_id, "works_count": 0, "error_reason": envelope["error_reason"]},
+        )
+        return envelope
+
+    if runtime is None:
+        raise ValueError("runtime_required_for_valid_input")
 
     raw, profile, works, missing = collect_and_adapt(
-        input_value=input_value,
+        input_value=normalized_input_value or input_value,
         base_url=runtime["base_url"],
         token=runtime["token"],
         timeout_ms=runtime["timeout_ms"],
         page_size=20,
         pages_max=50,
-        max_items=max(1, int(max_items)),
+        max_items=bounded_max_items,
         progress=progress.child(scope="author_home.collect"),
     )
 
@@ -138,7 +198,7 @@ def run_xiaohongshu_creator_home(
 
     request_id = ensure_request_id(
         raw.get("request_id") or profile.get("request_id"),
-        fallback_seed=input_value,
+        fallback_seed=normalized_input_value or input_value,
     )
     extract_trace = list(raw.get("extract_trace") or []) + list(asr_bundle.get("trace") or [])
 
@@ -206,7 +266,12 @@ def main() -> None:
     parser.add_argument("--allow-process-env", action="store_true", help="Allow process env overrides")
     parser.add_argument("--base-url", default=None, help="Override Tikomni base URL")
     parser.add_argument("--timeout-ms", type=int, default=None, help="Override timeout in ms")
-    parser.add_argument("--max-items", type=int, default=5, help="Max works to collect from homepage")
+    parser.add_argument(
+        "--max-items",
+        type=int,
+        default=DEFAULT_MAX_ITEMS,
+        help=f"Max works to collect from homepage (default full crawl, capped at {MAX_ITEMS_HARD_LIMIT})",
+    )
     parser.set_defaults(write_card=True, persist_output=True)
     parser.add_argument("--write-card", dest="write_card", action="store_true", help="Write work fact cards")
     parser.add_argument("--no-write-card", dest="write_card", action="store_false", help="Skip card writing")
@@ -215,6 +280,19 @@ def main() -> None:
     args = parser.parse_args()
 
     config, _ = load_tikomni_config(args.config, env_file=args.env_file, allow_process_env=args.allow_process_env)
+    preflight = normalize_xhs_creator_input(args.input)
+    if preflight.get("error_reason"):
+        write_json_stdout(
+            run_xiaohongshu_creator_home(
+                input_value=args.input,
+                config=config,
+                runtime=None,
+                max_items=int(args.max_items),
+                write_card=bool(args.write_card),
+                persist_output=bool(args.persist_output),
+            )
+        )
+        return
     runtime = resolve_runtime(
         env_file=args.env_file,
         api_key_env=str(config_get(config, "runtime.auth_env_key", "TIKOMNI_API_KEY")),
@@ -224,7 +302,7 @@ def main() -> None:
     )
     write_json_stdout(
         run_xiaohongshu_creator_home(
-            input_value=args.input,
+            input_value=str(preflight.get("input_value") or args.input),
             config=config,
             runtime=runtime,
             max_items=int(args.max_items),