@tikomni/skills 0.1.3 → 0.1.4

package/skills/social-media-crawl/scripts/pipelines/run_douyin_creator_home.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+"""Douyin creator-home fixed pipeline."""
+
+from __future__ import annotations
+
+if __package__ in {None, ""}:
+    import sys
+    from pathlib import Path
+
+    _self = Path(__file__).resolve()
+    for _parent in _self.parents:
+        if (_parent / "scripts" / "core" / "bootstrap_env.py").is_file():
+            sys.path.insert(0, str(_parent))
+            break
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+from scripts.core.bootstrap_env import bootstrap_for_direct_run
+
+bootstrap_for_direct_run(__file__, __package__)
+
+from scripts.core.completeness import ensure_request_id, evaluate_collection, normalize_missing_fields
+from scripts.core.config_loader import config_get, load_tikomni_config, resolve_storage_paths
+from scripts.core.progress_report import build_progress_reporter
+from scripts.core.storage_router import resolve_author_directory_name
+from scripts.core.tikomni_common import resolve_runtime, write_json_stdout
+from scripts.pipelines.douyin_creator_home_helpers import collect_and_adapt
+from scripts.pipelines.home_asr import enrich_author_home_asr
+from scripts.writers.write_work_fact_card import build_work_fact_card, persist_output_envelope, write_work_fact_card
+
+
+def _write_collection_artifacts(
+    *,
+    profile: Dict[str, Any],
+    works: List[Dict[str, Any]],
+    card_root: str,
+    extract_trace: List[Dict[str, Any]],
+    request_id: str,
+) -> Dict[str, str]:
+    author_dir_name = resolve_author_directory_name(
+        "douyin",
+        str(profile.get("author_handle") or ""),
+        str(profile.get("platform_author_id") or ""),
+    )
+    author_dir = Path(card_root) / "内容系统" / "作品库" / author_dir_name
+    author_dir.mkdir(parents=True, exist_ok=True)
+
+    creator_profile = dict(profile)
+    creator_profile["request_id"] = request_id
+    creator_profile["extract_trace"] = extract_trace
+
+    work_collection = {
+        "platform": "douyin",
+        "platform_author_id": profile.get("platform_author_id"),
+        "count": len(works),
+        "items": [
+            {
+                "platform_work_id": item.get("platform_work_id"),
+                "title": item.get("title"),
+                "published_date": item.get("published_date"),
+            }
+            for item in works
+            if isinstance(item, dict)
+        ],
+        "request_id": request_id,
+        "extract_trace": extract_trace,
+    }
+
+    profile_path = author_dir / "_creator_profile.json"
+    collection_path = author_dir / "_work_collection.json"
+    profile_path.write_text(json.dumps(creator_profile, ensure_ascii=False, indent=2), encoding="utf-8")
+    collection_path.write_text(json.dumps(work_collection, ensure_ascii=False, indent=2), encoding="utf-8")
+    return {"creator_profile_path": str(profile_path), "work_collection_path": str(collection_path)}
+
+
+def run_douyin_creator_home(
+    *,
+    input_value: str,
+    config: Dict[str, Any],
+    runtime: Dict[str, Any],
+    max_items: int,
+    write_card: bool,
+    persist_output: bool,
+) -> Dict[str, Any]:
+    progress = build_progress_reporter(
+        workflow="social-media-crawl",
+        platform="douyin",
+        content_kind="author_home",
+        input_value=input_value,
+        scope="workflow",
+    )
+    progress.started(stage="author_home.workflow", message="douyin author_home workflow started")
+
+    raw, profile, works, missing = collect_and_adapt(
+        input_value=input_value,
+        base_url=runtime["base_url"],
+        token=runtime["token"],
+        timeout_ms=runtime["timeout_ms"],
+        page_size=20,
+        pages_max=50,
+        max_items=max(1, int(max_items)),
+        progress=progress.child(scope="author_home.collect"),
+    )
+
+    asr_strategy = config_get(config, "asr_strategy", {})
+    asr_bundle = enrich_author_home_asr(
+        platform="douyin",
+        works=works,
+        base_url=runtime["base_url"],
+        token=runtime["token"],
+        timeout_ms=runtime["timeout_ms"],
+        poll_interval_sec=float(asr_strategy.get("poll_interval_sec", 3.0)),
+        max_polls=int(asr_strategy.get("max_polls", 30)),
+        douyin_submit_max_retries=int(config_get(config, "asr_strategy.submit_retry.douyin_video.max_retries", 2)),
+        douyin_submit_backoff_ms=int(config_get(config, "asr_strategy.submit_retry.douyin_video.backoff_ms", 1500)),
+        timeout_retry_enabled=bool(config_get(config, "asr_strategy.u2_timeout_retry.enabled", True)),
+        timeout_retry_max_retries=int(config_get(config, "asr_strategy.u2_timeout_retry.max_retries", 0)),
+        progress=progress.child(scope="author_home.asr"),
+    )
+    works = list(asr_bundle.get("works") or [])
+
+    card_root = resolve_storage_paths(config)["card_root"]
+    card_results: List[Dict[str, Any]] = []
+    if write_card:
+        for work in works:
+            card_results.append(
+                write_work_fact_card(
+                    payload=work,
+                    platform="douyin",
+                    card_root=card_root,
+                    storage_config=config,
+                )
+            )
+
+    request_id = ensure_request_id(
+        raw.get("request_id") or profile.get("request_id"),
+        fallback_seed=input_value,
+    )
+    extract_trace = list(raw.get("extract_trace") or []) + list(asr_bundle.get("trace") or [])
+
+    collection_artifacts = _write_collection_artifacts(
+        profile=profile,
+        works=works,
+        card_root=card_root,
+        extract_trace=extract_trace,
+        request_id=request_id,
+    )
+
+    normalized_profile = dict(profile)
+    normalized_profile["request_id"] = request_id
+    normalized_profile["extract_trace"] = extract_trace
+
+    normalized_works = [build_work_fact_card(work, platform="douyin") for work in works]
+    envelope = {
+        "object_type": "creator",
+        "platform": "douyin",
+        "input": input_value,
+        "normalized": {
+            "creator_profile": normalized_profile,
+            "work_collection": {
+                "platform": "douyin",
+                "platform_author_id": profile.get("platform_author_id"),
+                "count": len(normalized_works),
+                "items": normalized_works,
+                "request_id": request_id,
+                "extract_trace": extract_trace,
+            },
+        },
+        "completeness": evaluate_collection(profile, normalized_works),
+        "missing_fields": normalize_missing_fields(missing),
+        "error_reason": None,
+        "extract_trace": extract_trace,
+        "request_id": request_id,
+        "card_write": {
+            "enabled": bool(write_card),
+            "ok": all(item.get("ok") for item in card_results) if card_results else False,
+            "count": len(card_results),
+            "results": card_results,
+        },
+        "collection_artifacts": collection_artifacts,
+    }
+    envelope["output_persist"] = persist_output_envelope(
+        envelope=envelope,
+        storage_config=config,
+        platform="douyin",
+        fallback_identifier=str(profile.get("platform_author_id") or "author-home"),
+    ) if persist_output else {"enabled": False, "skipped": True, "reason": "disabled_by_flag"}
+
+    progress.done(
+        stage="author_home.workflow",
+        message="douyin author_home workflow finished",
+        data={"request_id": request_id, "works_count": len(normalized_works)},
+    )
+    return envelope
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run Douyin creator-home fixed pipeline")
+    parser.add_argument("input", help="Douyin creator homepage or share URL")
+    parser.add_argument("--config", default=None, help="Runtime config YAML path")
+    parser.add_argument("--env-file", default=None, help="Explicit .env path")
+    parser.add_argument("--allow-process-env", action="store_true", help="Allow process env overrides")
+    parser.add_argument("--base-url", default=None, help="Override Tikomni base URL")
+    parser.add_argument("--timeout-ms", type=int, default=None, help="Override timeout in ms")
+    parser.add_argument("--max-items", type=int, default=5, help="Max works to collect from homepage")
+    parser.set_defaults(write_card=True, persist_output=True)
+    parser.add_argument("--write-card", dest="write_card", action="store_true", help="Write work fact cards")
+    parser.add_argument("--no-write-card", dest="write_card", action="store_false", help="Skip card writing")
+    parser.add_argument("--persist-output", dest="persist_output", action="store_true", help="Persist output JSON")
+    parser.add_argument("--no-persist-output", dest="persist_output", action="store_false", help="Skip output JSON persist")
+    args = parser.parse_args()
+
+    config, _ = load_tikomni_config(args.config, env_file=args.env_file, allow_process_env=args.allow_process_env)
+    runtime = resolve_runtime(
+        env_file=args.env_file,
+        api_key_env=str(config_get(config, "runtime.auth_env_key", "TIKOMNI_API_KEY")),
+        base_url=args.base_url or config_get(config, "runtime.base_url", None),
+        timeout_ms=args.timeout_ms or config_get(config, "runtime.timeout_ms", None),
+        allow_process_env=args.allow_process_env,
+    )
+    write_json_stdout(
+        run_douyin_creator_home(
+            input_value=args.input,
+            config=config,
+            runtime=runtime,
+            max_items=int(args.max_items),
+            write_card=bool(args.write_card),
+            persist_output=bool(args.persist_output),
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

package/skills/{single-work-analysis/scripts/platform/douyin/run_douyin_single_video.py → social-media-crawl/scripts/pipelines/run_douyin_single_work.py}

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Douyin single-video low-quality runner (APP first, WEB fallback)."""
+"""Douyin single-work fixed pipeline runner (APP first, WEB fallback)."""
 
 from __future__ import annotations
 
@@ -27,14 +27,12 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
-from scripts.core.config_loader import config_get, load_tikomni_config, resolve_storage_paths
+from scripts.core.config_loader import config_get, load_tikomni_config
 from scripts.core.extract_pipeline import resolve_trace_error_context
 from scripts.core.progress_report import ProgressReporter, build_progress_reporter
-from scripts.core.storage_router import render_output_filename, resolve_json_filename_pattern
-from scripts.platform.douyin.douyin_video_type_matrix import normalize_douyin_video_type
-from scripts.pipeline.asr.asr_pipeline import derive_asr_clean_text, submit_u2_asr_with_retry
-from scripts.pipeline.asr.poll_u2_task import poll_u2_task
-from scripts.platform.douyin.select_low_quality_video_url import select_low_quality_video_url
+from scripts.pipelines.douyin_video_type_matrix import normalize_douyin_video_type
+from scripts.core.asr_pipeline import derive_asr_clean_text, run_u2_asr_with_timeout_retry
+from scripts.pipelines.select_low_quality_video_url import select_low_quality_video_url
 from scripts.core.tikomni_common import (
     call_json_api,
     deep_find_all,
@@ -43,7 +41,11 @@ from scripts.core.tikomni_common import (
     summarize_content,
     write_json_stdout,
 )
-from scripts.writers.write_benchmark_card import write_benchmark_card
+from scripts.writers.write_work_fact_card import (
+    build_work_output_envelope,
+    persist_output_envelope,
+    write_work_fact_card,
+)
 
 APP_ENDPOINT = "/api/u1/v1/douyin/app/v3/fetch_one_video_by_share_url"
 WEB_ENDPOINT = "/api/u1/v1/douyin/web/fetch_one_video_by_share_url"
@@ -128,109 +130,6 @@ def _report_u2_progress(progress: Optional[ProgressReporter], *, stage: str, eve
     progress.progress(stage=stage, message=message, data=payload)
 
 
-def _build_persist_payload(
-    *,
-    result: Dict[str, Any],
-    source_input: Dict[str, Optional[str]],
-    platform_work_id: Optional[str],
-    status: str,
-    written_at: datetime,
-) -> Dict[str, Any]:
-    summary = {
-        "summary": result.get("summary", ""),
-        "insights": result.get("insights", []),
-        "confidence": result.get("confidence"),
-        "error_reason": result.get("error_reason"),
-    }
-    normalized = {
-        "platform": result.get("platform", "douyin"),
-        "content_kind": result.get("content_kind", "single_video"),
-        "platform_work_id": result.get("platform_work_id"),
-        "title": result.get("title"),
-        "duration_ms": result.get("duration_ms"),
-        "is_video": result.get("is_video"),
-        "u2_task_id": result.get("u2_task_id"),
-        "u2_task_status": result.get("u2_task_status"),
-        "request_id": result.get("request_id"),
-        "source": source_input,
-    }
-    return {
-        "meta": {
-            "written_at": written_at.isoformat(timespec="seconds"),
-            "status": status,
-            "platform": "douyin",
-            "identifier": _traceable_identifier(source_input, platform_work_id),
-        },
-        "summary": summary,
-        "normalized": normalized,
-        "raw": result,
-    }
-
-
-def _persist_output_artifact(
-    *,
-    result: Dict[str, Any],
-    source_input: Dict[str, Optional[str]],
-    platform_work_id: Optional[str],
-    storage_config: Optional[Dict[str, Any]],
-    persist_output: bool,
-) -> Dict[str, Any]:
-    if not persist_output:
-        return {"enabled": False, "skipped": True, "reason": "disabled_by_flag"}
-
-    try:
-        paths = resolve_storage_paths(storage_config or {})
-    except Exception as error:
-        return {"enabled": True, "ok": False, "error": f"resolve_storage_paths_failed:{error}"}
-
-    now = datetime.now()
-    date_key = now.strftime("%Y%m%d")
-    timestamp = now.strftime("%Y%m%dT%H%M%S")
-    identifier = _traceable_identifier(source_input, platform_work_id)
-    has_error = bool(result.get("error_reason"))
-    status = "error" if has_error else "success"
-
-    if has_error:
-        target_dir = Path(paths.get("errors_root", "")) / date_key
-    else:
-        target_dir = Path(paths.get("results_root", "")) / date_key
-
-    target_dir.mkdir(parents=True, exist_ok=True)
-    file_name = render_output_filename(
-        pattern=resolve_json_filename_pattern(storage_config),
-        context={
-            "prefix": status,
-            "platform": "douyin",
-            "card_type": "single_work_result",
-            "author_slug": identifier,
-            "title_slug": identifier,
-            "identifier": identifier,
-            "timestamp": timestamp,
-            "date": date_key,
-            "ext": ".json",
-        },
-        default_filename=f"{timestamp}-douyin-{identifier}.json",
-        default_ext=".json",
-    )
-    file_path = target_dir / file_name
-
-    payload = _build_persist_payload(
-        result=result,
-        source_input=source_input,
-        platform_work_id=platform_work_id,
-        status=status,
-        written_at=now,
-    )
-    file_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
-
-    return {
-        "enabled": True,
-        "ok": True,
-        "status": status,
-        "path": str(file_path),
-    }
-
-
 def _finalize_result(
     *,
     result: Dict[str, Any],
@@ -239,14 +138,19 @@ def _finalize_result(
     storage_config: Optional[Dict[str, Any]],
     persist_output: bool,
 ) -> Dict[str, Any]:
-    result["output_persist"] = _persist_output_artifact(
-        result=result,
-        source_input=source_input,
-        platform_work_id=platform_work_id,
+    envelope = build_work_output_envelope(result, platform="douyin")
+    if "card_write" in result:
+        envelope["card_write"] = result.get("card_write")
+    if not persist_output:
+        envelope["output_persist"] = {"enabled": False, "skipped": True, "reason": "disabled_by_flag"}
+        return envelope
+    envelope["output_persist"] = persist_output_envelope(
+        envelope=envelope,
         storage_config=storage_config,
-        persist_output=persist_output,
+        platform="douyin",
+        fallback_identifier=platform_work_id or _traceable_identifier(source_input, platform_work_id),
     )
-    return result
+    return envelope
 
 def _normalize_input(
     input_value: Optional[str],
@@ -872,7 +776,7 @@ def run_douyin_single_video(
         )
         if write_card:
             card_started_at = time.perf_counter()
-            result["card_write"] = write_benchmark_card(
+            result["card_write"] = write_work_fact_card(
                 payload=result,
                 platform="douyin",
                 card_type=card_type,
@@ -982,7 +886,7 @@ def run_douyin_single_video(
         )
         if write_card:
             card_started_at = time.perf_counter()
-            result["card_write"] = write_benchmark_card(
+            result["card_write"] = write_work_fact_card(
                 payload=result,
                 platform="douyin",
                 card_type=card_type,
@@ -1038,7 +942,7 @@ def run_douyin_single_video(
         )
         if write_card:
             card_started_at = time.perf_counter()
-            result["card_write"] = write_benchmark_card(
+            result["card_write"] = write_work_fact_card(
                 payload=result,
                 platform="douyin",
                 card_type=card_type,
@@ -1143,20 +1047,23 @@ def run_douyin_single_video(
                 data={"video_down_url_present": True, "timeout_ms": u2_timeout_ms},
             )
         submit_started_at = time.perf_counter()
-        submit_bundle = submit_u2_asr_with_retry(
+        submit_bundle = run_u2_asr_with_timeout_retry(
             base_url=runtime["base_url"],
             token=runtime["token"],
             timeout_ms=u2_timeout_ms,
             video_url=video_down_url,
-            max_retries=u2_submit_max_retries,
-            backoff_ms=u2_submit_backoff_ms,
+            submit_max_retries=u2_submit_max_retries,
+            submit_backoff_ms=u2_submit_backoff_ms,
+            poll_interval_sec=poll_interval_sec,
+            max_polls=max_polls,
+            pending_timeout_sec=int(config_get(storage_config or {}, "runtime.u2_pending_timeout_sec", 60) or 60),
             progress_callback=(
                 lambda event: _report_u2_progress(progress, stage="single_video.u2", event=event, label="douyin")
             ) if progress is not None else None,
         )
-        timings["u2_submit_ms"] = _elapsed_ms(submit_started_at)
-        submit_response = submit_bundle["submit_response"]
-        u2_task_id = submit_bundle.get("task_id")
+        timings["u2_submit_ms"] = _safe_int(submit_bundle.get("submit_duration_ms")) or _elapsed_ms(submit_started_at)
+        submit_response = (submit_bundle.get("submit_bundle") or {}).get("submit_response", {})
+        u2_task_id = (submit_bundle.get("submit_bundle") or {}).get("task_id")
         if progress is not None:
             progress.http_event(
                 stage="single_video.u2",
@@ -1165,8 +1072,8 @@ def run_douyin_single_video(
                 route_label="u2_submit",
                 summary={
                     "task_id": u2_task_id,
-                    "final_submit_status": submit_bundle.get("final_submit_status"),
-                    "retry_count": len(submit_bundle.get("retry_chain", [])),
+                    "final_submit_status": (submit_bundle.get("submit_bundle") or {}).get("final_submit_status"),
+                    "retry_count": len(((submit_bundle.get("submit_bundle") or {}).get("retry_chain") or [])),
                 },
             )
 
@@ -1178,7 +1085,7 @@ def run_douyin_single_video(
                 extra={
                     "task_id": u2_task_id,
                     "video_down_url": video_down_url,
-                    "final_submit_status": submit_bundle.get("final_submit_status"),
+                    "final_submit_status": (submit_bundle.get("submit_bundle") or {}).get("final_submit_status"),
                 },
             )
         )
@@ -1191,7 +1098,7 @@ def run_douyin_single_video(
                     "u2_submit_max_retries": max(0, int(u2_submit_max_retries)),
                     "u2_submit_backoff_ms": max(0, int(u2_submit_backoff_ms)),
                 },
-                "attempts": submit_bundle.get("retry_chain", []),
+                "attempts": ((submit_bundle.get("submit_bundle") or {}).get("retry_chain") or []),
             }
         )
 
@@ -1205,21 +1112,8 @@ def run_douyin_single_video(
             error_reason = submit_response.get("error_reason") or "u2_submit_failed_or_missing_task_id"
             u2_task_status = "UNKNOWN"
         else:
-            if progress is not None:
-                progress.progress(stage="single_video.u2", message="polling douyin u2 task", data={"task_id": u2_task_id})
-            poll_started_at = time.perf_counter()
-            poll_result = poll_u2_task(
-                base_url=runtime["base_url"],
-                token=runtime["token"],
-                timeout_ms=u2_timeout_ms,
-                task_id=u2_task_id,
-                poll_interval_sec=poll_interval_sec,
-                max_polls=max_polls,
-                progress_callback=(
-                    lambda event: _report_u2_progress(progress, stage="single_video.u2", event=event, label="douyin")
-                ) if progress is not None else None,
-            )
-            timings["u2_poll_ms"] = _elapsed_ms(poll_started_at)
+            poll_result = submit_bundle.get("poll_result", {})
+            timings["u2_poll_ms"] = _safe_int(submit_bundle.get("poll_duration_ms"))
             u2_task_status = poll_result.get("task_status") or "UNKNOWN"
             raw_content = poll_result.get("transcript_text", "") if poll_result.get("ok") else ""
             error_reason = poll_result.get("error_reason")
@@ -1234,8 +1128,22 @@ def run_douyin_single_video(
                     "request_id": poll_result.get("request_id"),
                     "error_reason": poll_result.get("error_reason"),
                     "attempts": len(poll_result.get("trace", [])),
+                    "timeout_retry": submit_bundle.get("timeout_retry", {}),
+                    "u3_fallback": submit_bundle.get("u3_fallback", {}),
                 }
             )
+            if submit_bundle.get("u3_fallback", {}).get("triggered"):
+                trace.append(
+                    {
+                        "step": "u3_fallback",
+                        "triggered": True,
+                        "ok": submit_bundle.get("u3_fallback", {}).get("ok"),
+                        "result": submit_bundle.get("u3_fallback", {}).get("result"),
+                        "public_url": submit_bundle.get("u3_fallback", {}).get("public_url"),
+                        "error_reason": submit_bundle.get("u3_fallback", {}).get("error_reason"),
+                        "trace": submit_bundle.get("u3_fallback", {}).get("trace", []),
+                    }
+                )
             if progress is not None:
                 (progress.done if poll_result.get("ok") else progress.failed)(
                     stage="single_video.u2",
@@ -1292,7 +1200,7 @@ def run_douyin_single_video(
         if progress is not None:
             progress.progress(stage="single_video.card_write", message="writing douyin single_video card")
         card_started_at = time.perf_counter()
-        result["card_write"] = write_benchmark_card(
+        result["card_write"] = write_work_fact_card(
             payload=result,
             platform="douyin",
             card_type=card_type,
@@ -1332,7 +1240,7 @@ def run_douyin_single_video(
 
 
 def main() -> None:
-    parser = argparse.ArgumentParser(description="Run Douyin single-video low-quality extraction")
+    parser = argparse.ArgumentParser(description="Run Douyin single-work fixed pipeline")
     parser.add_argument("input", nargs="?", default=None, help="Douyin share URL")
     parser.add_argument("--share-url", default=None, help="Douyin share URL")
     parser.add_argument("--config", default=None, help="Runtime config YAML path")
@@ -1359,7 +1267,8 @@ def main() -> None:
     )
     parser.add_argument("--card-type", choices=["work"], default="work", help="Primary card type")
     parser.add_argument("--content-kind", default="single_video", help="Routing kind, e.g. single_video/work")
-    parser.add_argument("--analysis-mode", choices=["auto", "local"], default="auto", help="Card analysis mode")
+    parser.add_argument("--card-mode", dest="analysis_mode", choices=["standard", "auto", "local"], default="standard", help="Fact-card mode")
+    parser.add_argument("--analysis-mode", dest="analysis_mode", choices=["standard", "auto", "local"], help=argparse.SUPPRESS)
     parser.set_defaults(write_card=True, persist_output=True)
     parser.add_argument("--write-card", dest="write_card", action="store_true", help="Write final work card")
     parser.add_argument("--no-write-card", dest="write_card", action="store_false", help="Skip card writing")
@@ -1379,7 +1288,7 @@ def main() -> None:
     timeout_ms = args.timeout_ms if args.timeout_ms is not None else config_get(config, "runtime.timeout_ms", None)
 
     progress = build_progress_reporter(
-        workflow="single-work-analysis",
+        workflow="social-media-crawl",
         platform="douyin",
         content_kind=args.content_kind,
         input_value=args.share_url or args.input,