@tikomni/skills 0.1.3 → 0.1.4

package/skills/social-media-crawl/scripts/pipelines/run_xiaohongshu_creator_home.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+"""Xiaohongshu creator-home fixed pipeline."""
+
+from __future__ import annotations
+
+if __package__ in {None, ""}:
+    import sys
+    from pathlib import Path
+
+    _self = Path(__file__).resolve()
+    for _parent in _self.parents:
+        if (_parent / "scripts" / "core" / "bootstrap_env.py").is_file():
+            sys.path.insert(0, str(_parent))
+            break
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any, Dict, List
+
+from scripts.core.bootstrap_env import bootstrap_for_direct_run
+
+bootstrap_for_direct_run(__file__, __package__)
+
+from scripts.core.completeness import ensure_request_id, evaluate_collection, normalize_missing_fields
+from scripts.core.config_loader import config_get, load_tikomni_config, resolve_storage_paths
+from scripts.core.progress_report import build_progress_reporter
+from scripts.core.storage_router import resolve_author_directory_name
+from scripts.core.tikomni_common import resolve_runtime, write_json_stdout
+from scripts.pipelines.home_asr import enrich_author_home_asr
+from scripts.pipelines.xiaohongshu_creator_home_helpers import collect_and_adapt
+from scripts.writers.write_work_fact_card import build_work_fact_card, persist_output_envelope, write_work_fact_card
+
+
+def _write_collection_artifacts(
+    *,
+    profile: Dict[str, Any],
+    works: List[Dict[str, Any]],
+    card_root: str,
+    extract_trace: List[Dict[str, Any]],
+    request_id: str,
+) -> Dict[str, str]:
+    author_dir_name = resolve_author_directory_name(
+        "xiaohongshu",
+        str(profile.get("author_handle") or ""),
+        str(profile.get("platform_author_id") or ""),
+    )
+    author_dir = Path(card_root) / "内容系统" / "作品库" / author_dir_name
+    author_dir.mkdir(parents=True, exist_ok=True)
+
+    creator_profile = dict(profile)
+    creator_profile["request_id"] = request_id
+    creator_profile["extract_trace"] = extract_trace
+
+    work_collection = {
+        "platform": "xiaohongshu",
+        "platform_author_id": profile.get("platform_author_id"),
+        "count": len(works),
+        "items": [
+            {
+                "platform_work_id": item.get("platform_work_id"),
+                "title": item.get("title"),
+                "published_date": item.get("published_date"),
+            }
+            for item in works
+            if isinstance(item, dict)
+        ],
+        "request_id": request_id,
+        "extract_trace": extract_trace,
+    }
+
+    profile_path = author_dir / "_creator_profile.json"
+    collection_path = author_dir / "_work_collection.json"
+    profile_path.write_text(json.dumps(creator_profile, ensure_ascii=False, indent=2), encoding="utf-8")
+    collection_path.write_text(json.dumps(work_collection, ensure_ascii=False, indent=2), encoding="utf-8")
+    return {"creator_profile_path": str(profile_path), "work_collection_path": str(collection_path)}
+
+
+def run_xiaohongshu_creator_home(
+    *,
+    input_value: str,
+    config: Dict[str, Any],
+    runtime: Dict[str, Any],
+    max_items: int,
+    write_card: bool,
+    persist_output: bool,
+) -> Dict[str, Any]:
+    progress = build_progress_reporter(
+        workflow="social-media-crawl",
+        platform="xiaohongshu",
+        content_kind="author_home",
+        input_value=input_value,
+        scope="workflow",
+    )
+    progress.started(stage="author_home.workflow", message="xiaohongshu author_home workflow started")
+
+    raw, profile, works, missing = collect_and_adapt(
+        input_value=input_value,
+        base_url=runtime["base_url"],
+        token=runtime["token"],
+        timeout_ms=runtime["timeout_ms"],
+        page_size=20,
+        pages_max=50,
+        max_items=max(1, int(max_items)),
+        progress=progress.child(scope="author_home.collect"),
+    )
+
+    asr_strategy = config_get(config, "asr_strategy", {})
+    asr_bundle = enrich_author_home_asr(
+        platform="xiaohongshu",
+        works=works,
+        base_url=runtime["base_url"],
+        token=runtime["token"],
+        timeout_ms=runtime["timeout_ms"],
+        poll_interval_sec=float(asr_strategy.get("poll_interval_sec", 3.0)),
+        max_polls=int(asr_strategy.get("max_polls", 30)),
+        xhs_submit_max_retries=int(config_get(config, "asr_strategy.submit_retry.xiaohongshu_note.max_retries", 0)),
+        xhs_submit_backoff_ms=int(config_get(config, "asr_strategy.submit_retry.xiaohongshu_note.backoff_ms", 0)),
+        timeout_retry_enabled=bool(config_get(config, "asr_strategy.u2_timeout_retry.enabled", True)),
+        timeout_retry_max_retries=int(config_get(config, "asr_strategy.u2_timeout_retry.max_retries", 0)),
+        progress=progress.child(scope="author_home.asr"),
+    )
+    works = list(asr_bundle.get("works") or [])
+
+    card_root = resolve_storage_paths(config)["card_root"]
+    card_results: List[Dict[str, Any]] = []
+    if write_card:
+        for work in works:
+            card_results.append(
+                write_work_fact_card(
+                    payload=work,
+                    platform="xiaohongshu",
+                    card_root=card_root,
+                    storage_config=config,
+                )
+            )
+
+    request_id = ensure_request_id(
+        raw.get("request_id") or profile.get("request_id"),
+        fallback_seed=input_value,
+    )
+    extract_trace = list(raw.get("extract_trace") or []) + list(asr_bundle.get("trace") or [])
+
+    collection_artifacts = _write_collection_artifacts(
+        profile=profile,
+        works=works,
+        card_root=card_root,
+        extract_trace=extract_trace,
+        request_id=request_id,
+    )
+
+    normalized_profile = dict(profile)
+    normalized_profile["request_id"] = request_id
+    normalized_profile["extract_trace"] = extract_trace
+    normalized_works = [build_work_fact_card(work, platform="xiaohongshu") for work in works]
+
+    envelope = {
+        "object_type": "creator",
+        "platform": "xiaohongshu",
+        "input": input_value,
+        "normalized": {
+            "creator_profile": normalized_profile,
+            "work_collection": {
+                "platform": "xiaohongshu",
+                "platform_author_id": profile.get("platform_author_id"),
+                "count": len(normalized_works),
+                "items": normalized_works,
+                "request_id": request_id,
+                "extract_trace": extract_trace,
+            },
+        },
+        "completeness": evaluate_collection(profile, normalized_works),
+        "missing_fields": normalize_missing_fields(missing),
+        "error_reason": None,
+        "extract_trace": extract_trace,
+        "request_id": request_id,
+        "card_write": {
+            "enabled": bool(write_card),
+            "ok": all(item.get("ok") for item in card_results) if card_results else False,
+            "count": len(card_results),
+            "results": card_results,
+        },
+        "collection_artifacts": collection_artifacts,
+    }
+    envelope["output_persist"] = persist_output_envelope(
+        envelope=envelope,
+        storage_config=config,
+        platform="xiaohongshu",
+        fallback_identifier=str(profile.get("platform_author_id") or "author-home"),
+    ) if persist_output else {"enabled": False, "skipped": True, "reason": "disabled_by_flag"}
+
+    progress.done(
+        stage="author_home.workflow",
+        message="xiaohongshu author_home workflow finished",
+        data={"request_id": request_id, "works_count": len(normalized_works)},
+    )
+    return envelope
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Run Xiaohongshu creator-home fixed pipeline")
+    parser.add_argument("input", help="Xiaohongshu creator homepage or share URL")
+    parser.add_argument("--config", default=None, help="Runtime config YAML path")
+    parser.add_argument("--env-file", default=None, help="Explicit .env path")
+    parser.add_argument("--allow-process-env", action="store_true", help="Allow process env overrides")
+    parser.add_argument("--base-url", default=None, help="Override Tikomni base URL")
+    parser.add_argument("--timeout-ms", type=int, default=None, help="Override timeout in ms")
+    parser.add_argument("--max-items", type=int, default=5, help="Max works to collect from homepage")
+    parser.set_defaults(write_card=True, persist_output=True)
+    parser.add_argument("--write-card", dest="write_card", action="store_true", help="Write work fact cards")
+    parser.add_argument("--no-write-card", dest="write_card", action="store_false", help="Skip card writing")
+    parser.add_argument("--persist-output", dest="persist_output", action="store_true", help="Persist output JSON")
+    parser.add_argument("--no-persist-output", dest="persist_output", action="store_false", help="Skip output JSON persist")
+    args = parser.parse_args()
+
+    config, _ = load_tikomni_config(args.config, env_file=args.env_file, allow_process_env=args.allow_process_env)
+    runtime = resolve_runtime(
+        env_file=args.env_file,
+        api_key_env=str(config_get(config, "runtime.auth_env_key", "TIKOMNI_API_KEY")),
+        base_url=args.base_url or config_get(config, "runtime.base_url", None),
+        timeout_ms=args.timeout_ms or config_get(config, "runtime.timeout_ms", None),
+        allow_process_env=args.allow_process_env,
+    )
+    write_json_stdout(
+        run_xiaohongshu_creator_home(
+            input_value=args.input,
+            config=config,
+            runtime=runtime,
+            max_items=int(args.max_items),
+            write_card=bool(args.write_card),
+            persist_output=bool(args.persist_output),
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

package/skills/{single-work-analysis/scripts/platform/xiaohongshu/run_xiaohongshu_extract.py → social-media-crawl/scripts/pipelines/run_xiaohongshu_single_work.py}

@@ -27,10 +27,9 @@ from datetime import datetime
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 
-from scripts.pipeline.asr.asr_pipeline import derive_asr_clean_text, run_u2_asr_candidates_with_timeout_retry
-from scripts.core.config_loader import config_get, load_tikomni_config, resolve_storage_paths
+from scripts.core.asr_pipeline import derive_asr_clean_text, run_u2_asr_candidates_with_timeout_retry
+from scripts.core.config_loader import config_get, load_tikomni_config
 from scripts.core.progress_report import ProgressReporter, build_progress_reporter
-from scripts.core.storage_router import render_output_filename, resolve_json_filename_pattern
 from scripts.core.extract_pipeline import build_api_trace, resolve_trace_error_context
 from scripts.core.tikomni_common import (
     call_json_api,
@@ -41,7 +40,11 @@ from scripts.core.tikomni_common import (
     summarize_content,
     write_json_stdout,
 )
-from scripts.writers.write_benchmark_card import write_benchmark_card
+from scripts.writers.write_work_fact_card import (
+    build_work_output_envelope,
+    persist_output_envelope,
+    write_work_fact_card,
+)
 
 APP_V2_VIDEO_ENDPOINT = "/api/u1/v1/xiaohongshu/app_v2/get_video_note_detail"
 APP_V2_IMAGE_ENDPOINT = "/api/u1/v1/xiaohongshu/app_v2/get_image_note_detail"
@@ -167,106 +170,6 @@ def _traceable_identifier(source_input: Dict[str, Optional[str]], note_id: Optio
     return f"url-{digest}"
 
 
-def _build_persist_payload(
-    *,
-    result: Dict[str, Any],
-    source_input: Dict[str, Optional[str]],
-    note_id: Optional[str],
-    status: str,
-    written_at: datetime,
-) -> Dict[str, Any]:
-    summary = {
-        "summary": result.get("summary", ""),
-        "insights": result.get("insights", []),
-        "confidence": result.get("confidence"),
-        "error_reason": result.get("error_reason"),
-    }
-    normalized = {
-        "platform": "xiaohongshu",
-        "content_kind": result.get("content_kind", "note"),
-        "note_id": result.get("note_id") or note_id,
-        "note_content_type": result.get("note_content_type"),
-        "text_source": result.get("text_source"),
-        "request_id": result.get("request_id"),
-        "source": source_input,
-    }
-    return {
-        "meta": {
-            "written_at": written_at.isoformat(timespec="seconds"),
-            "status": status,
-            "platform": "xiaohongshu",
-            "identifier": _traceable_identifier(source_input, note_id),
-        },
-        "summary": summary,
-        "normalized": normalized,
-        "raw": result,
-    }
-
-
-def _persist_output_artifact(
-    *,
-    result: Dict[str, Any],
-    source_input: Dict[str, Optional[str]],
-    note_id: Optional[str],
-    storage_config: Optional[Dict[str, Any]],
-    persist_output: bool,
-) -> Dict[str, Any]:
-    if not persist_output:
-        return {"enabled": False, "skipped": True, "reason": "disabled_by_flag"}
-
-    try:
-        paths = resolve_storage_paths(storage_config or {})
-    except Exception as error:
-        return {"enabled": True, "ok": False, "error": f"resolve_storage_paths_failed:{error}"}
-
-    now = datetime.now()
-    date_key = now.strftime("%Y%m%d")
-    timestamp = now.strftime("%Y%m%dT%H%M%S")
-    identifier = _traceable_identifier(source_input, note_id)
-    has_error = bool(result.get("error_reason"))
-    status = "error" if has_error else "success"
-
-    if has_error:
-        target_dir = Path(paths.get("errors_root", "")) / date_key
-    else:
-        target_dir = Path(paths.get("results_root", "")) / date_key
-
-    target_dir.mkdir(parents=True, exist_ok=True)
-    file_name = render_output_filename(
-        pattern=resolve_json_filename_pattern(storage_config),
-        context={
-            "prefix": status,
-            "platform": "xiaohongshu",
-            "card_type": "single_work_result",
-            "author_slug": identifier,
-            "title_slug": identifier,
-            "identifier": identifier,
-            "timestamp": timestamp,
-            "date": date_key,
-            "ext": ".json",
-        },
-        default_filename=f"{timestamp}-xiaohongshu-{identifier}.json",
-        default_ext=".json",
-    )
-    file_path = target_dir / file_name
-
-    payload = _build_persist_payload(
-        result=result,
-        source_input=source_input,
-        note_id=note_id,
-        status=status,
-        written_at=now,
-    )
-    file_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
-
-    return {
-        "enabled": True,
-        "ok": True,
-        "status": status,
-        "path": str(file_path),
-    }
-
-
 def _finalize_result(
     *,
     result: Dict[str, Any],
@@ -275,14 +178,19 @@ def _finalize_result(
     storage_config: Optional[Dict[str, Any]],
     persist_output: bool,
 ) -> Dict[str, Any]:
-    result["output_persist"] = _persist_output_artifact(
-        result=result,
-        source_input=source_input,
-        note_id=note_id,
+    envelope = build_work_output_envelope(result, platform="xiaohongshu")
+    if "card_write" in result:
+        envelope["card_write"] = result.get("card_write")
+    if not persist_output:
+        envelope["output_persist"] = {"enabled": False, "skipped": True, "reason": "disabled_by_flag"}
+        return envelope
+    envelope["output_persist"] = persist_output_envelope(
+        envelope=envelope,
         storage_config=storage_config,
-        persist_output=persist_output,
+        platform="xiaohongshu",
+        fallback_identifier=note_id or _traceable_identifier(source_input, note_id),
     )
-    return result
+    return envelope
 
 
 def _normalize_input(input_value: Optional[str], share_text: Optional[str], note_id: Optional[str]) -> Dict[str, Optional[str]]:
@@ -1586,7 +1494,7 @@ def run_xiaohongshu_extract(
         )
         if write_card:
             card_started_at = time.perf_counter()
-            result["card_write"] = write_benchmark_card(
+            result["card_write"] = write_work_fact_card(
                 payload=result,
                 platform="xiaohongshu",
                 card_type=card_type,
@@ -1694,7 +1602,7 @@ def run_xiaohongshu_extract(
         )
         if write_card:
             card_started_at = time.perf_counter()
-            result["card_write"] = write_benchmark_card(
+            result["card_write"] = write_work_fact_card(
                 payload=result,
                 platform="xiaohongshu",
                 card_type=card_type,
@@ -1868,7 +1776,7 @@ def run_xiaohongshu_extract(
             )
             if write_card:
                 card_started_at = time.perf_counter()
-                result["card_write"] = write_benchmark_card(
+                result["card_write"] = write_work_fact_card(
                     payload=result,
                     platform="xiaohongshu",
                     card_type=card_type,
@@ -1933,7 +1841,7 @@ def run_xiaohongshu_extract(
             )
             if write_card:
                 card_started_at = time.perf_counter()
-                result["card_write"] = write_benchmark_card(
+                result["card_write"] = write_work_fact_card(
                     payload=result,
                     platform="xiaohongshu",
                     card_type=card_type,
@@ -1976,6 +1884,7 @@ def run_xiaohongshu_extract(
             max_polls=max_polls,
             timeout_retry_enabled=u2_timeout_retry_enabled,
             timeout_retry_max_retries=u2_timeout_retry_max_retries,
+            pending_timeout_sec=int(config_get(storage_config or {}, "runtime.u2_pending_timeout_sec", 60) or 60),
             progress_callback=(
                 lambda event: _report_u2_progress(progress, stage="note.u2", event=event, label="xiaohongshu")
             ) if progress is not None else None,
@@ -2015,6 +1924,7 @@ def run_xiaohongshu_extract(
                     "u2_submit_backoff_ms": max(0, int(u2_submit_backoff_ms)),
                 },
                 "timeout_retry": u2_bundle.get("timeout_retry", {}),
+                "u3_fallback": u2_bundle.get("u3_fallback", {}),
                 "rounds": u2_bundle.get("rounds", []),
                 "final_task_id": poll_result.get("task_id") or task_id,
                 "final_task_status": poll_result.get("task_status"),
@@ -2075,7 +1985,7 @@ def run_xiaohongshu_extract(
             )
             if write_card:
                 card_started_at = time.perf_counter()
-                result["card_write"] = write_benchmark_card(
+                result["card_write"] = write_work_fact_card(
                     payload=result,
                     platform="xiaohongshu",
                     card_type=card_type,
@@ -2140,7 +2050,7 @@ def run_xiaohongshu_extract(
 
         if write_card:
             card_started_at = time.perf_counter()
-            result["card_write"] = write_benchmark_card(
+            result["card_write"] = write_work_fact_card(
                 payload=result,
                 platform="xiaohongshu",
                 card_type=card_type,
@@ -2214,7 +2124,7 @@ def run_xiaohongshu_extract(
 
     if write_card:
         card_started_at = time.perf_counter()
-        result["card_write"] = write_benchmark_card(
+        result["card_write"] = write_work_fact_card(
             payload=result,
             platform="xiaohongshu",
             card_type=card_type,
@@ -2294,7 +2204,8 @@ def main() -> None:
     )
     parser.add_argument("--force-u2-fallback", action="store_true", help="Skip subtitle usage and force U2 fallback (test)")
     parser.add_argument("--card-type", choices=["work"], default="work", help="Primary card type")
-    parser.add_argument("--analysis-mode", choices=["auto", "local"], default="auto", help="Card analysis mode")
+    parser.add_argument("--card-mode", dest="analysis_mode", choices=["standard", "auto", "local"], default="standard", help="Fact-card mode")
+    parser.add_argument("--analysis-mode", dest="analysis_mode", choices=["standard", "auto", "local"], help=argparse.SUPPRESS)
     parser.set_defaults(write_card=True, persist_output=True)
     parser.add_argument("--write-card", dest="write_card", action="store_true", help="Write final work card")
     parser.add_argument("--no-write-card", dest="write_card", action="store_false", help="Skip card writing")
@@ -2340,7 +2251,7 @@ def main() -> None:
         else config_get(config, "asr_strategy.u2_timeout_retry.max_retries", 3)
     )
     progress = build_progress_reporter(
-        workflow="single-work-analysis",
+        workflow="social-media-crawl",
         platform="xiaohongshu",
         content_kind="note",
         input_value=args.share_text or args.note_id or args.input,

package/skills/{creator-analysis/scripts/author_home → social-media-crawl/scripts/pipelines}/schema.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Schema definitions and minimal validators for creator-analysis."""
+"""Schema definitions and minimal validators for social-media-crawl."""
 
 from __future__ import annotations
 

package/skills/social-media-crawl/scripts/pipelines/xiaohongshu_creator_home_helpers.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Xiaohongshu creator-home helper entrypoints."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Tuple
+
+from scripts.pipelines.homepage_collectors import collect_xhs_author_home_raw
+from scripts.pipelines.platform_adapters import adapt_xhs_author_home
+
+
+def collect_and_adapt(
+    *,
+    input_value: str,
+    base_url: str,
+    token: str,
+    timeout_ms: int,
+    page_size: int,
+    pages_max: int,
+    max_items: int,
+    progress: Any = None,
+) -> Tuple[Dict[str, Any], Dict[str, Any], List[Dict[str, Any]], List[Dict[str, str]]]:
+    raw = collect_xhs_author_home_raw(
+        input_value=input_value,
+        base_url=base_url,
+        token=token,
+        timeout_ms=timeout_ms,
+        page_size=page_size,
+        pages_max=pages_max,
+        max_items=max_items,
+        progress=progress,
+    )
+    profile, works, missing = adapt_xhs_author_home(raw)
+    return raw, profile, works, missing
+

package/skills/social-media-crawl/scripts/pipelines/xiaohongshu_platform_adapter.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+"""Compatibility wrapper for xiaohongshu platform adapter."""
+
+from scripts.pipelines.platform_adapters import adapt_xhs_author_home
+
+__all__ = ["adapt_xhs_author_home"]
+

package/skills/social-media-crawl/scripts/writers/__init__.py

@@ -0,0 +1 @@
+"""Writers for social-media-crawl."""