@tikomni/skills 0.1.2 → 0.1.4

package/skills/{creator-analysis/scripts/pipeline/asr → social-media-crawl/scripts/core}/asr_pipeline.py

@@ -2,11 +2,13 @@
 """Shared ASR pipeline helpers for runner scripts."""
 
 import json
+import re
+import threading
 import time
 import urllib.error
 import urllib.request
 from urllib.parse import urlparse, urlunparse
-from typing import Any, Dict, List, Optional
+from typing import Any, Callable, Dict, List, Optional
 
 from scripts.core.tikomni_common import (
     call_json_api,
@@ -17,8 +19,10 @@ from scripts.core.tikomni_common import (
     is_terminal_status,
     normalize_text,
 )
+from scripts.core.u3_fallback import run_u3_public_url_fallback
 
 U2_BATCH_SUBMIT_HARD_LIMIT = 100
+DEFAULT_U2_PENDING_TIMEOUT_SEC = 60
 
 
 def clamp_u2_batch_submit_size(size: int, *, default: int = 50, hard_limit: int = U2_BATCH_SUBMIT_HARD_LIMIT) -> int:
@@ -83,6 +87,7 @@ def submit_u2_asr_batch_with_retry(
     file_urls: List[str],
     max_retries: int,
     backoff_ms: int,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     retries = max(0, int(max_retries))
     base_backoff = max(0, int(backoff_ms))
@@ -110,20 +115,75 @@ def submit_u2_asr_batch_with_retry(
     final_task_id: Optional[str] = None
     final_submit_status = "failed_unknown"
 
+    def _emit_submit_progress(event: Dict[str, Any]) -> None:
+        if progress_callback is None:
+            return
+        try:
+            progress_callback(event)
+        except Exception:
+            pass
+
     for attempt in range(1, max_attempts + 1):
         wait_ms = 0 if attempt == 1 else base_backoff * (2 ** (attempt - 2))
         if wait_ms > 0:
             time.sleep(wait_ms / 1000.0)
 
-        submit_response = submit_u2_asr_batch(
-            base_url=base_url,
-            token=token,
-            timeout_ms=timeout_ms,
-            file_urls=limited_urls,
+        _emit_submit_progress(
+            {
+                "phase": "submit",
+                "state": "started",
+                "attempt": attempt,
+                "wait_ms": wait_ms,
+                "candidate_count": len(limited_urls),
+            }
         )
+
+        heartbeat_stop = threading.Event()
+
+        def _heartbeat() -> None:
+            while not heartbeat_stop.wait(5.0):
+                _emit_submit_progress(
+                    {
+                        "phase": "submit",
+                        "state": "heartbeat",
+                        "attempt": attempt,
+                        "wait_ms": wait_ms,
+                        "candidate_count": len(limited_urls),
+                    }
+                )
+
+        heartbeat_thread = threading.Thread(target=_heartbeat, daemon=True)
+        heartbeat_thread.start()
+
+        try:
+            submit_response = submit_u2_asr_batch(
+                base_url=base_url,
+                token=token,
+                timeout_ms=timeout_ms,
+                file_urls=limited_urls,
+            )
+        finally:
+            heartbeat_stop.set()
+            heartbeat_thread.join(timeout=0.2)
         task_id = extract_task_id(submit_response.get("data"))
         retriable = is_retriable_submit_failure(submit_response)
 
+        _emit_submit_progress(
+            {
+                "phase": "submit",
+                "state": "finished",
+                "attempt": attempt,
+                "wait_ms": wait_ms,
+                "candidate_count": len(limited_urls),
+                "task_id": task_id,
+                "status_code": submit_response.get("status_code"),
+                "ok": bool(submit_response.get("ok")),
+                "error_reason": submit_response.get("error_reason"),
+                "request_id": submit_response.get("request_id"),
+                "retriable": retriable,
+            }
+        )
+
         retry_chain.append(
             {
                 "attempt": attempt,
@@ -172,6 +232,7 @@ def submit_u2_asr_with_retry(
     video_url: str,
     max_retries: int,
     backoff_ms: int,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     return submit_u2_asr_batch_with_retry(
         base_url=base_url,
@@ -180,13 +241,55 @@ def submit_u2_asr_with_retry(
         file_urls=[video_url],
         max_retries=max_retries,
         backoff_ms=backoff_ms,
+        progress_callback=progress_callback,
     )
 
 
 def clean_transcript_text(raw_text: Any) -> str:
     if raw_text is None:
         return ""
-    return str(raw_text).strip()
+    return normalize_text(raw_text)
+
+
+def _ensure_sentence_end(text: str) -> str:
+    if not text:
+        return text
+    if text[-1] in "。！？!?" or text.endswith("..."):
+        return text
+    return f"{text}。"
+
+
+def derive_asr_clean_text(asr_raw: Any, legacy_clean: Any = None) -> str:
+    base = clean_transcript_text(asr_raw) or clean_transcript_text(legacy_clean)
+    if not base:
+        return ""
+
+    denoised = re.sub(r"\b(嗯|啊|呃|额|那个|这个|然后|就是)\b", " ", base)
+    denoised = re.sub(r"(嗯+|啊+|呃+)", " ", denoised)
+    denoised = re.sub(r"(就是就是|然后然后|这个这个|那个那个)", " ", denoised)
+    denoised = re.sub(r"\s+", " ", denoised).strip()
+
+    units = [clean_transcript_text(part) for part in re.split(r"[。！？!?；;\n]+", denoised)]
+    sentences = [_ensure_sentence_end(unit) for unit in units if unit]
+    if not sentences:
+        fallback = _ensure_sentence_end(denoised)
+        return fallback if fallback else ""
+
+    paragraphs: List[str] = []
+    bucket: List[str] = []
+    for sentence in sentences:
+        bucket.append(sentence)
+        if len(bucket) >= 3:
+            paragraphs.append("\n".join(bucket))
+            bucket = []
+
+    if bucket:
+        if len(bucket) == 1 and paragraphs:
+            paragraphs[-1] = f"{paragraphs[-1]}\n{bucket[0]}"
+        else:
+            paragraphs.append("\n".join(bucket))
+
+    return "\n\n".join(paragraphs)
 
 
 def extract_u2_task_metrics(payload: Any) -> Dict[str, Any]:
@@ -612,9 +715,12 @@ def poll_u2_task_core(
     max_polls: int,
     require_batch_complete: bool = False,
     expected_total: int = 0,
+    pending_timeout_sec: Optional[int] = None,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     trace = []
     last_request_id = None
+    started_at = time.perf_counter()
 
     last_status = "UNKNOWN"
     last_payload: Any = {}
@@ -656,6 +762,7 @@ def poll_u2_task_core(
         trace.append(
             {
                 "attempt": attempt,
+                "elapsed_ms": int((time.perf_counter() - started_at) * 1000),
                 "status_code": response.get("status_code"),
                 "task_status": status,
                 "platform_task_status": platform_status,
@@ -666,6 +773,26 @@ def poll_u2_task_core(
             }
         )
 
+        if progress_callback is not None:
+            try:
+                progress_callback(
+                    {
+                        "attempt": attempt,
+                        "elapsed_ms": int((time.perf_counter() - started_at) * 1000),
+                        "task_id": task_id,
+                        "task_status": status or "UNKNOWN",
+                        "platform_task_status": platform_status or "UNKNOWN",
+                        "pending_count": pending_count,
+                        "request_id": response.get("request_id"),
+                        "status_code": response.get("status_code"),
+                        "ok": bool(response.get("ok")),
+                        "error_reason": response.get("error_reason"),
+                        "batch_progress": batch_progress,
+                    }
+                )
+            except Exception:
+                pass
+
         if not response.get("ok"):
             if attempt < max_polls:
                 time.sleep(max(poll_interval_sec, 0.2))
@@ -689,6 +816,26 @@ def poll_u2_task_core(
         task_complete = status_terminal or platform_terminal
         batch_complete = bool(batch_progress.get("complete")) if require_batch_complete else task_complete
 
+        elapsed_sec = time.perf_counter() - started_at
+        if pending_timeout_sec and elapsed_sec >= max(float(pending_timeout_sec), 1.0):
+            timeout_reason = "u2_batch_pending_timeout" if require_batch_complete else "u2_pending_timeout"
+            return {
+                "ok": False,
+                "task_id": task_id,
+                "task_status": effective_status or last_status or "PENDING",
+                "platform_task_status": platform_status,
+                "pending_count": pending_count,
+                "request_id": last_request_id,
+                "error_reason": timeout_reason,
+                "transcript_text": "",
+                "raw_task": payload,
+                "task_metrics": last_metrics,
+                "batch_results": batch_results,
+                "batch_progress": batch_progress,
+                "batch_complete": bool(batch_complete),
+                "trace": trace,
+            }
+
         if require_batch_complete and not batch_complete:
             if attempt < max_polls:
                 time.sleep(max(poll_interval_sec, 0.2))
@@ -797,6 +944,8 @@ def run_u2_asr_candidates_with_timeout_retry(
     max_polls: int,
     timeout_retry_enabled: bool = True,
     timeout_retry_max_retries: int = 3,
+    pending_timeout_sec: int = DEFAULT_U2_PENDING_TIMEOUT_SEC,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     normalized_candidates = normalize_media_candidates(candidates)
     attempts: List[Dict[str, Any]] = []
@@ -831,6 +980,8 @@ def run_u2_asr_candidates_with_timeout_retry(
             max_polls=max_polls,
             timeout_retry_enabled=timeout_retry_enabled,
             timeout_retry_max_retries=timeout_retry_max_retries,
+            pending_timeout_sec=pending_timeout_sec,
+            progress_callback=progress_callback,
         )
         poll_result = bundle.get("poll_result", {})
         error_reason = str(poll_result.get("error_reason") or "")
@@ -843,6 +994,7 @@ def run_u2_asr_candidates_with_timeout_retry(
             "ok": ok,
             "error_reason": error_reason,
             "task_status": poll_result.get("task_status"),
+            "u3_fallback": bundle.get("u3_fallback", {}),
         })
 
         final_bundle = bundle
@@ -850,6 +1002,9 @@ def run_u2_asr_candidates_with_timeout_retry(
         if ok:
             break
 
+        if (bundle.get("u3_fallback") or {}).get("triggered"):
+            break
+
         if error_reason == "INVALID_SOURCE_URL":
             continue
 
@@ -871,6 +1026,8 @@ def run_u2_asr_batch_with_timeout_retry(
     max_polls: int,
     timeout_retry_enabled: bool = True,
     timeout_retry_max_retries: int = 3,
+    pending_timeout_sec: int = DEFAULT_U2_PENDING_TIMEOUT_SEC,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     normalized_urls = normalize_media_candidates(file_urls)
     limited_urls = normalized_urls[:U2_BATCH_SUBMIT_HARD_LIMIT]
@@ -921,6 +1078,7 @@ def run_u2_asr_batch_with_timeout_retry(
         }
 
     for round_index in range(1, max_rounds + 1):
+        submit_started_at = time.perf_counter()
         submit_bundle = submit_u2_asr_batch_with_retry(
             base_url=base_url,
             token=token,
@@ -928,12 +1086,16 @@ def run_u2_asr_batch_with_timeout_retry(
             file_urls=limited_urls,
             max_retries=submit_max_retries,
             backoff_ms=submit_backoff_ms,
+            progress_callback=progress_callback,
         )
+        submit_duration_ms = int((time.perf_counter() - submit_started_at) * 1000)
         submit_response = submit_bundle.get("submit_response", {})
         task_id = submit_bundle.get("task_id")
 
         poll_result: Dict[str, Any]
+        poll_duration_ms = 0
         if submit_response.get("ok") and task_id:
+            poll_started_at = time.perf_counter()
             poll_result = poll_u2_task_core(
                 base_url=base_url,
                 token=token,
@@ -943,7 +1105,10 @@ def run_u2_asr_batch_with_timeout_retry(
                 max_polls=max_polls,
                 require_batch_complete=True,
                 expected_total=len(limited_urls),
+                pending_timeout_sec=pending_timeout_sec,
+                progress_callback=progress_callback,
             )
+            poll_duration_ms = int((time.perf_counter() - poll_started_at) * 1000)
         else:
             poll_result = {
                 "ok": False,
@@ -975,6 +1140,7 @@ def run_u2_asr_batch_with_timeout_retry(
                     "error_reason": submit_response.get("error_reason"),
                     "retry_chain": submit_bundle.get("retry_chain", []),
                     "file_url_count": len(limited_urls),
+                    "duration_ms": submit_duration_ms,
                 },
                 "poll": {
                     "task_id": poll_result.get("task_id") or task_id,
@@ -986,6 +1152,7 @@ def run_u2_asr_batch_with_timeout_retry(
                     "task_metrics": poll_result.get("task_metrics", {}),
                     "batch_complete": bool(poll_result.get("batch_complete")),
                     "batch_progress": poll_result.get("batch_progress", {}),
+                    "duration_ms": poll_duration_ms,
                 },
             }
         )
@@ -993,7 +1160,7 @@ def run_u2_asr_batch_with_timeout_retry(
         final_submit_bundle = submit_bundle
         final_poll_result = poll_result
 
-        if poll_result.get("error_reason") in {"u2_poll_timeout", "u2_batch_incomplete_timeout"} and round_index < max_rounds:
+        if poll_result.get("error_reason") in {"u2_poll_timeout", "u2_batch_incomplete_timeout", "u2_batch_pending_timeout"} and round_index < max_rounds:
             timeout_retry_triggered = True
             timeout_retry_result = "retrying"
             continue
@@ -1002,7 +1169,7 @@ def run_u2_asr_batch_with_timeout_retry(
 
     if final_poll_result.get("ok"):
         timeout_retry_result = "retry_succeeded" if timeout_retry_triggered else "not_needed"
-    elif final_poll_result.get("error_reason") in {"u2_poll_timeout", "u2_batch_incomplete_timeout"}:
+    elif final_poll_result.get("error_reason") in {"u2_poll_timeout", "u2_batch_incomplete_timeout", "u2_batch_pending_timeout"}:
         timeout_retry_result = "retry_timeout_exhausted" if timeout_retry_triggered else "timeout_no_retry"
     elif timeout_retry_triggered:
         timeout_retry_result = "retry_failed_non_timeout"
@@ -1071,6 +1238,8 @@ def run_u2_asr_batch_with_timeout_retry(
         "task_metrics": final_poll_result.get("task_metrics") if isinstance(final_poll_result.get("task_metrics"), dict) else extract_u2_task_metrics(raw_task_payload),
         "batch_progress": final_poll_result.get("batch_progress") if isinstance(final_poll_result.get("batch_progress"), dict) else build_u2_batch_progress(payload=raw_task_payload, expected_total=len(limited_urls)),
         "batch_complete": bool(final_poll_result.get("batch_complete")),
+        "submit_duration_ms": _safe_int((rounds[-1].get("submit") if rounds else {}).get("duration_ms")),
+        "poll_duration_ms": _safe_int((rounds[-1].get("poll") if rounds else {}).get("duration_ms")),
     }
 
 
@@ -1086,6 +1255,9 @@ def run_u2_asr_with_timeout_retry(
     max_polls: int,
     timeout_retry_enabled: bool = True,
     timeout_retry_max_retries: int = 3,
+    pending_timeout_sec: int = DEFAULT_U2_PENDING_TIMEOUT_SEC,
+    u3_fallback_enabled: bool = True,
+    progress_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
 ) -> Dict[str, Any]:
     video_url = normalize_media_url(video_url)
     conservative_retries = max(0, min(3, int(timeout_retry_max_retries)))
@@ -1101,8 +1273,17 @@ def run_u2_asr_with_timeout_retry(
     }
     timeout_retry_triggered = False
     timeout_retry_result = "not_triggered"
+    u3_fallback_bundle: Dict[str, Any] = {
+        "enabled": bool(u3_fallback_enabled),
+        "triggered": False,
+        "ok": False,
+        "result": "not_triggered",
+        "public_url": "",
+        "trace": [],
+    }
 
     for round_index in range(1, max_rounds + 1):
+        submit_started_at = time.perf_counter()
         submit_bundle = submit_u2_asr_with_retry(
             base_url=base_url,
             token=token,
@@ -1110,12 +1291,16 @@ def run_u2_asr_with_timeout_retry(
             video_url=video_url,
             max_retries=submit_max_retries,
             backoff_ms=submit_backoff_ms,
+            progress_callback=progress_callback,
         )
+        submit_duration_ms = int((time.perf_counter() - submit_started_at) * 1000)
         submit_response = submit_bundle.get("submit_response", {})
         task_id = submit_bundle.get("task_id")
 
         poll_result: Dict[str, Any]
+        poll_duration_ms = 0
         if submit_response.get("ok") and task_id:
+            poll_started_at = time.perf_counter()
             poll_result = poll_u2_task_core(
                 base_url=base_url,
                 token=token,
@@ -1123,7 +1308,10 @@ def run_u2_asr_with_timeout_retry(
                 task_id=str(task_id),
                 poll_interval_sec=poll_interval_sec,
                 max_polls=max_polls,
+                pending_timeout_sec=pending_timeout_sec,
+                progress_callback=progress_callback,
             )
+            poll_duration_ms = int((time.perf_counter() - poll_started_at) * 1000)
         else:
             poll_result = {
                 "ok": False,
@@ -1145,6 +1333,7 @@ def run_u2_asr_with_timeout_retry(
                     "ok": submit_response.get("ok"),
                     "error_reason": submit_response.get("error_reason"),
                     "retry_chain": submit_bundle.get("retry_chain", []),
+                    "duration_ms": submit_duration_ms,
                 },
                 "poll": {
                     "task_id": poll_result.get("task_id") or task_id,
@@ -1153,6 +1342,7 @@ def run_u2_asr_with_timeout_retry(
                     "ok": poll_result.get("ok"),
                     "error_reason": poll_result.get("error_reason"),
                     "attempts": len(poll_result.get("trace", [])),
+                    "duration_ms": poll_duration_ms,
                 },
             }
         )
@@ -1160,6 +1350,56 @@ def run_u2_asr_with_timeout_retry(
         final_submit_bundle = submit_bundle
         final_poll_result = poll_result
 
+        if (
+            u3_fallback_enabled
+            and poll_result.get("error_reason") in {"u2_pending_timeout", "u2_poll_timeout"}
+        ):
+            u3_fallback_bundle["triggered"] = True
+            u3_result = run_u3_public_url_fallback(
+                base_url=base_url,
+                token=token,
+                timeout_ms=timeout_ms,
+                source_url=video_url,
+            )
+            u3_fallback_bundle.update(
+                {
+                    "ok": bool(u3_result.get("ok")),
+                    "result": "u3_completed" if u3_result.get("ok") else "u3_failed",
+                    "public_url": u3_result.get("public_url") or "",
+                    "request_id": u3_result.get("request_id"),
+                    "error_reason": u3_result.get("error_reason"),
+                    "trace": u3_result.get("trace", []),
+                }
+            )
+            if u3_result.get("ok") and u3_result.get("public_url"):
+                retry_bundle = run_u2_asr_with_timeout_retry(
+                    base_url=base_url,
+                    token=token,
+                    timeout_ms=timeout_ms,
+                    video_url=str(u3_result.get("public_url")),
+                    submit_max_retries=submit_max_retries,
+                    submit_backoff_ms=submit_backoff_ms,
+                    poll_interval_sec=poll_interval_sec,
+                    max_polls=max_polls,
+                    timeout_retry_enabled=timeout_retry_enabled,
+                    timeout_retry_max_retries=timeout_retry_max_retries,
+                    pending_timeout_sec=pending_timeout_sec,
+                    u3_fallback_enabled=False,
+                    progress_callback=progress_callback,
+                )
+                retry_bundle["u3_fallback"] = u3_fallback_bundle
+                retry_bundle["original_source_url"] = video_url
+                return retry_bundle
+            final_poll_result = {
+                "ok": False,
+                "task_id": poll_result.get("task_id") or task_id,
+                "task_status": poll_result.get("task_status") or "UNKNOWN",
+                "request_id": u3_result.get("request_id") or poll_result.get("request_id"),
+                "error_reason": u3_result.get("error_reason") or "u3_fallback_failed",
+                "trace": list(poll_result.get("trace", [])) + list(u3_result.get("trace", [])),
+            }
+            break
+
         if poll_result.get("error_reason") == "u2_poll_timeout" and round_index < max_rounds:
             timeout_retry_triggered = True
             timeout_retry_result = "retrying"
@@ -1186,4 +1426,7 @@ def run_u2_asr_with_timeout_retry(
             "triggered": timeout_retry_triggered,
             "result": timeout_retry_result,
         },
+        "u3_fallback": u3_fallback_bundle,
+        "submit_duration_ms": _safe_int((rounds[-1].get("submit") if rounds else {}).get("duration_ms")),
+        "poll_duration_ms": _safe_int((rounds[-1].get("poll") if rounds else {}).get("duration_ms")),
     }

package/skills/social-media-crawl/scripts/core/completeness.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python3
+"""Completeness evaluation helpers."""
+
+from __future__ import annotations
+
+import hashlib
+import time
+from typing import Any, Dict, List, Optional
+
+
+def normalize_missing_fields(items: Any) -> List[Dict[str, str]]:
+    normalized: List[Dict[str, str]] = []
+    seen = set()
+    if not isinstance(items, list):
+        return normalized
+    for item in items:
+        if isinstance(item, dict):
+            field = str(item.get("field") or "").strip()
+            reason = str(item.get("reason") or "").strip() or "missing"
+        else:
+            field = str(item or "").strip()
+            reason = "missing"
+        if not field:
+            continue
+        key = (field, reason)
+        if key in seen:
+            continue
+        seen.add(key)
+        normalized.append({"field": field, "reason": reason})
+    return normalized
+
+
+def ensure_request_id(value: Optional[str], fallback_seed: str = "") -> str:
+    text = str(value or "").strip()
+    if text:
+        return text
+    digest = hashlib.sha1(fallback_seed.encode("utf-8")).hexdigest()[:8] if fallback_seed else "unknown"
+    return f"local-{int(time.time())}-{digest}"
+
+
+def evaluate_work_fact_card(card: Dict[str, Any]) -> Dict[str, Any]:
+    missing = normalize_missing_fields(card.get("missing_fields"))
+    existing_fields = {entry["field"] for entry in missing}
+
+    required_pairs = [
+        ("platform", bool(str(card.get("platform") or "").strip())),
+        ("platform_work_id", bool(str(card.get("platform_work_id") or "").strip())),
+        ("platform_author_id", bool(str(card.get("platform_author_id") or "").strip())),
+        ("author_handle", bool(str(card.get("author_handle") or "").strip())),
+        ("title", bool(str(card.get("title") or "").strip())),
+        ("source_url", bool(str(card.get("source_url") or "").strip())),
+        ("share_url", bool(str(card.get("share_url") or "").strip())),
+    ]
+    for field, ok in required_pairs:
+        if ok or field in existing_fields:
+            continue
+        missing.append({"field": field, "reason": "missing_required"})
+        existing_fields.add(field)
+
+    primary_text = str(card.get("primary_text") or "").strip()
+    if not primary_text and "primary_text" not in existing_fields:
+        missing.append({"field": "primary_text", "reason": "missing_primary_text"})
+
+    required_missing = [entry for entry in missing if entry["reason"] == "missing_required"]
+    if required_missing:
+        completeness = "incomplete"
+    elif primary_text:
+        completeness = "complete"
+    else:
+        completeness = "partial"
+
+    return {
+        "completeness": completeness,
+        "missing_fields": missing,
+    }
+
+
+def evaluate_collection(profile: Dict[str, Any], works: List[Dict[str, Any]]) -> str:
+    if not str(profile.get("platform_author_id") or "").strip():
+        return "incomplete"
+    if works:
+        return "complete"
+    return "partial"