@tikomni/skills 0.1.0

package/skills/single-work-analysis/scripts/core/extract_pipeline.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Shared extraction pipeline helpers for runner scripts."""
+
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlparse
+
+from scripts.core.tikomni_common import call_json_api
+
+
+def build_api_trace(
+    *,
+    step: str,
+    endpoint: Optional[str],
+    response: Dict[str, Any],
+    extra: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+    payload: Dict[str, Any] = {
+        "step": step,
+        "endpoint": endpoint,
+        "status_code": response.get("status_code"),
+        "request_id": response.get("request_id"),
+        "ok": response.get("ok"),
+        "error_reason": response.get("error_reason"),
+        "rate_limit_wait_ms": response.get("rate_limit_wait_ms", 0),
+        "retry_attempt": response.get("retry_attempt", 0),
+        "fallback_trigger_reason": response.get("fallback_trigger_reason"),
+    }
+    if extra:
+        payload.update(extra)
+    return payload
+
+
+def build_fallback_trace_from_extract_trace(extract_trace: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
+    """Build a compact fallback trace from runner extract trace."""
+    if not extract_trace:
+        return []
+
+    trace: List[Dict[str, Any]] = []
+    include_tokens = ("primary", "effective", "fallback", "gate", "retry")
+
+    for step in extract_trace:
+        if not isinstance(step, dict):
+            continue
+
+        step_name = str(step.get("step") or "")
+        if not step_name:
+            continue
+
+        has_fallback_signal = any(token in step_name for token in include_tokens)
+        if not has_fallback_signal:
+            continue
+
+        compact: Dict[str, Any] = {"step": step_name}
+        for key in (
+            "endpoint",
+            "ok",
+            "status_code",
+            "request_id",
+            "error_reason",
+            "gate_reason",
+            "final_submit_status",
+            "subtitle_hit",
+            "force_u2_fallback",
+            "rate_limit_wait_ms",
+            "retry_attempt",
+            "fallback_trigger_reason",
+        ):
+            if key in step:
+                compact[key] = step.get(key)
+        trace.append(compact)
+
+    return trace
+
+
+def resolve_trace_error_context(
+    *,
+    responses: Optional[List[Optional[Dict[str, Any]]]] = None,
+    extract_trace: Optional[List[Dict[str, Any]]] = None,
+    explicit_error_reason: Optional[str] = None,
+    default_error_reason: Optional[str] = None,
+    explicit_request_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Resolve consistent error/request/fallback fields for result envelopes."""
+    chain = [item for item in (responses or []) if isinstance(item, dict)]
+
+    request_id = explicit_request_id
+    if not request_id:
+        for response in chain:
+            candidate = response.get("request_id")
+            if candidate:
+                request_id = str(candidate)
+                break
+
+    error_reason = explicit_error_reason
+    if not error_reason:
+        for response in chain:
+            candidate = response.get("error_reason")
+            if candidate:
+                error_reason = str(candidate)
+                break
+    if not error_reason:
+        error_reason = default_error_reason
+
+    return {
+        "request_id": request_id,
+        "error_reason": error_reason,
+        "fallback_trace": build_fallback_trace_from_extract_trace(extract_trace),
+    }
+
+
+def _resolve_fallback_trigger_reason(primary: Dict[str, Any]) -> str:
+    if primary.get("timeout_retry_exhausted"):
+        return "primary_timeout_retry_exhausted"
+    if primary.get("error_reason"):
+        return "primary_non_timeout_failure"
+    return "primary_unknown_failure"
+
+
+def request_with_optional_fallback(
+    *,
+    base_url: str,
+    token: str,
+    timeout_ms: int,
+    method: str,
+    params: Optional[Dict[str, Any]],
+    primary_path: str,
+    fallback_path: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Call primary endpoint, then fallback endpoint if primary fails."""
+    primary = call_json_api(
+        base_url=base_url,
+        path=primary_path,
+        token=token,
+        method=method,
+        timeout_ms=timeout_ms,
+        params=params,
+    )
+    primary["_endpoint"] = primary_path
+    if primary.get("ok") or not fallback_path:
+        return primary
+
+    fallback_reason = _resolve_fallback_trigger_reason(primary)
+    fallback = call_json_api(
+        base_url=base_url,
+        path=fallback_path,
+        token=token,
+        method=method,
+        timeout_ms=timeout_ms,
+        params=params,
+    )
+    fallback["_endpoint"] = fallback_path
+    fallback["_primary_failed"] = primary
+    fallback["fallback_trigger_reason"] = fallback_reason
+    return fallback
+
+
+def detect_platform_from_input(input_value: str) -> str:
+    candidate = (input_value or "").strip().lower()
+    if not candidate:
+        return "unknown"
+
+    if candidate.startswith("http://") or candidate.startswith("https://"):
+        host = urlparse(candidate).netloc.lower()
+        if "douyin.com" in host or "iesdouyin.com" in host:
+            return "douyin"
+        if "xiaohongshu.com" in host or "xhslink.com" in host:
+            return "xiaohongshu"
+
+    if "douyin" in candidate:
+        return "douyin"
+    if "xiaohongshu" in candidate or "xhs" in candidate:
+        return "xiaohongshu"
+    return "unknown"

package/skills/single-work-analysis/scripts/core/progress_report.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Structured stderr progress reporting for long-running TikOmni workflows.
+
+Design goals:
+- emit machine-readable progress events to stderr only
+- keep final JSON stdout contract untouched
+- offer a tiny shared helper so handlers do not duplicate logging logic
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from datetime import datetime, timezone
+from typing import Any, Dict, Optional
+
+
+VALID_EVENTS = {"started", "progress", "done", "failed"}
+
+
+class ProgressReporter:
+    def __init__(
+        self,
+        *,
+        workflow: str,
+        platform: str,
+        content_kind: str,
+        run_id: Optional[str] = None,
+        scope: str = "workflow",
+        enabled: bool = True,
+        defaults: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        self.workflow = str(workflow or "unknown")
+        self.platform = str(platform or "unknown")
+        self.content_kind = str(content_kind or "unknown")
+        self.run_id = str(run_id or f"{self.platform}.{self.content_kind}")
+        self.scope = str(scope or "workflow")
+        self.enabled = bool(enabled)
+        self.defaults = dict(defaults or {})
+
+    def child(self, *, scope: str, defaults: Optional[Dict[str, Any]] = None) -> "ProgressReporter":
+        merged = dict(self.defaults)
+        if defaults:
+            merged.update(defaults)
+        return ProgressReporter(
+            workflow=self.workflow,
+            platform=self.platform,
+            content_kind=self.content_kind,
+            run_id=self.run_id,
+            scope=scope,
+            enabled=self.enabled,
+            defaults=merged,
+        )
+
+    def emit(self, event: str, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
+        if not self.enabled:
+            return
+        normalized_event = str(event or "progress").strip().lower()
+        if normalized_event not in VALID_EVENTS:
+            normalized_event = "progress"
+        payload: Dict[str, Any] = {
+            "channel": "tikomni_progress",
+            "ts": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+            "workflow": self.workflow,
+            "platform": self.platform,
+            "content_kind": self.content_kind,
+            "run_id": self.run_id,
+            "scope": self.scope,
+            "event": normalized_event,
+            "stage": str(stage or "unknown"),
+        }
+        if message:
+            payload["message"] = str(message)
+        if self.defaults:
+            payload["data"] = dict(self.defaults)
+        if isinstance(data, dict) and data:
+            merged = dict(payload.get("data") or {})
+            merged.update(data)
+            payload["data"] = merged
+        sys.stderr.write(json.dumps(payload, ensure_ascii=False) + "\n")
+        sys.stderr.flush()
+
+    def started(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
+        self.emit("started", stage=stage, message=message, data=data)
+
+    def progress(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
+        self.emit("progress", stage=stage, message=message, data=data)
+
+    def done(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
+        self.emit("done", stage=stage, message=message, data=data)
+
+    def failed(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
+        self.emit("failed", stage=stage, message=message, data=data)
+
+
+def build_progress_reporter(
+    *,
+    workflow: str,
+    platform: str,
+    content_kind: str,
+    input_value: Optional[str] = None,
+    enabled: bool = True,
+) -> ProgressReporter:
+    return ProgressReporter(
+        workflow=workflow,
+        platform=platform,
+        content_kind=content_kind,
+        run_id=f"{platform}.{content_kind}",
+        enabled=enabled,
+        defaults={"input_value": str(input_value or "")[:240]},
+    )

package/skills/single-work-analysis/scripts/core/storage_router.py

@@ -0,0 +1,253 @@
+#!/usr/bin/env python3
+"""Storage routing helpers for benchmark card outputs."""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple
+
+DEFAULT_CARD_TYPE_ROUTES: Dict[str, Dict[str, Any]] = {
+    "work": {
+        "prefix": "CBV",
+        "parts": ["内容系统", "对标研究", "作品卡"],
+    },
+    "author": {
+        "prefix": "CBA",
+        "parts": ["内容系统", "对标研究", "作者卡"],
+    },
+    "author_sample_work": {
+        "prefix": "CBV",
+        "parts": ["内容系统", "对标研究", "作者样本卡", "{platform}-{author_slug}"],
+    },
+}
+
+DEFAULT_CONTENT_KIND_CARD_TYPE: Dict[str, str] = {
+    "single_video": "work",
+    "work": "work",
+    "author_home": "author_sample_work",
+    "author_sample_work": "author_sample_work",
+    "author_analysis": "author",
+}
+
+CARD_TYPE_ALIASES: Dict[str, str] = {
+    "sample": "author_sample_work",
+    "sample_work": "author_sample_work",
+    "homepage_sample": "author_sample_work",
+    "author_homepage_sample": "author_sample_work",
+    "author_home": "author_sample_work",
+    "author_analysis": "author",
+}
+
+CONTENT_KIND_ALIASES: Dict[str, str] = {
+    "author_homepage": "author_home",
+    "author_homepage_sample": "author_home",
+    "homepage_sample": "author_home",
+    "analysis_author": "author_analysis",
+}
+
+
+def normalize_card_type(card_type: str) -> str:
+    normalized = (card_type or "").strip().lower().replace("-", "_")
+    if normalized in CARD_TYPE_ALIASES:
+        normalized = CARD_TYPE_ALIASES[normalized]
+    if normalized in {"work", "author", "author_sample_work"}:
+        return normalized
+    return "work"
+
+
+def normalize_content_kind(content_kind: Optional[str]) -> str:
+    normalized = (content_kind or "").strip().lower().replace("-", "_")
+    if not normalized:
+        return ""
+    return CONTENT_KIND_ALIASES.get(normalized, normalized)
+
+
+def _storage_routes_cfg(storage_config: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+    if not isinstance(storage_config, dict):
+        return {}
+    routes = storage_config.get("storage_routes")
+    return routes if isinstance(routes, dict) else {}
+
+
+def _configured_content_kind_map(storage_config: Optional[Dict[str, Any]]) -> Dict[str, str]:
+    routes = _storage_routes_cfg(storage_config)
+    configured = routes.get("content_kind_card_type")
+    if not isinstance(configured, dict):
+        return DEFAULT_CONTENT_KIND_CARD_TYPE
+
+    merged = dict(DEFAULT_CONTENT_KIND_CARD_TYPE)
+    for key, value in configured.items():
+        k = normalize_content_kind(str(key))
+        v = normalize_card_type(str(value))
+        if k:
+            merged[k] = v
+    return merged
+
+
+def _configured_card_routes(storage_config: Optional[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
+    routes = _storage_routes_cfg(storage_config)
+    configured = routes.get("card_type_routes")
+    if not isinstance(configured, dict):
+        return DEFAULT_CARD_TYPE_ROUTES
+
+    merged: Dict[str, Dict[str, Any]] = {k: dict(v) for k, v in DEFAULT_CARD_TYPE_ROUTES.items()}
+    for key, value in configured.items():
+        card_type = normalize_card_type(str(key))
+        if not isinstance(value, dict):
+            continue
+
+        prefix = value.get("prefix")
+        parts = value.get("parts")
+        if not isinstance(parts, list) or not all(isinstance(x, str) and x for x in parts):
+            continue
+
+        merged[card_type] = {
+            "prefix": str(prefix) if isinstance(prefix, str) and prefix else merged.get(card_type, {}).get("prefix", ""),
+            "parts": parts,
+        }
+    return merged
+
+
+DEFAULT_CARD_FILENAME_PATTERN = "{prefix}-{author_slug}-{title_slug}{ext}"
+DEFAULT_JSON_FILENAME_PATTERN = "{timestamp}-{platform}-{identifier}{ext}"
+_INVALID_FILENAME_CHARS = re.compile(r"[\\\\/:*?\"<>|]+")
+_SPACE_RUN = re.compile(r"\s+")
+
+
+def _sanitize_filename_token(value: Any, fallback: str = "item") -> str:
+    text = str(value or "").strip()
+    if not text:
+        text = fallback
+    text = _INVALID_FILENAME_CHARS.sub("-", text)
+    text = _SPACE_RUN.sub("-", text)
+    text = text.replace("/", "-").replace("\\", "-")
+    text = re.sub(r"-{2,}", "-", text).strip(" .-_")
+    return text or fallback
+
+
+def resolve_card_filename_pattern(storage_config: Optional[Dict[str, Any]]) -> str:
+    if not isinstance(storage_config, dict):
+        return DEFAULT_CARD_FILENAME_PATTERN
+    naming_rules = storage_config.get("naming_rules")
+    if not isinstance(naming_rules, dict):
+        return DEFAULT_CARD_FILENAME_PATTERN
+    pattern = str(naming_rules.get("card_filename_pattern") or "").strip()
+    return pattern or DEFAULT_CARD_FILENAME_PATTERN
+
+
+def resolve_json_filename_pattern(storage_config: Optional[Dict[str, Any]]) -> str:
+    if not isinstance(storage_config, dict):
+        return DEFAULT_JSON_FILENAME_PATTERN
+    naming_rules = storage_config.get("naming_rules")
+    if not isinstance(naming_rules, dict):
+        return DEFAULT_JSON_FILENAME_PATTERN
+    pattern = str(naming_rules.get("json_filename_pattern") or "").strip()
+    return pattern or DEFAULT_JSON_FILENAME_PATTERN
+
+
+def render_output_filename(
+    *,
+    pattern: str,
+    context: Dict[str, Any],
+    default_filename: str,
+    default_ext: str,
+) -> str:
+    safe_context = {key: _sanitize_filename_token(value, fallback=key) for key, value in context.items()}
+    safe_context["ext"] = default_ext
+    try:
+        rendered = str(pattern).format(**safe_context).strip()
+    except Exception:
+        rendered = default_filename
+    rendered = _INVALID_FILENAME_CHARS.sub("-", rendered)
+    rendered = _SPACE_RUN.sub("-", rendered)
+    rendered = rendered.replace("/", "-").replace("\\", "-").strip()
+    if not rendered:
+        rendered = default_filename
+    if not Path(rendered).suffix:
+        rendered = f"{rendered}{default_ext}"
+    return rendered
+
+
+def resolve_effective_card_type(
+    *,
+    card_type: str,
+    content_kind: Optional[str],
+    storage_config: Optional[Dict[str, Any]],
+    force_card_type: bool = False,
+) -> str:
+    normalized_card_type = normalize_card_type(card_type)
+    if force_card_type:
+        return normalized_card_type
+
+    normalized_content_kind = normalize_content_kind(content_kind)
+    if not normalized_content_kind:
+        return normalized_card_type
+
+    card_type_map = _configured_content_kind_map(storage_config)
+    mapped = card_type_map.get(normalized_content_kind)
+    if mapped is not None:
+        return normalize_card_type(str(mapped))
+
+    return normalized_card_type
+
+
+def render_route_parts(parts: List[str], *, context: Dict[str, str]) -> List[str]:
+    rendered: List[str] = []
+    for item in parts:
+        try:
+            rendered.append(item.format(**context))
+        except Exception:
+            rendered.append(item)
+    return rendered
+
+
+def build_card_output_path(
+    *,
+    card_root: str,
+    platform: str,
+    card_type: str,
+    author_slug: str,
+    title_slug: str,
+    year: str,
+    year_month: str,
+    timestamp: str,
+    storage_config: Optional[Dict[str, Any]],
+) -> Tuple[str, str]:
+    card_routes = _configured_card_routes(storage_config)
+    route = card_routes.get(card_type) or DEFAULT_CARD_TYPE_ROUTES["work"]
+    parts = route.get("parts") if isinstance(route.get("parts"), list) else DEFAULT_CARD_TYPE_ROUTES["work"]["parts"]
+    prefix = str(route.get("prefix") or "CBV")
+
+    route_context = {
+        "platform": platform,
+        "author_slug": author_slug,
+        "title_slug": title_slug,
+        "year": year,
+        "year_month": year_month,
+        "timestamp": timestamp,
+    }
+    rendered_parts = render_route_parts(parts, context=route_context)
+    directory = os.path.join(card_root, *rendered_parts)
+    os.makedirs(directory, exist_ok=True)
+
+    default_filename = f"{prefix}-{author_slug}-{title_slug}.md"
+    filename = render_output_filename(
+        pattern=resolve_card_filename_pattern(storage_config),
+        context={
+            "prefix": prefix,
+            "platform": platform,
+            "card_type": card_type,
+            "author_slug": author_slug,
+            "title_slug": title_slug,
+            "timestamp": timestamp,
+            "date": timestamp.split("-", 1)[0],
+            "identifier": f"{platform}-{author_slug}-{title_slug}",
+            "ext": ".md",
+        },
+        default_filename=default_filename,
+        default_ext=".md",
+    )
+
+    return os.path.join(directory, filename), "/".join(rendered_parts)