@tikomni/skills 0.1.3 → 0.1.4

package/skills/single-work-analysis/scripts/core/progress_report.py

@@ -1,258 +0,0 @@
-#!/usr/bin/env python3
-"""Structured stderr progress reporting for long-running TikOmni workflows."""
-
-from __future__ import annotations
-
-import json
-import sys
-from datetime import datetime, timezone
-from typing import Any, Dict, Optional
-from urllib.parse import urlparse
-
-
-VALID_EVENTS = {"started", "progress", "done", "failed"}
-_MASKED_TEXT = "<redacted>"
-_MAX_TEXT_PREVIEW = 180
-_MAX_LIST_ITEMS = 8
-_SENSITIVE_KEYS = {
-    "api_key",
-    "authorization",
-    "cookie",
-    "cookies",
-    "set_cookie",
-    "token",
-    "xsec_token",
-}
-_LONG_TEXT_KEYS = {
-    "asr_clean",
-    "asr_raw",
-    "prompt",
-    "prompt_text",
-    "raw_content",
-    "stderr",
-    "stdout",
-    "transcript",
-    "transcript_text",
-}
-
-
-def _looks_like_url(text: str) -> bool:
-    return text.startswith("http://") or text.startswith("https://")
-
-
-def _mask_key(key: str) -> bool:
-    lowered = key.lower()
-    if lowered in _SENSITIVE_KEYS:
-        return True
-    return any(token in lowered for token in ("api_key", "token", "cookie", "authorization"))
-
-
-def _sanitize_url(text: str) -> str:
-    try:
-        parsed = urlparse(text)
-    except Exception:
-        return text[:_MAX_TEXT_PREVIEW]
-    if not parsed.scheme or not parsed.netloc:
-        return text[:_MAX_TEXT_PREVIEW]
-    return f"{parsed.scheme}://{parsed.netloc}{parsed.path}"
-
-
-def _truncate_text(text: str) -> str:
-    if len(text) <= _MAX_TEXT_PREVIEW:
-        return text
-    return f"{text[:_MAX_TEXT_PREVIEW]}…(len={len(text)})"
-
-
-def _sanitize_scalar(value: Any, *, key: str = "") -> Any:
-    if value is None or isinstance(value, (bool, int, float)):
-        return value
-
-    text = str(value)
-    if _mask_key(key):
-        return _MASKED_TEXT
-    if key.lower() in _LONG_TEXT_KEYS:
-        return f"<redacted:{key}:len={len(text)}>"
-    if _looks_like_url(text):
-        return _sanitize_url(text)
-    return _truncate_text(text)
-
-
-def _sanitize_payload(value: Any, *, key: str = "") -> Any:
-    if isinstance(value, dict):
-        sanitized: Dict[str, Any] = {}
-        for child_key, child_value in value.items():
-            child_key_text = str(child_key)
-            if _mask_key(child_key_text):
-                sanitized[child_key_text] = _MASKED_TEXT
-                continue
-            sanitized[child_key_text] = _sanitize_payload(child_value, key=child_key_text)
-        return sanitized
-
-    if isinstance(value, list):
-        items = value[:_MAX_LIST_ITEMS]
-        sanitized_items = [_sanitize_payload(item, key=key) for item in items]
-        if len(value) > _MAX_LIST_ITEMS:
-            sanitized_items.append(f"...({len(value) - _MAX_LIST_ITEMS} more)")
-        return sanitized_items
-
-    return _sanitize_scalar(value, key=key)
-
-
-class ProgressReporter:
-    def __init__(
-        self,
-        *,
-        workflow: str,
-        platform: str,
-        content_kind: str,
-        run_id: Optional[str] = None,
-        scope: str = "workflow",
-        enabled: bool = True,
-        defaults: Optional[Dict[str, Any]] = None,
-    ) -> None:
-        self.workflow = str(workflow or "unknown")
-        self.platform = str(platform or "unknown")
-        self.content_kind = str(content_kind or "unknown")
-        self.run_id = str(run_id or f"{self.platform}.{self.content_kind}")
-        self.scope = str(scope or "workflow")
-        self.enabled = bool(enabled)
-        self.defaults = _sanitize_payload(dict(defaults or {}))
-
-    def child(self, *, scope: str, defaults: Optional[Dict[str, Any]] = None) -> "ProgressReporter":
-        merged = dict(self.defaults)
-        if defaults:
-            merged.update(_sanitize_payload(defaults))
-        return ProgressReporter(
-            workflow=self.workflow,
-            platform=self.platform,
-            content_kind=self.content_kind,
-            run_id=self.run_id,
-            scope=scope,
-            enabled=self.enabled,
-            defaults=merged,
-        )
-
-    def emit(self, event: str, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
-        if not self.enabled:
-            return
-        normalized_event = str(event or "progress").strip().lower()
-        if normalized_event not in VALID_EVENTS:
-            normalized_event = "progress"
-        payload: Dict[str, Any] = {
-            "channel": "tikomni_progress",
-            "ts": datetime.now(timezone.utc).isoformat(timespec="seconds"),
-            "workflow": self.workflow,
-            "platform": self.platform,
-            "content_kind": self.content_kind,
-            "run_id": self.run_id,
-            "scope": self.scope,
-            "event": normalized_event,
-            "stage": str(stage or "unknown"),
-        }
-        if message:
-            payload["message"] = _truncate_text(str(message))
-        merged_data = dict(self.defaults)
-        if isinstance(data, dict) and data:
-            merged_data.update(_sanitize_payload(data))
-        if merged_data:
-            payload["data"] = merged_data
-        sys.stderr.write(json.dumps(payload, ensure_ascii=False) + "\n")
-        sys.stderr.flush()
-
-    def started(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
-        self.emit("started", stage=stage, message=message, data=data)
-
-    def progress(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
-        self.emit("progress", stage=stage, message=message, data=data)
-
-    def done(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
-        self.emit("done", stage=stage, message=message, data=data)
-
-    def failed(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
-        self.emit("failed", stage=stage, message=message, data=data)
-
-    def heartbeat(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
-        heartbeat_data = dict(data or {})
-        heartbeat_data["heartbeat"] = True
-        self.progress(stage=stage, message=message or "heartbeat", data=heartbeat_data)
-
-    def http_event(
-        self,
-        *,
-        stage: str,
-        endpoint: str,
-        response: Optional[Dict[str, Any]],
-        summary: Optional[Dict[str, Any]] = None,
-        route_label: Optional[str] = None,
-    ) -> None:
-        payload: Dict[str, Any] = {
-            "kind": "http",
-            "endpoint": endpoint,
-        }
-        if route_label:
-            payload["route_label"] = route_label
-        if isinstance(response, dict):
-            payload.update(
-                {
-                    "ok": bool(response.get("ok")),
-                    "status_code": response.get("status_code"),
-                    "request_id": response.get("request_id"),
-                    "attempt": int(response.get("retry_attempt", 0)) + 1,
-                    "fallback_trigger_reason": response.get("fallback_trigger_reason"),
-                    "timeout_retry_exhausted": bool(response.get("timeout_retry_exhausted")),
-                }
-            )
-        if isinstance(summary, dict) and summary:
-            payload["summary"] = summary
-        event_fn = self.done if isinstance(response, dict) and response.get("ok") else self.failed
-        event_fn(stage=stage, message="http request finished", data=payload)
-
-    def subprocess_event(
-        self,
-        *,
-        stage: str,
-        provider: str,
-        operation: str,
-        event: str,
-        duration_ms: Optional[int] = None,
-        exit_code: Optional[int] = None,
-        summary: Optional[Dict[str, Any]] = None,
-    ) -> None:
-        payload: Dict[str, Any] = {
-            "kind": "subprocess",
-            "provider": provider,
-            "operation": operation,
-        }
-        if duration_ms is not None:
-            payload["duration_ms"] = int(duration_ms)
-        if exit_code is not None:
-            payload["exit_code"] = int(exit_code)
-        if isinstance(summary, dict) and summary:
-            payload["summary"] = summary
-
-        if event == "started":
-            self.started(stage=stage, message="subprocess started", data=payload)
-        elif event == "done":
-            self.done(stage=stage, message="subprocess finished", data=payload)
-        elif event == "failed":
-            self.failed(stage=stage, message="subprocess failed", data=payload)
-        else:
-            self.progress(stage=stage, message="subprocess progress", data=payload)
-
-
-def build_progress_reporter(
-    *,
-    workflow: str,
-    platform: str,
-    content_kind: str,
-    input_value: Optional[str] = None,
-    enabled: bool = True,
-) -> ProgressReporter:
-    return ProgressReporter(
-        workflow=workflow,
-        platform=platform,
-        content_kind=content_kind,
-        run_id=f"{platform}.{content_kind}",
-        enabled=enabled,
-        defaults={"input_value": _sanitize_scalar(str(input_value or ""), key="input_value")},
-    )

package/skills/single-work-analysis/scripts/core/storage_router.py

@@ -1,220 +0,0 @@
-#!/usr/bin/env python3
-"""Storage routing helpers for single-work card outputs."""
-
-from __future__ import annotations
-
-import os
-import re
-from pathlib import Path
-from typing import Any, Dict, List, Optional, Tuple
-
-
-DEFAULT_CARD_TYPE_ROUTES: Dict[str, Dict[str, Any]] = {
-    "work": {
-        "prefix": "CBV",
-        "parts": ["内容系统", "对标研究", "作品卡"],
-    },
-}
-
-DEFAULT_CONTENT_KIND_CARD_TYPE: Dict[str, str] = {
-    "single_video": "work",
-    "note": "work",
-    "work": "work",
-}
-
-CONTENT_KIND_ALIASES: Dict[str, str] = {
-    "single-work": "work",
-}
-
-DEFAULT_CARD_FILENAME_PATTERN = "{prefix}-{platform}-{author_slug}-{title_slug}{ext}"
-DEFAULT_JSON_FILENAME_PATTERN = "{timestamp}-{platform}-{identifier}{ext}"
-_INVALID_FILENAME_CHARS = re.compile(r"[\\\\/:*?\"<>|]+")
-_SPACE_RUN = re.compile(r"\s+")
-
-
-def normalize_card_type(card_type: str) -> str:
-    normalized = (card_type or "").strip().lower().replace("-", "_")
-    return "work" if normalized == "work" else "work"
-
-
-def normalize_content_kind(content_kind: Optional[str]) -> str:
-    normalized = (content_kind or "").strip().lower().replace("-", "_")
-    if not normalized:
-        return ""
-    return CONTENT_KIND_ALIASES.get(normalized, normalized)
-
-
-def _storage_routes_cfg(storage_config: Optional[Dict[str, Any]]) -> Dict[str, Any]:
-    if not isinstance(storage_config, dict):
-        return {}
-    routes = storage_config.get("storage_routes")
-    return routes if isinstance(routes, dict) else {}
-
-
-def _configured_content_kind_map(storage_config: Optional[Dict[str, Any]]) -> Dict[str, str]:
-    routes = _storage_routes_cfg(storage_config)
-    configured = routes.get("content_kind_card_type")
-    if not isinstance(configured, dict):
-        return dict(DEFAULT_CONTENT_KIND_CARD_TYPE)
-
-    merged = dict(DEFAULT_CONTENT_KIND_CARD_TYPE)
-    for key, value in configured.items():
-        normalized_key = normalize_content_kind(str(key))
-        if normalized_key:
-            merged[normalized_key] = normalize_card_type(str(value))
-    return merged
-
-
-def _configured_card_routes(storage_config: Optional[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
-    routes = _storage_routes_cfg(storage_config)
-    configured = routes.get("card_type_routes")
-    if not isinstance(configured, dict):
-        return {key: dict(value) for key, value in DEFAULT_CARD_TYPE_ROUTES.items()}
-
-    merged = {key: dict(value) for key, value in DEFAULT_CARD_TYPE_ROUTES.items()}
-    for key, value in configured.items():
-        card_type = normalize_card_type(str(key))
-        if card_type != "work" or not isinstance(value, dict):
-            continue
-        parts = value.get("parts")
-        if not isinstance(parts, list) or not all(isinstance(item, str) and item for item in parts):
-            continue
-        prefix = str(value.get("prefix") or merged["work"]["prefix"])
-        merged["work"] = {
-            "prefix": prefix,
-            "parts": parts,
-        }
-    return merged
-
-
-def _sanitize_filename_token(value: Any, fallback: str = "item") -> str:
-    text = str(value or "").strip()
-    if not text:
-        text = fallback
-    text = _INVALID_FILENAME_CHARS.sub("-", text)
-    text = _SPACE_RUN.sub("-", text)
-    text = text.replace("/", "-").replace("\\", "-")
-    text = re.sub(r"-{2,}", "-", text).strip(" .-_")
-    return text or fallback
-
-
-def resolve_card_filename_pattern(storage_config: Optional[Dict[str, Any]]) -> str:
-    if not isinstance(storage_config, dict):
-        return DEFAULT_CARD_FILENAME_PATTERN
-    naming_rules = storage_config.get("naming_rules")
-    if not isinstance(naming_rules, dict):
-        return DEFAULT_CARD_FILENAME_PATTERN
-    pattern = str(naming_rules.get("card_filename_pattern") or "").strip()
-    return pattern or DEFAULT_CARD_FILENAME_PATTERN
-
-
-def resolve_json_filename_pattern(storage_config: Optional[Dict[str, Any]]) -> str:
-    if not isinstance(storage_config, dict):
-        return DEFAULT_JSON_FILENAME_PATTERN
-    naming_rules = storage_config.get("naming_rules")
-    if not isinstance(naming_rules, dict):
-        return DEFAULT_JSON_FILENAME_PATTERN
-    pattern = str(naming_rules.get("json_filename_pattern") or "").strip()
-    return pattern or DEFAULT_JSON_FILENAME_PATTERN
-
-
-def render_output_filename(
-    *,
-    pattern: str,
-    context: Dict[str, Any],
-    default_filename: str,
-    default_ext: str,
-) -> str:
-    safe_context = {key: _sanitize_filename_token(value, fallback=key) for key, value in context.items()}
-    safe_context["ext"] = default_ext
-    try:
-        rendered = str(pattern).format(**safe_context).strip()
-    except Exception:
-        rendered = default_filename
-    rendered = _INVALID_FILENAME_CHARS.sub("-", rendered)
-    rendered = _SPACE_RUN.sub("-", rendered)
-    rendered = rendered.replace("/", "-").replace("\\", "-").strip()
-    if not rendered:
-        rendered = default_filename
-    if not Path(rendered).suffix:
-        rendered = f"{rendered}{default_ext}"
-    return rendered
-
-
-def resolve_effective_card_type(
-    *,
-    card_type: str,
-    content_kind: Optional[str],
-    storage_config: Optional[Dict[str, Any]],
-    force_card_type: bool = False,
-) -> str:
-    normalized_card_type = normalize_card_type(card_type)
-    if force_card_type:
-        return normalized_card_type
-
-    normalized_content_kind = normalize_content_kind(content_kind)
-    if not normalized_content_kind:
-        return normalized_card_type
-
-    mapped = _configured_content_kind_map(storage_config).get(normalized_content_kind)
-    return normalize_card_type(str(mapped)) if mapped is not None else normalized_card_type
-
-
-def render_route_parts(parts: List[str], *, context: Dict[str, str]) -> List[str]:
-    rendered: List[str] = []
-    for item in parts:
-        try:
-            rendered.append(item.format(**context))
-        except Exception:
-            rendered.append(item)
-    return rendered
-
-
-def build_card_output_path(
-    *,
-    card_root: str,
-    platform: str,
-    card_type: str,
-    author_slug: str,
-    title_slug: str,
-    year: str,
-    year_month: str,
-    timestamp: str,
-    storage_config: Optional[Dict[str, Any]],
-) -> Tuple[str, str]:
-    card_routes = _configured_card_routes(storage_config)
-    route = card_routes.get(card_type) or DEFAULT_CARD_TYPE_ROUTES["work"]
-    parts = route.get("parts") if isinstance(route.get("parts"), list) else DEFAULT_CARD_TYPE_ROUTES["work"]["parts"]
-    prefix = str(route.get("prefix") or "CBV")
-
-    route_context = {
-        "platform": platform,
-        "author_slug": author_slug,
-        "title_slug": title_slug,
-        "year": year,
-        "year_month": year_month,
-        "timestamp": timestamp,
-    }
-    rendered_parts = render_route_parts(parts, context=route_context)
-    directory = os.path.join(card_root, *rendered_parts)
-    os.makedirs(directory, exist_ok=True)
-
-    default_filename = f"{prefix}-{platform}-{author_slug}-{title_slug}.md"
-    filename = render_output_filename(
-        pattern=resolve_card_filename_pattern(storage_config),
-        context={
-            "prefix": prefix,
-            "platform": platform,
-            "card_type": card_type,
-            "author_slug": author_slug,
-            "title_slug": title_slug,
-            "timestamp": timestamp,
-            "date": timestamp.split("-", 1)[0],
-            "identifier": f"{platform}-{author_slug}-{title_slug}",
-            "ext": ".md",
-        },
-        default_filename=default_filename,
-        default_ext=".md",
-    )
-
-    return os.path.join(directory, filename), "/".join(rendered_parts)