@tikomni/skills 0.1.3 → 0.1.4

package/skills/{creator-analysis → social-media-crawl}/scripts/core/progress_report.py

@@ -15,6 +15,10 @@ from datetime import datetime, timezone
 from typing import Any, Dict, Optional
 
 
+def _normalize_http_summary(summary: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+    return dict(summary) if isinstance(summary, dict) else {}
+
+
 VALID_EVENTS = {"started", "progress", "done", "failed"}
 
 
@@ -92,6 +96,32 @@ class ProgressReporter:
     def failed(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
         self.emit("failed", stage=stage, message=message, data=data)
 
+    def http_event(
+        self,
+        *,
+        stage: str,
+        endpoint: Optional[str],
+        response: Optional[Dict[str, Any]],
+        route_label: Optional[str] = None,
+        summary: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        response_obj = response if isinstance(response, dict) else {}
+        payload = _normalize_http_summary(summary)
+        payload.update(
+            {
+                "endpoint": str(endpoint or ""),
+                "route_label": str(route_label or ""),
+                "ok": bool(response_obj.get("ok")),
+                "status_code": response_obj.get("status_code"),
+                "request_id": response_obj.get("request_id"),
+                "error_reason": response_obj.get("error_reason"),
+            }
+        )
+        self.progress(stage=stage, message="http event", data=payload)
+
+    def heartbeat(self, *, stage: str, message: str = "", data: Optional[Dict[str, Any]] = None) -> None:
+        self.progress(stage=stage, message=message, data=data)
+
 
 def build_progress_reporter(
     *,
@@ -99,6 +129,7 @@ def build_progress_reporter(
     platform: str,
     content_kind: str,
     input_value: Optional[str] = None,
+    scope: str = "workflow",
     enabled: bool = True,
 ) -> ProgressReporter:
     return ProgressReporter(
@@ -106,6 +137,7 @@ def build_progress_reporter(
         platform=platform,
         content_kind=content_kind,
         run_id=f"{platform}.{content_kind}",
+        scope=scope,
         enabled=enabled,
         defaults={"input_value": str(input_value or "")[:240]},
     )

package/skills/social-media-crawl/scripts/core/storage_router.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Storage routing helpers for work fact cards."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+DEFAULT_CARD_FILENAME_PATTERN = "{identifier}{ext}"
+DEFAULT_JSON_FILENAME_PATTERN = "{timestamp}-{platform}-{identifier}{ext}"
+_INVALID_FILENAME_CHARS = re.compile(r"[\\\\/:*?\"<>|]+")
+_SPACE_RUN = re.compile(r"\s+")
+_INVALID_AUTHOR_SLUGS = {"0", "unknown", "none", "null", "nil", "na", "n-a"}
+
+
+def slugify_token(value: Any, fallback: str = "unknown") -> str:
+    text = str(value or "").strip().lower()
+    if not text:
+        text = fallback
+    text = text.replace(" ", "-")
+    text = _INVALID_FILENAME_CHARS.sub("-", text)
+    text = re.sub(r"[^a-z0-9_-]+", "-", text)
+    text = re.sub(r"-{2,}", "-", text).strip("-_")
+    return text or fallback
+
+
+def resolve_card_filename_pattern(storage_config: Optional[Dict[str, Any]]) -> str:
+    if not isinstance(storage_config, dict):
+        return DEFAULT_CARD_FILENAME_PATTERN
+    naming_rules = storage_config.get("naming_rules")
+    if not isinstance(naming_rules, dict):
+        return DEFAULT_CARD_FILENAME_PATTERN
+    return str(naming_rules.get("card_filename_pattern") or "").strip() or DEFAULT_CARD_FILENAME_PATTERN
+
+
+def resolve_json_filename_pattern(storage_config: Optional[Dict[str, Any]]) -> str:
+    if not isinstance(storage_config, dict):
+        return DEFAULT_JSON_FILENAME_PATTERN
+    naming_rules = storage_config.get("naming_rules")
+    if not isinstance(naming_rules, dict):
+        return DEFAULT_JSON_FILENAME_PATTERN
+    return str(naming_rules.get("json_filename_pattern") or "").strip() or DEFAULT_JSON_FILENAME_PATTERN
+
+
+def render_output_filename(
+    *,
+    pattern: str,
+    context: Dict[str, Any],
+    default_filename: str,
+    default_ext: str,
+) -> str:
+    safe_context = {key: slugify_token(value, fallback=key) for key, value in context.items()}
+    safe_context["ext"] = default_ext
+    try:
+        rendered = str(pattern).format(**safe_context).strip()
+    except Exception:
+        rendered = default_filename
+    rendered = _INVALID_FILENAME_CHARS.sub("-", rendered)
+    rendered = _SPACE_RUN.sub("-", rendered)
+    rendered = rendered.replace("/", "-").replace("\\", "-").strip()
+    if not rendered:
+        rendered = default_filename
+    if not Path(rendered).suffix:
+        rendered = f"{rendered}{default_ext}"
+    return rendered
+
+
+def _storage_routes_cfg(storage_config: Optional[Dict[str, Any]]) -> Dict[str, Any]:
+    if not isinstance(storage_config, dict):
+        return {}
+    routes = storage_config.get("storage_routes")
+    return routes if isinstance(routes, dict) else {}
+
+
+def resolve_card_root(storage_config: Optional[Dict[str, Any]], explicit_card_root: Optional[str] = None) -> str:
+    if explicit_card_root:
+        candidate = Path(explicit_card_root).expanduser()
+        if not candidate.is_absolute():
+            raise ValueError("card_root must be an absolute path")
+        return str(candidate.resolve())
+
+    routes = _storage_routes_cfg(storage_config)
+    default_route = routes.get("default") if isinstance(routes.get("default"), dict) else {}
+    card_root = str(default_route.get("card_root") or "").strip()
+    if not card_root:
+        raise ValueError("missing_card_root")
+    return str(Path(card_root).expanduser().resolve())
+
+
+def resolve_author_slug(platform: str, author_handle: str, platform_author_id: str) -> str:
+    handle_slug = slugify_token(author_handle, fallback="")
+    if handle_slug and handle_slug not in _INVALID_AUTHOR_SLUGS:
+        return handle_slug
+    author_id_slug = slugify_token(platform_author_id, fallback="")
+    if author_id_slug:
+        return author_id_slug
+    return f"{slugify_token(platform)}-unknown"
+
+
+def resolve_author_directory_name(platform: str, author_handle: str, platform_author_id: str) -> str:
+    return f"{slugify_token(platform)}-{resolve_author_slug(platform, author_handle, platform_author_id)}"
+
+
+def resolve_card_route_parts(
+    storage_config: Optional[Dict[str, Any]],
+    *,
+    platform: str,
+    author_slug: str,
+) -> List[str]:
+    routes = _storage_routes_cfg(storage_config)
+    card_type_routes = routes.get("card_type_routes") if isinstance(routes.get("card_type_routes"), dict) else {}
+    work_route = card_type_routes.get("work") if isinstance(card_type_routes.get("work"), dict) else {}
+    parts = work_route.get("parts") if isinstance(work_route.get("parts"), list) else ["内容系统", "作品库", "{platform}-{author_slug}"]
+    rendered: List[str] = []
+    context = {"platform": slugify_token(platform), "author_slug": author_slug}
+    for part in parts:
+        try:
+            rendered.append(str(part).format(**context))
+        except Exception:
+            rendered.append(str(part))
+    return rendered
+
+
+def build_work_fact_card_paths(
+    *,
+    card_root: str,
+    platform: str,
+    platform_work_id: str,
+    author_handle: str,
+    platform_author_id: str,
+    storage_config: Optional[Dict[str, Any]],
+    fallback_identifier: str,
+) -> Dict[str, str]:
+    author_slug = resolve_author_slug(platform, author_handle, platform_author_id)
+    route_parts = resolve_card_route_parts(storage_config, platform=platform, author_slug=author_slug)
+    directory = Path(card_root).joinpath(*route_parts)
+    directory.mkdir(parents=True, exist_ok=True)
+
+    identifier = slugify_token(platform_work_id, fallback="") or slugify_token(fallback_identifier, fallback="unknown")
+    json_filename = render_output_filename(
+        pattern=resolve_card_filename_pattern(storage_config),
+        context={"identifier": identifier, "platform": platform, "author_slug": author_slug, "ext": ".json"},
+        default_filename=f"{identifier}.json",
+        default_ext=".json",
+    )
+    markdown_filename = render_output_filename(
+        pattern=resolve_card_filename_pattern(storage_config),
+        context={"identifier": identifier, "platform": platform, "author_slug": author_slug, "ext": ".md"},
+        default_filename=f"{identifier}.md",
+        default_ext=".md",
+    )
+    return {
+        "identifier": identifier,
+        "author_slug": author_slug,
+        "directory": str(directory),
+        "route": "/".join(route_parts),
+        "json_path": str(directory / json_filename),
+        "markdown_path": str(directory / markdown_filename),
+    }

package/skills/social-media-crawl/scripts/core/u3_fallback.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+"""U3 public URL fallback helpers."""
+
+from __future__ import annotations
+
+import mimetypes
+import os
+import tempfile
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+from scripts.core.tikomni_common import DEFAULT_USER_AGENT, call_json_api, normalize_text
+
+DEFAULT_U3_PROVIDER = "oss"
+DEFAULT_CONTENT_TYPE = "video/mp4"
+DOWNLOAD_CHUNK_SIZE = 1024 * 1024
+
+
+def _safe_name_from_url(source_url: str) -> str:
+    parsed = urllib.parse.urlparse(source_url)
+    name = Path(parsed.path).name.strip()
+    return name or "media.bin"
+
+
+def _guess_content_type(file_name: str, header_value: Optional[str]) -> str:
+    header_text = normalize_text(header_value)
+    if header_text:
+        return header_text.split(";", 1)[0].strip() or DEFAULT_CONTENT_TYPE
+    guessed, _ = mimetypes.guess_type(file_name)
+    return guessed or DEFAULT_CONTENT_TYPE
+
+
+def download_media_to_tempfile(
+    *,
+    source_url: str,
+    timeout_ms: int,
+) -> Dict[str, Any]:
+    parsed = urllib.parse.urlparse(source_url)
+    host = parsed.netloc.lower()
+    headers = {
+        "User-Agent": os.getenv("TIKOMNI_HTTP_USER_AGENT", DEFAULT_USER_AGENT),
+        "Accept": "*/*",
+    }
+    if "xhscdn.com" in host:
+        headers.update(
+            {
+                "Referer": "https://www.xiaohongshu.com/",
+                "Origin": "https://www.xiaohongshu.com",
+                "Range": "bytes=0-",
+            }
+        )
+    request = urllib.request.Request(
+        source_url,
+        headers=headers,
+        method="GET",
+    )
+
+    temp_path = ""
+    file_size = 0
+    request_id = None
+    try:
+        with urllib.request.urlopen(request, timeout=max(timeout_ms / 1000.0, 1.0)) as response:
+            final_url = response.geturl() or source_url
+            content_type = response.headers.get("Content-Type")
+            file_name = _safe_name_from_url(final_url)
+            suffix = Path(file_name).suffix or ".bin"
+            with tempfile.NamedTemporaryFile(prefix="tikomni-u3-", suffix=suffix, delete=False) as handle:
+                temp_path = handle.name
+                while True:
+                    chunk = response.read(DOWNLOAD_CHUNK_SIZE)
+                    if not chunk:
+                        break
+                    handle.write(chunk)
+                    file_size += len(chunk)
+
+            return {
+                "ok": file_size > 0,
+                "file_path": temp_path,
+                "size_bytes": file_size,
+                "file_name": file_name,
+                "content_type": _guess_content_type(file_name, content_type),
+                "final_url": final_url,
+                "request_id": request_id,
+                "error_reason": None if file_size > 0 else "u3_download_empty",
+            }
+    except urllib.error.HTTPError as error:
+        return {
+            "ok": False,
+            "file_path": temp_path,
+            "size_bytes": file_size,
+            "file_name": _safe_name_from_url(source_url),
+            "content_type": DEFAULT_CONTENT_TYPE,
+            "final_url": source_url,
+            "request_id": request_id,
+            "error_reason": f"u3_download_http_{error.code}",
+        }
+    except Exception as error:
+        return {
+            "ok": False,
+            "file_path": temp_path,
+            "size_bytes": file_size,
+            "file_name": _safe_name_from_url(source_url),
+            "content_type": DEFAULT_CONTENT_TYPE,
+            "final_url": source_url,
+            "request_id": request_id,
+            "error_reason": f"u3_download_failed:{normalize_text(error)}",
+        }
+
+
+def create_u3_upload(
+    *,
+    base_url: str,
+    token: str,
+    timeout_ms: int,
+    file_name: str,
+    content_type: str,
+    size_bytes: int,
+    provider: str = DEFAULT_U3_PROVIDER,
+) -> Dict[str, Any]:
+    return call_json_api(
+        base_url=base_url,
+        path="/api/u3/v1/uploads",
+        token=token,
+        method="POST",
+        timeout_ms=timeout_ms,
+        body={
+            "provider": provider,
+            "file_name": file_name,
+            "content_type": content_type,
+            "size_bytes": int(size_bytes),
+        },
+    )
+
+
+def upload_file_to_presigned_url(
+    *,
+    upload_url: str,
+    upload_headers: Optional[Dict[str, Any]],
+    upload_method: str,
+    file_path: str,
+    content_type: str,
+    timeout_ms: int,
+) -> Dict[str, Any]:
+    try:
+        with open(file_path, "rb") as handle:
+            data = handle.read()
+
+        headers = {
+            "Content-Type": content_type or DEFAULT_CONTENT_TYPE,
+            "User-Agent": os.getenv("TIKOMNI_HTTP_USER_AGENT", DEFAULT_USER_AGENT),
+        }
+        if isinstance(upload_headers, dict):
+            for key, value in upload_headers.items():
+                header_key = str(key).strip()
+                if not header_key:
+                    continue
+                headers[header_key] = str(value)
+
+        request = urllib.request.Request(
+            upload_url,
+            data=data,
+            headers=headers,
+            method=(upload_method or "PUT").upper(),
+        )
+        with urllib.request.urlopen(request, timeout=max(timeout_ms / 1000.0, 1.0)) as response:
+            status_code = response.getcode()
+            return {
+                "ok": 200 <= int(status_code) < 300,
+                "status_code": status_code,
+                "error_reason": None if 200 <= int(status_code) < 300 else f"u3_upload_http_{status_code}",
+            }
+    except urllib.error.HTTPError as error:
+        return {"ok": False, "status_code": error.code, "error_reason": f"u3_upload_http_{error.code}"}
+    except Exception as error:
+        return {"ok": False, "status_code": None, "error_reason": f"u3_upload_failed:{normalize_text(error)}"}
+
+
+def complete_u3_upload(
+    *,
+    base_url: str,
+    token: str,
+    timeout_ms: int,
+    upload_id: str,
+) -> Dict[str, Any]:
+    return call_json_api(
+        base_url=base_url,
+        path=f"/api/u3/v1/uploads/{upload_id}/complete",
+        token=token,
+        method="POST",
+        timeout_ms=timeout_ms,
+        body={},
+    )
+
+
+def cleanup_temp_file(file_path: Optional[str]) -> None:
+    if not file_path:
+        return
+    try:
+        Path(file_path).unlink(missing_ok=True)
+    except Exception:
+        pass
+
+
+def run_u3_public_url_fallback(
+    *,
+    base_url: str,
+    token: str,
+    timeout_ms: int,
+    source_url: str,
+    provider: str = DEFAULT_U3_PROVIDER,
+) -> Dict[str, Any]:
+    download_bundle = download_media_to_tempfile(source_url=source_url, timeout_ms=timeout_ms)
+    trace = [
+        {
+            "step": "u3_download_source",
+            "ok": bool(download_bundle.get("ok")),
+            "error_reason": download_bundle.get("error_reason"),
+            "size_bytes": download_bundle.get("size_bytes"),
+            "content_type": download_bundle.get("content_type"),
+            "file_name": download_bundle.get("file_name"),
+        }
+    ]
+    if not download_bundle.get("ok"):
+        cleanup_temp_file(download_bundle.get("file_path"))
+        return {
+            "ok": False,
+            "error_reason": download_bundle.get("error_reason") or "u3_download_failed",
+            "public_url": "",
+            "trace": trace,
+            "request_id": download_bundle.get("request_id"),
+        }
+
+    create_response = create_u3_upload(
+        base_url=base_url,
+        token=token,
+        timeout_ms=timeout_ms,
+        provider=provider,
+        file_name=str(download_bundle.get("file_name") or "media.bin"),
+        content_type=str(download_bundle.get("content_type") or DEFAULT_CONTENT_TYPE),
+        size_bytes=int(download_bundle.get("size_bytes") or 0),
+    )
+    create_body = create_response.get("data") if isinstance(create_response.get("data"), dict) else {}
+    create_payload = create_body.get("data") if isinstance(create_body.get("data"), dict) else {}
+    upload_id = normalize_text(create_payload.get("upload_id"))
+    upload_url = normalize_text(create_payload.get("upload_url"))
+    upload_method = normalize_text(create_payload.get("upload_method")) or "PUT"
+    upload_headers = create_payload.get("upload_headers") if isinstance(create_payload.get("upload_headers"), dict) else {}
+    trace.append(
+        {
+            "step": "u3_create_upload",
+            "ok": bool(create_response.get("ok") and upload_id and upload_url),
+            "request_id": create_response.get("request_id"),
+            "status_code": create_response.get("status_code"),
+            "error_reason": create_response.get("error_reason"),
+            "upload_id": upload_id,
+            "provider": create_payload.get("provider"),
+            "upload_method": upload_method,
+        }
+    )
+    if not create_response.get("ok") or not upload_id or not upload_url:
+        cleanup_temp_file(download_bundle.get("file_path"))
+        return {
+            "ok": False,
+            "error_reason": create_response.get("error_reason") or "u3_create_upload_failed",
+            "public_url": "",
+            "trace": trace,
+            "request_id": create_response.get("request_id"),
+        }
+
+    upload_response = upload_file_to_presigned_url(
+        upload_url=upload_url,
+        upload_headers=upload_headers,
+        upload_method=upload_method,
+        file_path=str(download_bundle.get("file_path") or ""),
+        content_type=str(download_bundle.get("content_type") or DEFAULT_CONTENT_TYPE),
+        timeout_ms=timeout_ms,
+    )
+    trace.append(
+        {
+            "step": "u3_upload_object",
+            "ok": bool(upload_response.get("ok")),
+            "status_code": upload_response.get("status_code"),
+            "error_reason": upload_response.get("error_reason"),
+        }
+    )
+    if not upload_response.get("ok"):
+        cleanup_temp_file(download_bundle.get("file_path"))
+        return {
+            "ok": False,
+            "error_reason": upload_response.get("error_reason") or "u3_upload_failed",
+            "public_url": "",
+            "trace": trace,
+            "request_id": create_response.get("request_id"),
+        }
+
+    complete_response = complete_u3_upload(
+        base_url=base_url,
+        token=token,
+        timeout_ms=timeout_ms,
+        upload_id=upload_id,
+    )
+    complete_body = complete_response.get("data") if isinstance(complete_response.get("data"), dict) else {}
+    complete_payload = complete_body.get("data") if isinstance(complete_body.get("data"), dict) else {}
+    public_url = normalize_text(complete_payload.get("file_url"))
+    trace.append(
+        {
+            "step": "u3_complete_upload",
+            "ok": bool(complete_response.get("ok") and public_url),
+            "request_id": complete_response.get("request_id"),
+            "status_code": complete_response.get("status_code"),
+            "error_reason": complete_response.get("error_reason"),
+            "upload_id": upload_id,
+            "public_url_present": bool(public_url),
+        }
+    )
+    cleanup_temp_file(download_bundle.get("file_path"))
+    return {
+        "ok": bool(complete_response.get("ok") and public_url),
+        "error_reason": None if complete_response.get("ok") and public_url else (complete_response.get("error_reason") or "u3_complete_failed"),
+        "public_url": public_url,
+        "upload_id": upload_id,
+        "provider": complete_payload.get("provider") or provider,
+        "request_id": complete_response.get("request_id") or create_response.get("request_id"),
+        "trace": trace,
+    }

package/skills/social-media-crawl/scripts/pipelines/__init__.py

@@ -0,0 +1,2 @@
+"""Fixed pipelines for social-media-crawl."""
+

package/skills/social-media-crawl/scripts/pipelines/douyin_creator_home_helpers.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Douyin creator-home helper entrypoints."""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List, Tuple
+
+from scripts.pipelines.homepage_collectors import collect_douyin_author_home_raw
+from scripts.pipelines.platform_adapters import adapt_douyin_author_home
+
+
+def collect_and_adapt(
+    *,
+    input_value: str,
+    base_url: str,
+    token: str,
+    timeout_ms: int,
+    page_size: int,
+    pages_max: int,
+    max_items: int,
+    progress: Any = None,
+) -> Tuple[Dict[str, Any], Dict[str, Any], List[Dict[str, Any]], List[Dict[str, str]]]:
+    raw = collect_douyin_author_home_raw(
+        input_value=input_value,
+        base_url=base_url,
+        token=token,
+        timeout_ms=timeout_ms,
+        page_size=page_size,
+        pages_max=pages_max,
+        max_items=max_items,
+        progress=progress,
+    )
+    profile, works, missing = adapt_douyin_author_home(raw)
+    return raw, profile, works, missing
+

package/skills/social-media-crawl/scripts/pipelines/douyin_platform_adapter.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+"""Compatibility wrapper for douyin platform adapter."""
+
+from scripts.pipelines.platform_adapters import adapt_douyin_author_home
+
+__all__ = ["adapt_douyin_author_home"]
+

package/skills/{creator-analysis/scripts/author_home/asr → social-media-crawl/scripts/pipelines}/home_asr.py

@@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional, Tuple
 
 from scripts.core.progress_report import ProgressReporter
 from scripts.core.tikomni_common import normalize_text
-from scripts.pipeline.asr.asr_pipeline import (
+from scripts.core.asr_pipeline import (
     clamp_u2_batch_submit_size,
     normalize_media_url,
     run_u2_asr_batch_with_timeout_retry,

package/skills/{creator-analysis/scripts/author_home/adapters → social-media-crawl/scripts/pipelines}/platform_adapters.py

@@ -6,9 +6,15 @@ from __future__ import annotations
 from datetime import datetime
 from typing import Any, Dict, List, Tuple
 
-from scripts.author_home.schema import build_author_profile, build_work_item, validate_author_profile, validate_work_item, validate_works_collection
+from scripts.pipelines.schema import (
+    build_author_profile,
+    build_work_item,
+    validate_author_profile,
+    validate_work_item,
+    validate_works_collection,
+)
 from scripts.core.tikomni_common import deep_find_all, deep_find_first
-from scripts.platform.douyin.select_low_quality_video_url import select_low_quality_video_url
+from scripts.pipelines.select_low_quality_video_url import select_low_quality_video_url
 
 
 def _t(value: Any) -> str: