@tikomni/skills 0.1.0

package/skills/creator-analysis/scripts/core/tikomni_common.py

@@ -0,0 +1,588 @@
+#!/usr/bin/env python3
+"""Common helpers for local Tikomni Track A scripts."""
+
+import json
+import os
+import re
+import socket
+import threading
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Tuple
+
+DEFAULT_BASE_URL = "https://api.tikomni.com"
+DEFAULT_TIMEOUT_MS = 60000
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
+)
+
+TERMINAL_TASK_STATUS = {"SUCCEEDED", "FAILED", "CANCELED", "CANCELLED"}
+
+DEFAULT_GLOBAL_QPS = 5.0
+DEFAULT_TIMEOUT_RETRY_MAX = 3
+DEFAULT_TIMEOUT_RETRY_BACKOFF_MS = 300
+
+_RATE_LIMIT_LOCK = threading.Lock()
+_NEXT_ALLOWED_TS = 0.0
+
+
+def _parse_env_file(env_file: Optional[str]) -> Dict[str, str]:
+    if not env_file:
+        return {}
+
+    path = Path(env_file)
+    if not path.exists() or not path.is_file():
+        return {}
+
+    parsed: Dict[str, str] = {}
+    with path.open("r", encoding="utf-8") as handle:
+        for raw_line in handle:
+            line = raw_line.strip()
+            if not line or line.startswith("#") or "=" not in line:
+                continue
+            key, value = line.split("=", 1)
+            key = key.strip()
+            value = value.strip().strip('"').strip("'")
+            if key:
+                parsed[key] = value
+    return parsed
+
+
+def get_skills_root() -> Path:
+    """Return the user-facing skills root."""
+    script_path = Path(__file__).resolve()
+    return script_path.parents[3]
+
+
+def get_repo_root() -> Path:
+    """Return repository root for internal development-only relative paths."""
+    return get_skills_root().parent
+
+
+def _resolve_env_file_path(env_file: Optional[str]) -> Path:
+    skills_root = get_skills_root()
+    if not env_file:
+        return (skills_root / ".env").resolve()
+
+    candidate = Path(env_file).expanduser()
+    if not candidate.is_absolute():
+        candidate = skills_root / candidate
+    return candidate.resolve()
+
+
+def _infer_default_env_paths(primary_env_file: Optional[str]) -> Tuple[Path, Path]:
+    script_path = Path(__file__).resolve()
+    skill_root = script_path.parents[2]
+
+    workspace_env = _resolve_env_file_path(primary_env_file)
+    local_env = (skill_root / ".env.local").resolve()
+    return workspace_env, local_env
+
+
+def bootstrap_runtime_env(
+    primary_env_file: Optional[str] = None,
+    allow_process_env: bool = False,
+) -> Dict[str, Any]:
+    """Load env values from .env + .env.local with deterministic priority.
+
+    Default priority (highest -> lowest): .env.local > .env
+    Optional (when allow_process_env=True): process env > .env.local > .env
+    """
+
+    process_env = dict(os.environ)
+    workspace_env_path, local_env_path = _infer_default_env_paths(primary_env_file)
+
+    base_values = _parse_env_file(str(workspace_env_path))
+    local_values = _parse_env_file(str(local_env_path))
+
+    merged: Dict[str, str] = {}
+    merged.update(base_values)
+    merged.update(local_values)
+    if allow_process_env:
+        merged.update(process_env)
+
+    os.environ.update(merged)
+
+    loaded_files: List[str] = []
+    if workspace_env_path.exists():
+        loaded_files.append(str(workspace_env_path))
+    if local_env_path.exists():
+        loaded_files.append(str(local_env_path))
+
+    key_source: Dict[str, str] = {}
+    source_chain: Dict[str, List[str]] = {}
+    all_keys = set(base_values.keys()) | set(local_values.keys())
+    if allow_process_env:
+        all_keys |= set(process_env.keys())
+
+    for key in all_keys:
+        chain: List[str] = []
+        if key in base_values:
+            chain.append(".env")
+        if key in local_values:
+            chain.append(".env.local")
+        if allow_process_env and key in process_env:
+            chain.append("process_env")
+        source_chain[key] = chain
+
+        if allow_process_env and key in process_env:
+            key_source[key] = "process_env"
+        elif key in local_values:
+            key_source[key] = ".env.local"
+        elif key in base_values:
+            key_source[key] = ".env"
+
+    priority = [".env.local", ".env"]
+    if allow_process_env:
+        priority.insert(0, "process_env")
+
+    return {
+        "skills_root": str(get_skills_root()),
+        "repo_root": str(get_repo_root()),
+        "workspace_env": str(workspace_env_path),
+        "local_env": str(local_env_path),
+        "loaded_files": loaded_files,
+        "key_source": key_source,
+        "source_chain": source_chain,
+        "priority": priority,
+        "allow_process_env": bool(allow_process_env),
+        "effective_env": merged,
+    }
+
+
+def load_env_file(env_file: Optional[str], allow_process_env: bool = False) -> None:
+    """Backward-compatible wrapper. Prefer bootstrap_runtime_env."""
+    bootstrap_runtime_env(primary_env_file=env_file, allow_process_env=allow_process_env)
+
+
+def resolve_runtime(
+    env_file: Optional[str],
+    api_key_env: str,
+    base_url: Optional[str],
+    timeout_ms: Optional[int],
+    allow_process_env: bool = False,
+) -> Dict[str, Any]:
+    bootstrap_info = bootstrap_runtime_env(
+        primary_env_file=env_file,
+        allow_process_env=allow_process_env,
+    )
+    effective_env = bootstrap_info.get("effective_env", {}) if isinstance(bootstrap_info, dict) else {}
+
+    token = str(effective_env.get(api_key_env, "") or "").strip()
+    if not token and allow_process_env:
+        token = os.getenv(api_key_env, "").strip()
+    if not token:
+        raise ValueError(
+            f"missing_api_key:{api_key_env} (set in .env/.env.local or pass --allow-process-env)"
+        )
+
+    resolved_base_url = str(base_url or effective_env.get("TIKOMNI_BASE_URL") or DEFAULT_BASE_URL).strip().rstrip("/")
+    timeout_raw = timeout_ms if timeout_ms is not None else effective_env.get("TIKOMNI_TIMEOUT_MS")
+    try:
+        resolved_timeout_ms = int(timeout_raw or DEFAULT_TIMEOUT_MS)
+    except Exception:
+        resolved_timeout_ms = DEFAULT_TIMEOUT_MS
+
+    return {
+        "token": token,
+        "base_url": resolved_base_url,
+        "timeout_ms": resolved_timeout_ms,
+        "env_bootstrap": {
+            "loaded_files": bootstrap_info.get("loaded_files", []),
+            "api_key_source": bootstrap_info.get("key_source", {}).get(api_key_env, "unknown"),
+            "priority": bootstrap_info.get("priority", [".env.local", ".env"]),
+            "allow_process_env": bool(allow_process_env),
+        },
+    }
+
+
+def safe_json_loads(text: str) -> Any:
+    try:
+        return json.loads(text)
+    except Exception:
+        return {"_raw": text}
+
+
+def _build_url(base_url: str, path: str, params: Optional[Dict[str, Any]]) -> str:
+    url = f"{base_url}{path}"
+    if params:
+        normalized = {}
+        for key, value in params.items():
+            if value is None:
+                continue
+            normalized[key] = value
+        if normalized:
+            url = f"{url}?{urllib.parse.urlencode(normalized)}"
+    return url
+
+
+def _resolve_global_qps() -> float:
+    raw = str(os.getenv("TIKOMNI_GLOBAL_QPS", DEFAULT_GLOBAL_QPS)).strip()
+    try:
+        qps = float(raw)
+    except Exception:
+        qps = DEFAULT_GLOBAL_QPS
+    return max(0.1, qps)
+
+
+def _resolve_timeout_retry_max() -> int:
+    raw = str(os.getenv("TIKOMNI_TIMEOUT_RETRY_MAX", DEFAULT_TIMEOUT_RETRY_MAX)).strip()
+    try:
+        retries = int(raw)
+    except Exception:
+        retries = DEFAULT_TIMEOUT_RETRY_MAX
+    return max(0, min(retries, 8))
+
+
+def _resolve_timeout_retry_backoff_ms() -> int:
+    raw = str(os.getenv("TIKOMNI_TIMEOUT_RETRY_BACKOFF_MS", DEFAULT_TIMEOUT_RETRY_BACKOFF_MS)).strip()
+    try:
+        backoff = int(raw)
+    except Exception:
+        backoff = DEFAULT_TIMEOUT_RETRY_BACKOFF_MS
+    return max(0, min(backoff, 5000))
+
+
+def _wait_rate_limit_slot(qps: float) -> int:
+    global _NEXT_ALLOWED_TS
+    interval_sec = 1.0 / max(qps, 0.1)
+    now = time.monotonic()
+    with _RATE_LIMIT_LOCK:
+        scheduled = max(now, _NEXT_ALLOWED_TS)
+        wait_sec = max(0.0, scheduled - now)
+        _NEXT_ALLOWED_TS = scheduled + interval_sec
+
+    if wait_sec > 0:
+        time.sleep(wait_sec)
+    return int(wait_sec * 1000)
+
+
+def _is_timeout_like_failure(status_code: Optional[int], error_reason: Optional[str]) -> bool:
+    if isinstance(status_code, (int, float)) and int(status_code) in {408, 429, 502, 503, 504}:
+        return True
+
+    reason = str(error_reason or "").strip().upper()
+    if not reason:
+        return False
+    timeout_tokens = ["TIMEOUT", "TIMED_OUT", "UPSTREAM_TIMEOUT", "DEADLINE_EXCEEDED"]
+    return any(token in reason for token in timeout_tokens)
+
+
+def _execute_json_request(request: urllib.request.Request, timeout_ms: int, url: str) -> Tuple[Dict[str, Any], bool]:
+    status_code: Optional[int] = None
+    raw_text = ""
+    payload: Any = {}
+
+    try:
+        with urllib.request.urlopen(request, timeout=max(timeout_ms / 1000.0, 1.0)) as response:
+            status_code = response.getcode()
+            raw_text = response.read().decode("utf-8", errors="replace")
+            payload = safe_json_loads(raw_text)
+    except urllib.error.HTTPError as error:
+        status_code = error.code
+        raw_text = error.read().decode("utf-8", errors="replace")
+        payload = safe_json_loads(raw_text)
+    except urllib.error.URLError as error:
+        reason_obj = getattr(error, "reason", error)
+        reason_text = normalize_text(reason_obj)
+        timeout_like = isinstance(reason_obj, socket.timeout) or "timeout" in reason_text.lower()
+        error_reason = "network_error:timeout" if timeout_like else f"network_error:{reason_text or 'unknown'}"
+        return {
+            "ok": False,
+            "status_code": None,
+            "data": {},
+            "request_id": None,
+            "error_reason": error_reason,
+            "raw_text": str(error),
+            "url": url,
+        }, timeout_like
+    except (TimeoutError, socket.timeout) as error:
+        return {
+            "ok": False,
+            "status_code": None,
+            "data": {},
+            "request_id": None,
+            "error_reason": "network_error:timeout",
+            "raw_text": str(error),
+            "url": url,
+        }, True
+
+    request_id = extract_request_id(payload)
+    ok = bool(status_code is not None and 200 <= status_code < 300)
+    error_reason = None if ok else extract_error_reason(payload, status_code)
+    timeout_like = _is_timeout_like_failure(status_code, error_reason)
+
+    return {
+        "ok": ok,
+        "status_code": status_code,
+        "data": payload,
+        "request_id": request_id,
+        "error_reason": error_reason,
+        "raw_text": raw_text,
+        "url": url,
+    }, timeout_like
+
+
+def call_json_api(
+    *,
+    base_url: str,
+    path: str,
+    token: str,
+    method: str,
+    timeout_ms: int,
+    params: Optional[Dict[str, Any]] = None,
+    body: Optional[Dict[str, Any]] = None,
+    extra_headers: Optional[Dict[str, str]] = None,
+) -> Dict[str, Any]:
+    url = _build_url(base_url, path, params)
+    headers = {
+        "Authorization": f"Bearer {token}",
+        "Accept": "application/json",
+        "Content-Type": "application/json",
+        "User-Agent": os.getenv("TIKOMNI_HTTP_USER_AGENT", DEFAULT_USER_AGENT),
+        "Accept-Language": os.getenv("TIKOMNI_HTTP_ACCEPT_LANGUAGE", "zh-CN,zh;q=0.9,en;q=0.8"),
+    }
+    if extra_headers:
+        headers.update(extra_headers)
+
+    data: Optional[bytes] = None
+    if body is not None:
+        data = json.dumps(body, ensure_ascii=False).encode("utf-8")
+
+    request = urllib.request.Request(url=url, data=data, headers=headers, method=method.upper())
+
+    qps = _resolve_global_qps()
+    timeout_retry_max = _resolve_timeout_retry_max()
+    retry_backoff_ms = _resolve_timeout_retry_backoff_ms()
+
+    cumulative_wait_ms = 0
+    last_result: Dict[str, Any] = {
+        "ok": False,
+        "status_code": None,
+        "data": {},
+        "request_id": None,
+        "error_reason": "unknown_error",
+        "raw_text": "",
+        "url": url,
+    }
+
+    max_attempts = 1 + timeout_retry_max
+    for attempt in range(1, max_attempts + 1):
+        if attempt > 1 and retry_backoff_ms > 0:
+            sleep_ms = retry_backoff_ms * (2 ** (attempt - 2))
+            time.sleep(sleep_ms / 1000.0)
+
+        cumulative_wait_ms += _wait_rate_limit_slot(qps)
+        response, timeout_like = _execute_json_request(request=request, timeout_ms=timeout_ms, url=url)
+        response["rate_limit_wait_ms"] = cumulative_wait_ms
+        response["retry_attempt"] = max(0, attempt - 1)
+        response["fallback_trigger_reason"] = None
+        response["timeout_retry_max"] = timeout_retry_max
+
+        if response.get("ok"):
+            response["timeout_retry_exhausted"] = False
+            return response
+
+        last_result = response
+        if timeout_like and attempt < max_attempts:
+            continue
+
+        response["timeout_retry_exhausted"] = bool(timeout_like and attempt >= max_attempts)
+        return response
+
+    last_result["timeout_retry_exhausted"] = True
+    return last_result
+
+
+def deep_find_first(payload: Any, keys: Iterable[str]) -> Optional[Any]:
+    key_set = {str(key) for key in keys}
+
+    def _walk(node: Any) -> Optional[Any]:
+        if isinstance(node, dict):
+            for key, value in node.items():
+                if key in key_set and value not in (None, "", [], {}):
+                    return value
+            for value in node.values():
+                hit = _walk(value)
+                if hit not in (None, "", [], {}):
+                    return hit
+        elif isinstance(node, list):
+            for item in node:
+                hit = _walk(item)
+                if hit not in (None, "", [], {}):
+                    return hit
+        return None
+
+    return _walk(payload)
+
+
+def deep_find_all(payload: Any, keys: Iterable[str]) -> List[Any]:
+    key_set = {str(key) for key in keys}
+    results: List[Any] = []
+
+    def _walk(node: Any) -> None:
+        if isinstance(node, dict):
+            for key, value in node.items():
+                if key in key_set and value not in (None, "", [], {}):
+                    results.append(value)
+                _walk(value)
+        elif isinstance(node, list):
+            for item in node:
+                _walk(item)
+
+    _walk(payload)
+    return results
+
+
+def deep_collect_strings(payload: Any, max_items: int = 20) -> List[str]:
+    results: List[str] = []
+
+    def _walk(node: Any) -> None:
+        if len(results) >= max_items:
+            return
+        if isinstance(node, str):
+            text = normalize_text(node)
+            if text:
+                results.append(text)
+        elif isinstance(node, dict):
+            for value in node.values():
+                _walk(value)
+        elif isinstance(node, list):
+            for item in node:
+                _walk(item)
+
+    _walk(payload)
+    return results
+
+
+def extract_request_id(payload: Any) -> Optional[str]:
+    value = deep_find_first(payload, ["request_id"])
+    if value is None:
+        return None
+    return str(value)
+
+
+def extract_error_reason(payload: Any, status_code: Optional[int] = None) -> str:
+    if isinstance(payload, dict):
+        error = payload.get("error")
+        if isinstance(error, dict):
+            code = error.get("code")
+            message = error.get("message")
+            if code:
+                return str(code)
+            if message:
+                return normalize_text(str(message)) or "unknown_error"
+
+        code = payload.get("code")
+        if code not in (None, 0, 200, "0", "200", "SUCCESS", "success"):
+            return str(code)
+
+        message = payload.get("message")
+        if isinstance(message, str) and message.strip() and message.strip().lower() not in {"ok", "success"}:
+            return normalize_text(message)
+
+    if status_code is not None:
+        return f"http_{status_code}"
+    return "unknown_error"
+
+
+def normalize_text(value: Any) -> str:
+    if value is None:
+        return ""
+    text = str(value)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+def extract_task_id(payload: Any) -> Optional[str]:
+    task_id = deep_find_first(payload, ["task_id", "id"])
+    if task_id is None:
+        return None
+    return str(task_id)
+
+
+def extract_task_status(payload: Any) -> str:
+    status = deep_find_first(payload, ["status", "task_status", "state"])
+    if status is None:
+        return ""
+    return normalize_text(status).upper()
+
+
+def is_terminal_status(status: str) -> bool:
+    return status.upper() in TERMINAL_TASK_STATUS
+
+
+def extract_transcript_text(payload: Any) -> str:
+    candidates = deep_find_all(
+        payload,
+        [
+            "text",
+            "transcript",
+            "transcription",
+            "content",
+            "raw_content",
+            "result",
+            "sentence",
+            "paragraph",
+        ],
+    )
+
+    texts: List[str] = []
+    for candidate in candidates:
+        if isinstance(candidate, str):
+            cleaned = normalize_text(candidate)
+            if cleaned:
+                texts.append(cleaned)
+        elif isinstance(candidate, list):
+            for item in candidate:
+                if isinstance(item, str):
+                    cleaned = normalize_text(item)
+                    if cleaned:
+                        texts.append(cleaned)
+                elif isinstance(item, dict):
+                    maybe = deep_find_first(item, ["text", "sentence", "content"])
+                    if isinstance(maybe, str):
+                        cleaned = normalize_text(maybe)
+                        if cleaned:
+                            texts.append(cleaned)
+
+    merged = "\n".join(dict.fromkeys(texts))
+    return merged.strip()
+
+
+def summarize_content(raw_content: str, source: str) -> Dict[str, Any]:
+    clean = normalize_text(raw_content)
+    if not clean:
+        return {
+            "summary": "",
+            "insights": [
+                f"source={source}",
+                "no_text_extracted",
+            ],
+        }
+
+    clipped = clean[:180]
+    if len(clean) > 180:
+        clipped = f"{clipped}..."
+
+    insights = [
+        f"source={source}",
+        f"char_len={len(clean)}",
+    ]
+    return {
+        "summary": clipped,
+        "insights": insights,
+    }
+
+
+def read_json_file(path: str) -> Dict[str, Any]:
+    with open(path, "r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def write_json_stdout(payload: Dict[str, Any]) -> None:
+    print(json.dumps(payload, ensure_ascii=False, indent=2))