@genduo/desk 0.0.0-stage → 0.4.0

package/run.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""店铺 Agent 常驻入口（LaunchAgent 调用）。
+
+调度（本机时区；日期窗口按日历日，与 JST 店铺日基本一致）：
+  · 心跳：约每 12 分钟
+  · 访问人数：约每 20 小时拉近 14 天（幂等 upsert）
+  · RPP：约每天检查；拉当月（月初 1–3 日兼拉上月）；勿高频狂拉
+
+用法：
+  python3 run.py           # 常驻循环（进程内 sleep；崩溃由 KeepAlive 拉起）
+  python3 run.py --once    # 跑一轮心跳+到期采集后退出（冒烟）
+  python3 run.py --once --dry-run-ingest
+  python3 run.py --once --skip-collect   # 只心跳
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import signal
+import sys
+import time
+import traceback
+from datetime import date, datetime, timedelta
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+# monorepo pipeline（ads parse）
+ROOT = HERE.parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+import api_client  # noqa: E402
+import collect_access  # noqa: E402
+import collect_ads  # noqa: E402
+import ingest_util  # noqa: E402
+import keychain_util as kc  # noqa: E402
+import local_config as cfg  # noqa: E402
+
+JST = ZoneInfo("Asia/Tokyo")
+LOG_DIR = HERE / "logs"
+STATE_PATH = HERE / "state" / "daemon_state.json"
+
+HEARTBEAT_EVERY = 12 * 60          # 12 min
+ACCESS_EVERY = 20 * 3600           # ~20 h
+ADS_CHECK_EVERY = 20 * 3600        # check ~daily; months_to_pull 决定是否拉
+LOOP_SLEEP = 60                    # wake every 60s
+ACCESS_LOOKBACK_DAYS = 14
+ERR_MAX_LEN = 240
+
+_STOP = False
+
+
+def _setup_log() -> logging.Logger:
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+    logger = logging.getLogger("agent")
+    if logger.handlers:
+        return logger
+    logger.setLevel(logging.INFO)
+    fmt = logging.Formatter("[%(asctime)s] %(levelname)s %(message)s",
+                            datefmt="%Y-%m-%d %H:%M:%S")
+    fh = RotatingFileHandler(LOG_DIR / "agent.log",
+                             maxBytes=2_000_000, backupCount=3,
+                             encoding="utf-8")
+    fh.setFormatter(fmt)
+    sh = logging.StreamHandler(sys.stdout)
+    sh.setFormatter(fmt)
+    logger.addHandler(fh)
+    logger.addHandler(sh)
+    return logger
+
+
+def _on_signal(signum, _frame) -> None:
+    global _STOP
+    _STOP = True
+
+
+def _ago_ok(state: dict, key: str, every: float) -> bool:
+    """True if last success is fresh enough (skip)."""
+    raw = state.get(key)
+    if not raw:
+        return False
+    try:
+        ts = datetime.fromisoformat(raw)
+        if ts.tzinfo is None:
+            ts = ts.replace(tzinfo=JST)
+        return (datetime.now(JST) - ts).total_seconds() < every
+    except (TypeError, ValueError):
+        return False
+
+
+def _set_err(state: dict, msg: str | None) -> None:
+    if msg:
+        state["last_error"] = msg[:ERR_MAX_LEN]
+        state["last_error_at"] = datetime.now(JST).isoformat(timespec="seconds")
+    else:
+        state["last_error"] = None
+
+
+def _do_heartbeat(base: str, token: str, shop: str, state: dict,
+                  log: logging.Logger) -> None:
+    last_err = state.get("last_error")
+    hb = api_client.heartbeat(base, token, shop, cfg.VERSION,
+                              last_error=last_err)
+    agent = (hb.get("agent") or {}) if isinstance(hb, dict) else {}
+    log.info("heartbeat ok shop=%s ver=%s last_seen=%s",
+             agent.get("shop") or shop,
+             agent.get("agent_version"),
+             agent.get("last_seen_at"))
+    state["last_heartbeat"] = datetime.now(JST).isoformat(timespec="seconds")
+    try:
+        jobs = api_client.jobs(base, token, shop, limit=5)
+        n = len((jobs or {}).get("jobs") or []) if isinstance(jobs, dict) else 0
+        state["last_jobs_count"] = n
+        if n:
+            log.info("jobs pending=%s", n)
+    except api_client.ApiError as exc:
+        log.warning("jobs fetch fail HTTP %s", exc.code)
+
+
+def _do_access(base: str, token: str, shop: str, state: dict,
+               log: logging.Logger, *, dry_run: bool) -> None:
+    start, end = collect_access.default_window(ACCESS_LOOKBACK_DAYS)
+    log.info("access due window=%s..%s", start, end)
+    rows = collect_access.collect_access(
+        shop, start, end, log=lambda m: log.info("%s", m))
+    if not rows:
+        log.warning("access: 0 rows")
+        state["last_access_ok"] = datetime.now(JST).isoformat(timespec="seconds")
+        state["last_access_rows"] = 0
+        return
+    tag = f"{start.isoformat().replace('-', '')}-{end.isoformat().replace('-', '')}"
+    body = ingest_util.build_payload(shop, "rms_access", rows, tag=tag)
+    ingest_util.push_ingest(
+        base, token, "rms_access", body, dry_run=dry_run,
+        buffer_on_fail=not dry_run, log=lambda m: log.info("%s", m))
+    state["last_access_ok"] = datetime.now(JST).isoformat(timespec="seconds")
+    state["last_access_rows"] = len(rows)
+    state["last_access_window"] = f"{start}..{end}"
+    _set_err(state, None)
+
+
+def _do_ads(base: str, token: str, shop: str, state: dict,
+            log: logging.Logger, *, dry_run: bool, force: bool) -> None:
+    today = date.today()
+    months = collect_ads.months_to_pull(today)
+    # 若当月已成功且非 force、且非月初兼拉窗口，可跳过
+    last_month = state.get("last_ads_month")
+    if (not force and last_month == months[-1] and today.day > 3
+            and _ago_ok(state, "last_ads_ok", ADS_CHECK_EVERY)):
+        log.info("ads skip (month %s already ok)", last_month)
+        return
+    total = 0
+    for month in months:
+        log.info("ads due month=%s", month)
+        rows = collect_ads.collect_ads(
+            shop, month, log=lambda m: log.info("%s", m))
+        if not rows:
+            log.warning("ads %s: 0 rows", month)
+            continue
+        body = ingest_util.build_payload(
+            shop, "rms_ads", rows, tag=month.replace("-", ""))
+        ingest_util.push_ingest(
+            base, token, "rms_ads", body, dry_run=dry_run,
+            buffer_on_fail=not dry_run, log=lambda m: log.info("%s", m))
+        total += len(rows)
+        state["last_ads_month"] = month
+    state["last_ads_ok"] = datetime.now(JST).isoformat(timespec="seconds")
+    state["last_ads_rows"] = total
+    _set_err(state, None)
+
+
+def run_cycle(base: str, token: str, shop: str, state: dict,
+              log: logging.Logger, *, dry_run: bool, skip_collect: bool,
+              force_collect: bool) -> None:
+    # 1) flush buffers
+    try:
+        summary = ingest_util.flush_buffers(
+            base, token, log=lambda m: log.info("%s", m))
+        if summary.get("tried"):
+            log.info("buffer flush %s", summary)
+    except Exception as exc:  # noqa: BLE001
+        log.warning("buffer flush error: %s", type(exc).__name__)
+
+    # 2) heartbeat
+    if force_collect or not _ago_ok(state, "last_heartbeat", HEARTBEAT_EVERY):
+        try:
+            _do_heartbeat(base, token, shop, state, log)
+        except api_client.ApiError as exc:
+            _set_err(state, f"heartbeat HTTP {exc.code}")
+            log.error("heartbeat fail: %s", exc)
+            raise
+
+    if skip_collect:
+        return
+
+    # 3) access
+    if force_collect or not _ago_ok(state, "last_access_ok", ACCESS_EVERY):
+        try:
+            _do_access(base, token, shop, state, log, dry_run=dry_run)
+        except collect_access.SessionExpiredError as exc:
+            _set_err(state, f"access session: {exc}")
+            log.error("access session expired — re-login required")
+        except Exception as exc:  # noqa: BLE001
+            _set_err(state, f"access: {type(exc).__name__}: {exc}")
+            log.error("access fail: %s: %s", type(exc).__name__, exc)
+            log.debug(traceback.format_exc())
+
+    # 4) ads
+    need_ads = force_collect or not _ago_ok(state, "last_ads_ok", ADS_CHECK_EVERY)
+    if need_ads:
+        try:
+            _do_ads(base, token, shop, state, log,
+                    dry_run=dry_run, force=force_collect)
+        except collect_ads.SessionExpiredError as exc:
+            _set_err(state, f"ads session: {exc}")
+            log.error("ads session expired — re-login required")
+        except Exception as exc:  # noqa: BLE001
+            _set_err(state, f"ads: {type(exc).__name__}: {exc}")
+            log.error("ads fail: %s: %s", type(exc).__name__, exc)
+            log.debug(traceback.format_exc())
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description="dataanalyst shop agent daemon")
+    ap.add_argument("--once", action="store_true",
+                    help="跑一轮后退出（冒烟 / 手动）")
+    ap.add_argument("--dry-run-ingest", action="store_true",
+                    help="ingest 带 dry_run=1，不落库")
+    ap.add_argument("--skip-collect", action="store_true",
+                    help="本轮只心跳，不采集")
+    ap.add_argument("--force-collect", action="store_true",
+                    help="忽略 last_* 间隔，强制采集")
+    args = ap.parse_args(argv)
+
+    log = _setup_log()
+    conf = cfg.load()
+    shop = (conf.get("shop") or "").upper()
+    if not shop:
+        log.error("未配置店铺：请先 dataanalyst-agent login --shop <SHOP>")
+        return 2
+    base = conf.get("base") or cfg.DEFAULT_BASE
+    try:
+        token = kc.get_agent_token()
+    except RuntimeError as exc:
+        log.error("%s", exc)
+        return 2
+
+    signal.signal(signal.SIGTERM, _on_signal)
+    signal.signal(signal.SIGINT, _on_signal)
+
+    state = ingest_util.load_daemon_state(STATE_PATH)
+    state["shop"] = shop
+    state["agent_version"] = cfg.VERSION
+    state["started_at"] = datetime.now(JST).isoformat(timespec="seconds")
+    log.info("agent start shop=%s ver=%s once=%s dry_run=%s",
+             shop, cfg.VERSION, args.once, args.dry_run_ingest)
+
+    # --once：强制本轮采集（除非 skip）
+    force = args.force_collect or args.once
+
+    while not _STOP:
+        try:
+            run_cycle(base, token, shop, state, log,
+                      dry_run=args.dry_run_ingest,
+                      skip_collect=args.skip_collect,
+                      force_collect=force)
+        except api_client.ApiError:
+            # heartbeat 硬失败已记 state；once 模式直接退
+            ingest_util.save_daemon_state(state, STATE_PATH)
+            if args.once:
+                return 1
+        except Exception as exc:  # noqa: BLE001
+            _set_err(state, f"cycle: {type(exc).__name__}: {exc}")
+            log.error("cycle crash: %s: %s", type(exc).__name__, exc)
+            ingest_util.save_daemon_state(state, STATE_PATH)
+            if args.once:
+                return 1
+        else:
+            ingest_util.save_daemon_state(state, STATE_PATH)
+
+        if args.once:
+            log.info("once done last_error=%s", state.get("last_error"))
+            return 0 if not state.get("last_error") else 1
+
+        # 常驻：只在首轮 force；之后按间隔
+        force = False
+        # 睡眠，可被信号打断
+        for _ in range(LOOP_SLEEP):
+            if _STOP:
+                break
+            time.sleep(1)
+
+    log.info("agent stop")
+    ingest_util.save_daemon_state(state, STATE_PATH)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/postinstall.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+'use strict';
+
+const { spawnSync } = require('child_process');
+
+if (process.platform !== 'darwin') {
+  console.warn(
+    '[@genduo/desk] 本包仅支持 macOS（darwin）。当前平台: ' + process.platform
+  );
+  process.exit(0);
+}
+
+const check = spawnSync('python3', ['--version'], { encoding: 'utf8' });
+if (check.error || check.status !== 0) {
+  console.warn(
+    '[@genduo/desk] 未检测到 python3。请先安装：brew install python'
+  );
+  process.exit(0);
+}
+
+console.log(
+  '[@genduo/desk] python3 ok。若要用 --verify-rms：pip3 install playwright && python3 -m playwright install chromium'
+);