@genduo/desk 0.0.0-stage → 0.5.0

package/rms_browser_login.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""R-Login 浏览器登录（Playwright）并保存会话。
+
+NOTE (npm / shop CLI):
+  源真相仍是 monorepo `tools/rms_browser_login.py`（VPS 路径也用那份）。
+  本文件是 collector-agent 的 vendored 副本，供 `npm i -g dataanalyst-agent`
+  后门店机无 monorepo 也能跑 `--verify-rms`。打包前若源文件有变，请同步拷贝；
+  勿再读 dataanalyst/rms/* Keychain；JDY_MCP 见 jdy_secrets.py（与销量系统一致）。
+
+Rakuten 已把 R-Login 切到 Rakuten 会员 SSO：
+  1. glogin 填 R-Login ID / 密码（第一组）；
+  2. Rakuten 会员填邮箱 / 密码（第二组）；
+  3. R-Login 提示页「次へ」→ RMS 主菜单「RMSを利用します」。
+
+凭据从 env 文件读，不打印。会话保存为 600 的 storage_state。
+简道云 JDY_MCP / API Key 逻辑见 collector-agent/jdy_secrets.py（与销量计算系统一致）。
+
+用法：
+    # VPS / env 文件
+    PLAYWRIGHT_BROWSERS_PATH=/opt/ms-playwright \\
+    python tools/rms_browser_login.py --shop 7RT2 \\
+      --env-file /etc/dataanalyst/rms.env \\
+      --out /etc/dataanalyst/rms_session/7RT2.json
+
+    # 本机：Keychain 仅 JDY_MCP；RMS 凭据从简道云密码表取
+    python tools/rms_browser_login.py --shop 7RT2 --from-jdy \\
+      --out tools/collector-agent/state/7RT2.json
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+# Prefer existing PLAYWRIGHT_BROWSERS_PATH; only default to VPS path if present.
+_vps_browsers = "/opt/ms-playwright"
+if "PLAYWRIGHT_BROWSERS_PATH" not in os.environ and Path(_vps_browsers).is_dir():
+    os.environ["PLAYWRIGHT_BROWSERS_PATH"] = _vps_browsers
+
+from playwright.sync_api import TimeoutError as PWTimeout  # noqa: E402
+from playwright.sync_api import sync_playwright  # noqa: E402
+
+
+def load_env(path: str) -> dict:
+    out = {}
+    for line in Path(path).read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if line and not line.startswith("#") and "=" in line:
+            k, v = line.split("=", 1)
+            out[k.strip()] = v.strip()
+    return out
+
+
+
+
+def load_from_keychain(shop: str) -> dict:
+    """已废弃：不再读 dataanalyst/rms/* Keychain。请用 --env-file 或环境变量。"""
+    raise SystemExit(
+        "已废弃 --from-keychain（dataanalyst/rms/*）。"
+        "请使用 --env-file 或环境变量 RMS_<SHOP>_LOGIN_ID/PASSWORD/SHOP_USER/SHOP_PASSWORD。"
+    )
+
+
+
+
+def _try_click(page, role: str, name: str, timeout: int = 4000) -> bool:
+    try:
+        page.get_by_role(role, name=name).first.click(timeout=timeout)
+        return True
+    except PWTimeout:
+        return False
+
+
+def login(shop: str, env: dict, out: str, headless: bool = True,
+          screenshot: str | None = None) -> None:
+    prefix = f"RMS_{shop.upper()}_"
+    login_id = env.get(prefix + "LOGIN_ID")
+    password = env.get(prefix + "PASSWORD")
+    sso_user = env.get(prefix + "SHOP_USER")
+    sso_pass = env.get(prefix + "SHOP_PASSWORD")
+    missing = [k for k, v in {
+        "LOGIN_ID": login_id, "PASSWORD": password,
+        "SHOP_USER": sso_user, "SHOP_PASSWORD": sso_pass,
+    }.items() if not v]
+    if missing:
+        raise SystemExit(f"{shop} 缺少凭据: {prefix}{','.join(missing)}")
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=headless,
+                                    args=["--no-sandbox", "--disable-dev-shm-usage"])
+        ctx = browser.new_context(locale="ja-JP", viewport={"width": 1440, "height": 1000},
+                                  user_agent=("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                                              "AppleWebKit/537.36 (KHTML, like Gecko) "
+                                              "Chrome/124.0.0.0 Safari/537.36"))
+        page = ctx.new_page()
+        page.goto("https://glogin.rms.rakuten.co.jp/?sp_id=1",
+                  wait_until="domcontentloaded", timeout=60000)
+        page.fill("#rlogin-username-ja", login_id)
+        page.fill("#rlogin-password-ja", password)
+        page.evaluate("document.querySelector('button[name=submit]')?.click()")
+        page.wait_for_timeout(3000)
+
+        page.fill("input[name=username]", sso_user)
+        page.press("input[name=username]", "Enter")
+        page.wait_for_timeout(3000)
+        page.fill("input[name=password]", sso_pass)
+        page.press("input[name=password]", "Enter")
+        page.wait_for_timeout(5000)
+
+        _try_click(page, "button", "次へ")
+        page.wait_for_timeout(2500)
+        try:
+            page.get_by_text("RMSを利用します").click(timeout=5000)
+        except PWTimeout:
+            pass
+        page.wait_for_timeout(3000)
+
+        url = page.url
+        title = page.title()
+        print("url", url)
+        print("title", title)
+        body_snip = ""
+        try:
+            body_snip = page.inner_text("body")[:500]
+        except Exception:
+            pass
+        # Shop name check (no secrets)
+        if "IORI" in body_snip or "iori" in body_snip.lower():
+            print("shop_hint", "IORI living (matched)")
+        else:
+            # try common shop name selectors without dumping full body
+            for sel in ["text=IORI living", "text=IORI"]:
+                try:
+                    if page.locator(sel).count():
+                        print("shop_hint", "IORI living (locator)")
+                        break
+                except Exception:
+                    pass
+            else:
+                print("shop_hint", "not found in first 500 chars of body")
+
+        if "mainmenu.rms.rakuten.co.jp" not in url:
+            if screenshot:
+                Path(screenshot).parent.mkdir(parents=True, exist_ok=True)
+                page.screenshot(path=screenshot, full_page=True)
+                print("screenshot", screenshot)
+            raise SystemExit("未进入 RMS 主菜单，可能遇到 2FA/验证码；请查看截图")
+
+        Path(out).parent.mkdir(parents=True, exist_ok=True)
+        ctx.storage_state(path=out)
+        os.chmod(out, 0o600)
+        print("saved", out)
+        if screenshot:
+            Path(screenshot).parent.mkdir(parents=True, exist_ok=True)
+            page.screenshot(path=screenshot, full_page=False)
+            os.chmod(screenshot, 0o600)
+            print("screenshot", screenshot)
+        browser.close()
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="R-Login 浏览器登录并保存会话")
+    ap.add_argument("--shop", required=True)
+    ap.add_argument("--env-file", default="/etc/dataanalyst/rms.env")
+    ap.add_argument("--from-keychain", action="store_true",
+                    help="已废弃（原 dataanalyst/rms/* Keychain）")
+    ap.add_argument("--out", help="默认 tools/collector-agent/state/<shop>.json（本机）")
+    ap.add_argument("--screenshot", help="可选截图路径（不含密钥）")
+    ap.add_argument("--headed", action="store_true")
+    a = ap.parse_args()
+    out = a.out or f"tools/collector-agent/state/{a.shop.upper()}.json"
+    try:
+        if a.from_keychain:
+            env = load_from_keychain(a.shop)
+            # Confirm accounts only (no passwords)
+            prefix = f"RMS_{a.shop.upper()}_"
+            print("rlogin_id_set", bool(env.get(prefix + "LOGIN_ID")))
+            print("sso_user_set", bool(env.get(prefix + "SHOP_USER")))
+        else:
+            print("cred_source", "env-file")
+            env = load_env(a.env_file)
+        login(a.shop, env, out, headless=not a.headed, screenshot=a.screenshot)
+    except Exception as e:                              # noqa: BLE001
+        print(f"登录失败: {type(e).__name__}: {e}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/run.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""店铺 Agent 常驻入口（LaunchAgent 调用）。
+
+调度（本机时区；日期窗口按日历日，与 JST 店铺日基本一致）：
+  · 心跳：约每 12 分钟
+  · 访问人数：约每 20 小时拉近 14 天（幂等 upsert）
+  · RPP：约每天检查；拉当月（月初 1–3 日兼拉上月）；勿高频狂拉
+
+用法：
+  python3 run.py           # 常驻循环（进程内 sleep；崩溃由 KeepAlive 拉起）
+  python3 run.py --once    # 跑一轮心跳+到期采集后退出（冒烟）
+  python3 run.py --once --dry-run-ingest
+  python3 run.py --once --skip-collect   # 只心跳
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+import signal
+import sys
+import time
+import traceback
+from datetime import date, datetime, timedelta
+from logging.handlers import RotatingFileHandler
+from pathlib import Path
+from zoneinfo import ZoneInfo
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+# monorepo pipeline（ads parse）
+ROOT = HERE.parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+import api_client  # noqa: E402
+import collect_access  # noqa: E402
+import collect_ads  # noqa: E402
+import ingest_util  # noqa: E402
+import keychain_util as kc  # noqa: E402
+import local_config as cfg  # noqa: E402
+
+JST = ZoneInfo("Asia/Tokyo")
+LOG_DIR = HERE / "logs"
+STATE_PATH = HERE / "state" / "daemon_state.json"
+
+HEARTBEAT_EVERY = 12 * 60          # 12 min
+ACCESS_EVERY = 20 * 3600           # ~20 h
+ADS_CHECK_EVERY = 20 * 3600        # check ~daily; months_to_pull 决定是否拉
+LOOP_SLEEP = 60                    # wake every 60s
+ACCESS_LOOKBACK_DAYS = 14
+ERR_MAX_LEN = 240
+
+_STOP = False
+
+
+def _setup_log() -> logging.Logger:
+    LOG_DIR.mkdir(parents=True, exist_ok=True)
+    logger = logging.getLogger("agent")
+    if logger.handlers:
+        return logger
+    logger.setLevel(logging.INFO)
+    fmt = logging.Formatter("[%(asctime)s] %(levelname)s %(message)s",
+                            datefmt="%Y-%m-%d %H:%M:%S")
+    fh = RotatingFileHandler(LOG_DIR / "agent.log",
+                             maxBytes=2_000_000, backupCount=3,
+                             encoding="utf-8")
+    fh.setFormatter(fmt)
+    sh = logging.StreamHandler(sys.stdout)
+    sh.setFormatter(fmt)
+    logger.addHandler(fh)
+    logger.addHandler(sh)
+    return logger
+
+
+def _on_signal(signum, _frame) -> None:
+    global _STOP
+    _STOP = True
+
+
+def _ago_ok(state: dict, key: str, every: float) -> bool:
+    """True if last success is fresh enough (skip)."""
+    raw = state.get(key)
+    if not raw:
+        return False
+    try:
+        ts = datetime.fromisoformat(raw)
+        if ts.tzinfo is None:
+            ts = ts.replace(tzinfo=JST)
+        return (datetime.now(JST) - ts).total_seconds() < every
+    except (TypeError, ValueError):
+        return False
+
+
+def _set_err(state: dict, msg: str | None) -> None:
+    if msg:
+        state["last_error"] = msg[:ERR_MAX_LEN]
+        state["last_error_at"] = datetime.now(JST).isoformat(timespec="seconds")
+    else:
+        state["last_error"] = None
+
+
+def _do_heartbeat(base: str, token: str, shop: str, state: dict,
+                  log: logging.Logger) -> None:
+    last_err = state.get("last_error")
+    hb = api_client.heartbeat(base, token, shop, cfg.VERSION,
+                              last_error=last_err)
+    agent = (hb.get("agent") or {}) if isinstance(hb, dict) else {}
+    log.info("heartbeat ok shop=%s ver=%s last_seen=%s",
+             agent.get("shop") or shop,
+             agent.get("agent_version"),
+             agent.get("last_seen_at"))
+    state["last_heartbeat"] = datetime.now(JST).isoformat(timespec="seconds")
+    try:
+        jobs = api_client.jobs(base, token, shop, limit=5)
+        n = len((jobs or {}).get("jobs") or []) if isinstance(jobs, dict) else 0
+        state["last_jobs_count"] = n
+        if n:
+            log.info("jobs pending=%s", n)
+    except api_client.ApiError as exc:
+        log.warning("jobs fetch fail HTTP %s", exc.code)
+
+
+def _do_access(base: str, token: str, shop: str, state: dict,
+               log: logging.Logger, *, dry_run: bool) -> None:
+    start, end = collect_access.default_window(ACCESS_LOOKBACK_DAYS)
+    log.info("access due window=%s..%s", start, end)
+    rows = collect_access.collect_access(
+        shop, start, end, log=lambda m: log.info("%s", m))
+    if not rows:
+        log.warning("access: 0 rows")
+        state["last_access_ok"] = datetime.now(JST).isoformat(timespec="seconds")
+        state["last_access_rows"] = 0
+        return
+    tag = f"{start.isoformat().replace('-', '')}-{end.isoformat().replace('-', '')}"
+    body = ingest_util.build_payload(shop, "rms_access", rows, tag=tag)
+    ingest_util.push_ingest(
+        base, token, "rms_access", body, dry_run=dry_run,
+        buffer_on_fail=not dry_run, log=lambda m: log.info("%s", m))
+    state["last_access_ok"] = datetime.now(JST).isoformat(timespec="seconds")
+    state["last_access_rows"] = len(rows)
+    state["last_access_window"] = f"{start}..{end}"
+    _set_err(state, None)
+
+
+def _do_ads(base: str, token: str, shop: str, state: dict,
+            log: logging.Logger, *, dry_run: bool, force: bool) -> None:
+    today = date.today()
+    months = collect_ads.months_to_pull(today)
+    # 若当月已成功且非 force、且非月初兼拉窗口，可跳过
+    last_month = state.get("last_ads_month")
+    if (not force and last_month == months[-1] and today.day > 3
+            and _ago_ok(state, "last_ads_ok", ADS_CHECK_EVERY)):
+        log.info("ads skip (month %s already ok)", last_month)
+        return
+    total = 0
+    for month in months:
+        log.info("ads due month=%s", month)
+        rows = collect_ads.collect_ads(
+            shop, month, log=lambda m: log.info("%s", m))
+        if not rows:
+            log.warning("ads %s: 0 rows", month)
+            continue
+        body = ingest_util.build_payload(
+            shop, "rms_ads", rows, tag=month.replace("-", ""))
+        ingest_util.push_ingest(
+            base, token, "rms_ads", body, dry_run=dry_run,
+            buffer_on_fail=not dry_run, log=lambda m: log.info("%s", m))
+        total += len(rows)
+        state["last_ads_month"] = month
+    state["last_ads_ok"] = datetime.now(JST).isoformat(timespec="seconds")
+    state["last_ads_rows"] = total
+    _set_err(state, None)
+
+
+def run_cycle(base: str, token: str, shop: str, state: dict,
+              log: logging.Logger, *, dry_run: bool, skip_collect: bool,
+              force_collect: bool) -> None:
+    # 1) flush buffers
+    try:
+        summary = ingest_util.flush_buffers(
+            base, token, log=lambda m: log.info("%s", m))
+        if summary.get("tried"):
+            log.info("buffer flush %s", summary)
+    except Exception as exc:  # noqa: BLE001
+        log.warning("buffer flush error: %s", type(exc).__name__)
+
+    # 2) heartbeat
+    if force_collect or not _ago_ok(state, "last_heartbeat", HEARTBEAT_EVERY):
+        try:
+            _do_heartbeat(base, token, shop, state, log)
+        except api_client.ApiError as exc:
+            _set_err(state, f"heartbeat HTTP {exc.code}")
+            log.error("heartbeat fail: %s", exc)
+            raise
+
+    if skip_collect:
+        return
+
+    # 3) access
+    if force_collect or not _ago_ok(state, "last_access_ok", ACCESS_EVERY):
+        try:
+            _do_access(base, token, shop, state, log, dry_run=dry_run)
+        except collect_access.SessionExpiredError as exc:
+            _set_err(state, f"access session: {exc}")
+            log.error("access session expired — re-login required")
+        except Exception as exc:  # noqa: BLE001
+            _set_err(state, f"access: {type(exc).__name__}: {exc}")
+            log.error("access fail: %s: %s", type(exc).__name__, exc)
+            log.debug(traceback.format_exc())
+
+    # 4) ads
+    need_ads = force_collect or not _ago_ok(state, "last_ads_ok", ADS_CHECK_EVERY)
+    if need_ads:
+        try:
+            _do_ads(base, token, shop, state, log,
+                    dry_run=dry_run, force=force_collect)
+        except collect_ads.SessionExpiredError as exc:
+            _set_err(state, f"ads session: {exc}")
+            log.error("ads session expired — re-login required")
+        except Exception as exc:  # noqa: BLE001
+            _set_err(state, f"ads: {type(exc).__name__}: {exc}")
+            log.error("ads fail: %s: %s", type(exc).__name__, exc)
+            log.debug(traceback.format_exc())
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description="dataanalyst shop agent daemon")
+    ap.add_argument("--once", action="store_true",
+                    help="跑一轮后退出（冒烟 / 手动）")
+    ap.add_argument("--dry-run-ingest", action="store_true",
+                    help="ingest 带 dry_run=1，不落库")
+    ap.add_argument("--skip-collect", action="store_true",
+                    help="本轮只心跳，不采集")
+    ap.add_argument("--force-collect", action="store_true",
+                    help="忽略 last_* 间隔，强制采集")
+    args = ap.parse_args(argv)
+
+    log = _setup_log()
+    conf = cfg.load()
+    shop = (conf.get("shop") or "").upper()
+    if not shop:
+        log.error("未配置店铺：请先 dataanalyst-agent login --shop <SHOP>")
+        return 2
+    base = conf.get("base") or cfg.DEFAULT_BASE
+    try:
+        token = kc.get_agent_token(shop)
+    except RuntimeError as exc:
+        log.error("%s", exc)
+        return 2
+
+    signal.signal(signal.SIGTERM, _on_signal)
+    signal.signal(signal.SIGINT, _on_signal)
+
+    state = ingest_util.load_daemon_state(STATE_PATH)
+    state["shop"] = shop
+    state["agent_version"] = cfg.VERSION
+    state["started_at"] = datetime.now(JST).isoformat(timespec="seconds")
+    log.info("agent start shop=%s ver=%s once=%s dry_run=%s",
+             shop, cfg.VERSION, args.once, args.dry_run_ingest)
+
+    # --once：强制本轮采集（除非 skip）
+    force = args.force_collect or args.once
+
+    while not _STOP:
+        try:
+            run_cycle(base, token, shop, state, log,
+                      dry_run=args.dry_run_ingest,
+                      skip_collect=args.skip_collect,
+                      force_collect=force)
+        except api_client.ApiError:
+            # heartbeat 硬失败已记 state；once 模式直接退
+            ingest_util.save_daemon_state(state, STATE_PATH)
+            if args.once:
+                return 1
+        except Exception as exc:  # noqa: BLE001
+            _set_err(state, f"cycle: {type(exc).__name__}: {exc}")
+            log.error("cycle crash: %s: %s", type(exc).__name__, exc)
+            ingest_util.save_daemon_state(state, STATE_PATH)
+            if args.once:
+                return 1
+        else:
+            ingest_util.save_daemon_state(state, STATE_PATH)
+
+        if args.once:
+            log.info("once done last_error=%s", state.get("last_error"))
+            return 0 if not state.get("last_error") else 1
+
+        # 常驻：只在首轮 force；之后按间隔
+        force = False
+        # 睡眠，可被信号打断
+        for _ in range(LOOP_SLEEP):
+            if _STOP:
+                break
+            time.sleep(1)
+
+    log.info("agent stop")
+    ingest_util.save_daemon_state(state, STATE_PATH)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/postinstall.js

@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+'use strict';
+
+const { spawnSync } = require('child_process');
+
+if (process.platform !== 'darwin') {
+  console.warn(
+    '[@genduo/desk] 本包仅支持 macOS（darwin）。当前平台: ' + process.platform
+  );
+  process.exit(0);
+}
+
+const check = spawnSync('python3', ['--version'], { encoding: 'utf8' });
+if (check.error || check.status !== 0) {
+  console.warn(
+    '[@genduo/desk] 未检测到 python3。请先安装：brew install python'
+  );
+  process.exit(0);
+}
+
+console.log(
+  '[@genduo/desk] python3 ok。若要用 --verify-rms：pip3 install playwright && python3 -m playwright install chromium'
+);