@genduo/desk 0.0.0-stage → 0.4.0

package/local_config.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""本机非机密配置（shop / base URL）。不存 token / 密码。"""
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+SHOPS = ["0RT1", "3RT1", "5RT1", "3RT2", "5RT2", "7RT2"]
+DEFAULT_BASE = "https://dataanalyst.jp"
+VERSION = "0.4.0"
+HERE = Path(__file__).resolve().parent
+# 优先用户级配置，其次包内 state（POC 兼容）
+USER_CFG = Path.home() / ".config" / "dataanalyst-agent" / "config.json"
+STATE_CFG = HERE / "state" / "config.json"
+
+
+def config_path() -> Path:
+    if USER_CFG.is_file():
+        return USER_CFG
+    return STATE_CFG
+
+
+def load() -> dict:
+    path = config_path()
+    if not path.is_file():
+        return {
+            "shop": os.environ.get("DATAANALYST_SHOP", "").upper() or None,
+            "base": os.environ.get("DATAANALYST_BASE", DEFAULT_BASE).rstrip("/"),
+        }
+    data = json.loads(path.read_text(encoding="utf-8"))
+    shop = (data.get("shop") or os.environ.get("DATAANALYST_SHOP") or "").upper() or None
+    base = (data.get("base") or os.environ.get("DATAANALYST_BASE") or DEFAULT_BASE).rstrip("/")
+    return {"shop": shop, "base": base}
+
+
+def save(shop: str, base: str = DEFAULT_BASE) -> Path:
+    shop_u = shop.upper()
+    if shop_u not in SHOPS:
+        raise ValueError(f"非法店铺: {shop}（允许: {', '.join(SHOPS)}）")
+    path = USER_CFG
+    path.parent.mkdir(parents=True, exist_ok=True)
+    payload = {"shop": shop_u, "base": base.rstrip("/")}
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
+                    encoding="utf-8")
+    os.chmod(path, 0o600)
+    # 同步一份到 state，方便 POC 路径发现
+    STATE_CFG.parent.mkdir(parents=True, exist_ok=True)
+    STATE_CFG.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n",
+                         encoding="utf-8")
+    os.chmod(STATE_CFG, 0o600)
+    return path

package/package.json

@@ -1,6 +1,44 @@
 {
   "name": "@genduo/desk",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.4.0",
+  "description": "Local Mac desk agent CLI for genduo",
+  "bin": {
+    "genduo-desk": "bin/genduo-desk.js"
+  },
+  "files": [
+    "bin/",
+    "scripts/",
+    "cli.py",
+    "api_client.py",
+    "keychain_util.py",
+    "local_config.py",
+    "run.py",
+    "rms_browser_login.py",
+    "README.md",
+    "collect_access.py",
+    "collect_ads.py",
+    "ingest_util.py",
+    "rms_ads_parse.py"
+  ],
+  "scripts": {
+    "postinstall": "node scripts/postinstall.js"
+  },
+  "os": [
+    "darwin"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "preferGlobal": true,
+  "license": "UNLICENSED",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "keywords": [
+    "genduo",
+    "macos",
+    "cli",
+    "desk"
+  ]
+}

package/rms_ads_parse.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""RPP 报表 ZIP/CSV 解析。"""
+from __future__ import annotations
+
+import csv
+import io
+import re
+import zipfile
+from datetime import date
+from pathlib import Path
+
+YM_RE = re.compile(r"(\d{4})年(\d{1,2})月")
+DATE_RE = re.compile(r"(\d{4})-(\d{2})-(\d{2})")
+
+
+def _num(v):
+    if v is None:
+        return None
+    s = str(v).replace(",", "").replace("%", "").replace("円", "").strip()
+    if not s or s in ("-", "--"):
+        return None
+    try:
+        return float(s)
+    except ValueError:
+        return None
+
+
+def _period(value: str) -> tuple[date, date] | None:
+    m = YM_RE.search(value or "")
+    if m:
+        y, mo = int(m.group(1)), int(m.group(2))
+        start = date(y, mo, 1)
+        nxt = date(y + (mo == 12), 1 if mo == 12 else mo + 1, 1)
+        return start, date.fromordinal(nxt.toordinal() - 1)
+    m = DATE_RE.search(value or "")
+    if m:
+        d = date(*map(int, m.groups()))
+        return d, d
+    return None
+
+
+def _text(buf: bytes) -> str:
+    for enc in ("cp932", "utf-8-sig", "utf-8"):
+        try:
+            return buf.decode(enc)
+        except UnicodeDecodeError:
+            continue
+    return buf.decode("cp932", "replace")
+
+
+def parse_zip(buf: bytes, ad_type: str, shop: str) -> list[dict]:
+    with zipfile.ZipFile(io.BytesIO(buf)) as z:
+        names = [n for n in z.namelist() if n.lower().endswith(".csv")]
+        if not names:
+            return []
+        text = _text(z.read(names[0]))
+    rows = list(csv.reader(io.StringIO(text)))
+    header_idx = next((i for i, r in enumerate(rows)
+                       if r and "コントロールカラム" in r[0]), None)
+    if header_idx is None:
+        raise ValueError("找不到表头（コントロールカラム）")
+    header = [h.strip() for h in rows[header_idx]]
+    idx = {h: i for i, h in enumerate(header)}
+
+    def col(row, name):
+        i = idx.get(name)
+        return row[i] if i is not None and i < len(row) else None
+
+    out = []
+    for row in rows[header_idx + 1:]:
+        if not row or not any(x.strip() for x in row):
+            continue
+        d = col(row, "日付")
+        period = _period(d or "")
+        if not period:
+            continue
+        item = (col(row, "商品管理番号") or "").strip()
+        keyword = (col(row, "キーワード") or "").strip()
+        if ad_type == "rpp_item" and not item:
+            continue
+        if ad_type == "rpp_keyword" and not keyword:
+            continue
+        start, end = period
+        out.append({
+            "shop": shop, "ad_type": ad_type, "period_type": "monthly",
+            "period_start": start, "period_end": end,
+            "item_number": item, "item_name": col(row, "商品名"),
+            "keyword": keyword,
+            "clicks": int(float(col(row, "クリック数(合計)") or 0)),
+            "cost": _num(col(row, "実績額(合計)")),
+            "sales": _num(col(row, "売上金額(合計720時間)")),
+            "orders": int(float(col(row, "売上件数(合計720時間)") or 0)),
+            "cpc": _num(col(row, "CPC実績(合計)")),
+            "ctr": _num(col(row, "CTR(%)")),
+            "roas": _num(col(row, "ROAS(合計720時間)(%)")),
+            "cvr": _num(col(row, "CVR(合計720時間)(%)")),
+        })
+    return out

package/rms_browser_login.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""R-Login 浏览器登录（Playwright）并保存会话。
+
+NOTE (npm / shop CLI):
+  源真相仍是 monorepo `tools/rms_browser_login.py`（VPS 路径也用那份）。
+  本文件是 collector-agent 的 vendored 副本，供 `npm i -g dataanalyst-agent`
+  后门店机无 monorepo 也能跑 `--verify-rms`。打包前若源文件有变，请同步拷贝。
+
+Rakuten 已把 R-Login 切到 Rakuten 会员 SSO：
+  1. glogin 填 R-Login ID / 密码（第一组）；
+  2. Rakuten 会员填邮箱 / 密码（第二组）；
+  3. R-Login 提示页「次へ」→ RMS 主菜单「RMSを利用します」。
+
+凭据从 env 文件或 macOS Keychain 读，不打印。会话保存为 600 的 storage_state。
+
+用法：
+    # VPS / env 文件
+    PLAYWRIGHT_BROWSERS_PATH=/opt/ms-playwright \\
+    python tools/rms_browser_login.py --shop 7RT2 \\
+      --env-file /etc/dataanalyst/rms.env \\
+      --out /etc/dataanalyst/rms_session/7RT2.json
+
+    # 本机 Keychain（service: dataanalyst/rms/<shop>-rlogin 与 -ai）
+    python tools/rms_browser_login.py --shop 7RT2 --from-keychain \\
+      --out tools/collector-agent/state/7RT2.json
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+# Prefer existing PLAYWRIGHT_BROWSERS_PATH; only default to VPS path if present.
+_vps_browsers = "/opt/ms-playwright"
+if "PLAYWRIGHT_BROWSERS_PATH" not in os.environ and Path(_vps_browsers).is_dir():
+    os.environ["PLAYWRIGHT_BROWSERS_PATH"] = _vps_browsers
+
+from playwright.sync_api import TimeoutError as PWTimeout  # noqa: E402
+from playwright.sync_api import sync_playwright  # noqa: E402
+
+
+def load_env(path: str) -> dict:
+    out = {}
+    for line in Path(path).read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if line and not line.startswith("#") and "=" in line:
+            k, v = line.split("=", 1)
+            out[k.strip()] = v.strip()
+    return out
+
+
+def keychain_get(service: str) -> tuple[str, str]:
+    """Return (account, password) from macOS Keychain. Never prints password."""
+    meta = subprocess.check_output(
+        ["security", "find-generic-password", "-s", service],
+        stderr=subprocess.DEVNULL,
+    ).decode("utf-8", errors="replace")
+    m = re.search(r'"acct"<blob>="([^"]*)"', meta)
+    if not m:
+        raise SystemExit(f"Keychain 缺 account: {service}")
+    account = m.group(1)
+    password = subprocess.check_output(
+        ["security", "find-generic-password", "-s", service, "-w"],
+        stderr=subprocess.DEVNULL,
+    ).decode("utf-8", errors="replace").rstrip("\n")
+    if not password:
+        raise SystemExit(f"Keychain 缺 password: {service}")
+    return account, password
+
+
+def load_from_keychain(shop: str) -> dict:
+    shop_u = shop.upper()
+    login_id, password = keychain_get(f"dataanalyst/rms/{shop_u}-rlogin")
+    sso_user, sso_pass = keychain_get(f"dataanalyst/rms/{shop_u}-ai")
+    prefix = f"RMS_{shop_u}_"
+    return {
+        prefix + "LOGIN_ID": login_id,
+        prefix + "PASSWORD": password,
+        prefix + "SHOP_USER": sso_user,
+        prefix + "SHOP_PASSWORD": sso_pass,
+    }
+
+
+def _try_click(page, role: str, name: str, timeout: int = 4000) -> bool:
+    try:
+        page.get_by_role(role, name=name).first.click(timeout=timeout)
+        return True
+    except PWTimeout:
+        return False
+
+
+def login(shop: str, env: dict, out: str, headless: bool = True,
+          screenshot: str | None = None) -> None:
+    prefix = f"RMS_{shop.upper()}_"
+    login_id = env.get(prefix + "LOGIN_ID")
+    password = env.get(prefix + "PASSWORD")
+    sso_user = env.get(prefix + "SHOP_USER")
+    sso_pass = env.get(prefix + "SHOP_PASSWORD")
+    missing = [k for k, v in {
+        "LOGIN_ID": login_id, "PASSWORD": password,
+        "SHOP_USER": sso_user, "SHOP_PASSWORD": sso_pass,
+    }.items() if not v]
+    if missing:
+        raise SystemExit(f"{shop} 缺少凭据: {prefix}{','.join(missing)}")
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=headless,
+                                    args=["--no-sandbox", "--disable-dev-shm-usage"])
+        ctx = browser.new_context(locale="ja-JP", viewport={"width": 1440, "height": 1000},
+                                  user_agent=("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                                              "AppleWebKit/537.36 (KHTML, like Gecko) "
+                                              "Chrome/124.0.0.0 Safari/537.36"))
+        page = ctx.new_page()
+        page.goto("https://glogin.rms.rakuten.co.jp/?sp_id=1",
+                  wait_until="domcontentloaded", timeout=60000)
+        page.fill("#rlogin-username-ja", login_id)
+        page.fill("#rlogin-password-ja", password)
+        page.evaluate("document.querySelector('button[name=submit]')?.click()")
+        page.wait_for_timeout(3000)
+
+        page.fill("input[name=username]", sso_user)
+        page.press("input[name=username]", "Enter")
+        page.wait_for_timeout(3000)
+        page.fill("input[name=password]", sso_pass)
+        page.press("input[name=password]", "Enter")
+        page.wait_for_timeout(5000)
+
+        _try_click(page, "button", "次へ")
+        page.wait_for_timeout(2500)
+        try:
+            page.get_by_text("RMSを利用します").click(timeout=5000)
+        except PWTimeout:
+            pass
+        page.wait_for_timeout(3000)
+
+        url = page.url
+        title = page.title()
+        print("url", url)
+        print("title", title)
+        body_snip = ""
+        try:
+            body_snip = page.inner_text("body")[:500]
+        except Exception:
+            pass
+        # Shop name check (no secrets)
+        if "IORI" in body_snip or "iori" in body_snip.lower():
+            print("shop_hint", "IORI living (matched)")
+        else:
+            # try common shop name selectors without dumping full body
+            for sel in ["text=IORI living", "text=IORI"]:
+                try:
+                    if page.locator(sel).count():
+                        print("shop_hint", "IORI living (locator)")
+                        break
+                except Exception:
+                    pass
+            else:
+                print("shop_hint", "not found in first 500 chars of body")
+
+        if "mainmenu.rms.rakuten.co.jp" not in url:
+            if screenshot:
+                Path(screenshot).parent.mkdir(parents=True, exist_ok=True)
+                page.screenshot(path=screenshot, full_page=True)
+                print("screenshot", screenshot)
+            raise SystemExit("未进入 RMS 主菜单，可能遇到 2FA/验证码；请查看截图")
+
+        Path(out).parent.mkdir(parents=True, exist_ok=True)
+        ctx.storage_state(path=out)
+        os.chmod(out, 0o600)
+        print("saved", out)
+        if screenshot:
+            Path(screenshot).parent.mkdir(parents=True, exist_ok=True)
+            page.screenshot(path=screenshot, full_page=False)
+            os.chmod(screenshot, 0o600)
+            print("screenshot", screenshot)
+        browser.close()
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description="R-Login 浏览器登录并保存会话")
+    ap.add_argument("--shop", required=True)
+    ap.add_argument("--env-file", default="/etc/dataanalyst/rms.env")
+    ap.add_argument("--from-keychain", action="store_true",
+                    help="从 macOS Keychain 读 dataanalyst/rms/<SHOP>-rlogin 与 -ai")
+    ap.add_argument("--out", help="默认 tools/collector-agent/state/<shop>.json（本机）")
+    ap.add_argument("--screenshot", help="可选截图路径（不含密钥）")
+    ap.add_argument("--headed", action="store_true")
+    a = ap.parse_args()
+    out = a.out or f"tools/collector-agent/state/{a.shop.upper()}.json"
+    try:
+        if a.from_keychain:
+            print("cred_source", "keychain")
+            env = load_from_keychain(a.shop)
+            # Confirm accounts only (no passwords)
+            prefix = f"RMS_{a.shop.upper()}_"
+            print("rlogin_id_set", bool(env.get(prefix + "LOGIN_ID")))
+            print("sso_user_set", bool(env.get(prefix + "SHOP_USER")))
+        else:
+            print("cred_source", "env-file")
+            env = load_env(a.env_file)
+        login(a.shop, env, out, headless=not a.headed, screenshot=a.screenshot)
+    except Exception as e:                              # noqa: BLE001
+        print(f"登录失败: {type(e).__name__}: {e}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())