@genduo/desk 0.0.0-stage → 0.5.0

package/collect_access.py

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""日次访问人数采集（店舗カルテ get-shop-access-general）。
+
+Mac Agent 用：会话在 state/<SHOP>.json；不强制 VPS Playwright 路径；
+默认经本地代理（HTTPS_PROXY 或 http://127.0.0.1:7897）。
+会话过期（跳登录页）抛 SessionExpiredError，由上层记 last_error，不刷登录。
+"""
+from __future__ import annotations
+
+import os
+import time
+from datetime import date, datetime, timedelta
+from pathlib import Path
+from typing import Callable
+from urllib.parse import urlencode
+
+HERE = Path(__file__).resolve().parent
+STATE_DIR = HERE / "state"
+DEFAULT_PROXY = "http://127.0.0.1:7897"
+API = "https://datatool.rms.rakuten.co.jp/datatool/api/get-shop-access-general"
+ACCESS_PAGE = "https://datatool.rms.rakuten.co.jp/datatool/access"
+CHUNK_DAYS = 365
+
+
+class SessionExpiredError(RuntimeError):
+    """storage_state 失效，被重定向到登录页。"""
+
+
+class CollectError(RuntimeError):
+    pass
+
+
+def _proxy() -> str:
+    return (os.environ.get("HTTPS_PROXY")
+            or os.environ.get("https_proxy")
+            or os.environ.get("DATAANALYST_PROXY")
+            or DEFAULT_PROXY)
+
+
+def _prepare_browsers_env() -> None:
+    # 不强制 VPS 路径；仅当本机确有 /opt/ms-playwright 时才用
+    vps = Path("/opt/ms-playwright")
+    if "PLAYWRIGHT_BROWSERS_PATH" in os.environ:
+        if not Path(os.environ["PLAYWRIGHT_BROWSERS_PATH"]).is_dir():
+            os.environ.pop("PLAYWRIGHT_BROWSERS_PATH", None)
+    elif vps.is_dir():
+        os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(vps)
+
+
+def session_path(shop: str, state_dir: Path | None = None) -> Path:
+    return (state_dir or STATE_DIR) / f"{shop.upper()}.json"
+
+
+def _fetch(pg, start: date, end: date) -> list[dict]:
+    query = urlencode({
+        "resourceType": "visit",
+        "period": "daily",
+        "startDate": start.strftime("%Y%m%d"),
+        "endDate": end.strftime("%Y%m%d"),
+        "axisType": "overall",
+        "latestDate": end.strftime("%Y%m%d"),
+    })
+    r = pg.request.get(f"{API}?{query}", timeout=60000)
+    ctype = (r.headers.get("content-type") or "").lower()
+    if r.status != 200 or "json" not in ctype:
+        raise CollectError(f"access API HTTP {r.status} ctype={ctype[:40]}")
+    data = r.json()
+    rows = []
+    for item in data.get("baseData") or []:
+        raw = str(item.get("date") or "")
+        if len(raw) != 8:
+            continue
+        d = datetime.strptime(raw, "%Y%m%d").date()
+        rows.append({"d": d, "access": int(item.get("metricTotal") or 0)})
+    return rows
+
+
+def _collect_range(pg, shop: str, start: date, end: date) -> list[dict]:
+    rows: list[dict] = []
+    cursor = start
+    while cursor <= end:
+        stop = min(cursor + timedelta(days=CHUNK_DAYS - 1), end)
+        rows.extend(_fetch(pg, cursor, stop))
+        cursor = stop + timedelta(days=1)
+        time.sleep(0.4)
+    first = next((i for i, r in enumerate(rows) if r["access"] > 0), None)
+    rows = rows[first:] if first is not None else []
+    return [
+        {"shop": shop.upper(), "date": r["d"].isoformat(),
+         "access": r["access"], "source": "agent"}
+        for r in rows
+    ]
+
+
+def collect_access(
+    shop: str,
+    start: date,
+    end: date,
+    *,
+    session: Path | None = None,
+    proxy: str | None = None,
+    headless: bool = True,
+    log: Callable[[str], None] | None = None,
+) -> list[dict]:
+    """拉 [start, end] 日次访问人数。返回 ingest 友好行（date/access/source=agent）。"""
+    _log = log or (lambda m: None)
+    shop_u = shop.upper()
+    sess = session or session_path(shop_u)
+    if not sess.is_file():
+        raise CollectError(f"missing session: {sess}")
+    if start > end:
+        raise CollectError("start > end")
+    _prepare_browsers_env()
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError as exc:
+        raise CollectError(
+            "playwright 未安装：pip3 install playwright && "
+            "python3 -m playwright install chromium"
+        ) from exc
+
+    px = proxy if proxy is not None else _proxy()
+    _log(f"access collect shop={shop_u} {start}..{end} proxy={px}")
+
+    with sync_playwright() as p:
+        launch_kwargs: dict = {
+            "headless": headless,
+            "args": ["--no-sandbox", "--disable-dev-shm-usage"],
+        }
+        ctx_kwargs: dict = {
+            "storage_state": str(sess),
+            "locale": "ja-JP",
+            "viewport": {"width": 1440, "height": 1000},
+        }
+        if px:
+            launch_kwargs["proxy"] = {"server": px}
+            ctx_kwargs["proxy"] = {"server": px}
+        browser = p.chromium.launch(**launch_kwargs)
+        try:
+            ctx = browser.new_context(**ctx_kwargs)
+            pg = ctx.new_page()
+            pg.goto(ACCESS_PAGE, wait_until="domcontentloaded", timeout=90000)
+            pg.wait_for_timeout(1500)
+            url = pg.url or ""
+            if "login" in url.lower() or "glogin" in url.lower():
+                raise SessionExpiredError(
+                    f"session expired (redirect {url[:120]})")
+            rows = _collect_range(pg, shop_u, start, end)
+            _log(f"access rows={len(rows)}")
+            ctx.close()
+            return rows
+        finally:
+            browser.close()
+
+
+def default_window(lookback_days: int = 14,
+                   today: date | None = None) -> tuple[date, date]:
+    """日常窗口：昨天往前 lookback_days 天（含昨天）。"""
+    end = (today or date.today()) - timedelta(days=1)
+    start = end - timedelta(days=max(1, lookback_days) - 1)
+    return start, end

package/collect_ads.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""RPP 月报采集（商品别 + 关键词别）。
+
+从 poc_pull_rpp 提炼；会话 state/<SHOP>.json；默认本地代理。
+优先用包内 `rms_ads_parse.parse_zip`；找不到再回退 monorepo `pipeline.rms_ads.parse`。
+"""
+from __future__ import annotations
+
+import calendar
+import os
+import re
+import sys
+import time
+from datetime import date
+from pathlib import Path
+from typing import Callable
+
+HERE = Path(__file__).resolve().parent
+ROOT = HERE.parents[1]  # repo root (…/dataanalyst)
+STATE_DIR = HERE / "state"
+DEFAULT_PROXY = "http://127.0.0.1:7897"
+REPORT_TYPE = {"rpp_item": 13, "rpp_keyword": 14}
+RPP_PAGE = "https://ad.rms.rakuten.co.jp/rpp/reports"
+
+
+class SessionExpiredError(RuntimeError):
+    pass
+
+
+class CollectError(RuntimeError):
+    pass
+
+
+def _proxy() -> str:
+    return (os.environ.get("HTTPS_PROXY")
+            or os.environ.get("https_proxy")
+            or os.environ.get("DATAANALYST_PROXY")
+            or DEFAULT_PROXY)
+
+
+def _prepare_browsers_env() -> None:
+    vps = Path("/opt/ms-playwright")
+    if "PLAYWRIGHT_BROWSERS_PATH" in os.environ:
+        if not Path(os.environ["PLAYWRIGHT_BROWSERS_PATH"]).is_dir():
+            os.environ.pop("PLAYWRIGHT_BROWSERS_PATH", None)
+    elif vps.is_dir():
+        os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(vps)
+    # Mac：勿强制 VPS 路径
+    if not Path(os.environ.get("PLAYWRIGHT_BROWSERS_PATH", "")).is_dir():
+        os.environ.pop("PLAYWRIGHT_BROWSERS_PATH", None)
+
+
+def _ensure_parse():
+    # npm 包内自带；monorepo 机也可回退到 pipeline
+    try:
+        from rms_ads_parse import parse_zip  # type: ignore
+        return parse_zip
+    except ImportError:
+        pass
+    if str(ROOT) not in sys.path:
+        sys.path.insert(0, str(ROOT))
+    try:
+        from pipeline.rms_ads.parse import parse_zip  # noqa: WPS433
+    except ImportError as exc:
+        raise CollectError(
+            "无法导入 rms_ads_parse / pipeline.rms_ads.parse"
+        ) from exc
+    return parse_zip
+
+
+def session_path(shop: str, state_dir: Path | None = None) -> Path:
+    return (state_dir or STATE_DIR) / f"{shop.upper()}.json"
+
+
+def _wait_report(page, report_type: int, before_id: int, timeout: int = 120) -> int:
+    end = time.time() + timeout
+    while time.time() < end:
+        data = page.request.get(
+            "https://ad.rms.rakuten.co.jp/rpp/api/download/list").json()
+        for row in (data.get("data") or {}).get("userHistoryList") or []:
+            if (row.get("reportType") == report_type and row.get("id", 0) > before_id
+                    and row.get("status") == 2):
+                return int(row["id"])
+        time.sleep(3)
+    raise CollectError(f"report timeout reportType={report_type}")
+
+
+def _latest_id(page, report_type: int) -> int:
+    data = page.request.get(
+        "https://ad.rms.rakuten.co.jp/rpp/api/download/list").json()
+    ids = [r.get("id", 0) for r in (data.get("data") or {}).get("userHistoryList") or []
+           if r.get("reportType") == report_type]
+    return max(ids) if ids else 0
+
+
+def _filename(resp, ad_type: str) -> str:
+    cd = resp.headers.get("content-disposition", "")
+    m = re.search(r'filename="?([^";]+)', cd)
+    return m.group(1) if m else f"{ad_type}.zip"
+
+
+def pull_one(page, shop: str, ad_type: str, month: str, parse_zip) -> tuple[list[dict], str, bytes]:
+    radio = "#rdReportTypeItem" if ad_type == "rpp_item" else "#rdReportTypeKeyword"
+    page.check(radio)
+    page.check("#rdPeriodMonth")
+    y, m = int(month[:4]), int(month[5:7])
+    last = calendar.monthrange(y, m)[1]
+    dates = page.locator("input[type=text]")
+    dates.nth(0).fill(f"{y:04d}-{m:02d}-01")
+    dates.nth(1).fill(f"{y:04d}-{m:02d}-{last:02d}")
+    page.wait_for_timeout(500)
+    page.get_by_role("button", name="この条件で検索").click(timeout=15000)
+    page.wait_for_timeout(3000)
+    before = _latest_id(page, REPORT_TYPE[ad_type])
+    label = ("全商品レポートダウンロード" if ad_type == "rpp_item"
+             else "全キーワードレポートダウンロード")
+    page.get_by_role("button", name=label).click(timeout=15000)
+    download_id = _wait_report(page, REPORT_TYPE[ad_type], before)
+    resp = page.request.get(
+        f"https://ad.rms.rakuten.co.jp/rpp/api/download/report"
+        f"?downloadId={download_id}&reportType={REPORT_TYPE[ad_type]}")
+    if resp.status != 200:
+        raise CollectError(f"download HTTP {resp.status}")
+    body = resp.body()
+    filename = _filename(resp, ad_type)
+    rows = parse_zip(body, ad_type, shop.upper())
+    return rows, filename, body
+
+
+def _serialize_rows(rows: list[dict], source_file: str) -> list[dict]:
+    out = []
+    for r in rows:
+        o = {
+            "shop": r["shop"],
+            "ad_type": r["ad_type"],
+            "period_type": r.get("period_type") or "monthly",
+            "period_start": (r["period_start"].isoformat()
+                             if isinstance(r.get("period_start"), date)
+                             else r.get("period_start")),
+            "period_end": (r["period_end"].isoformat()
+                           if isinstance(r.get("period_end"), date)
+                           else r.get("period_end")),
+            "cost": r.get("cost"),
+            "clicks": r.get("clicks"),
+            "orders": r.get("orders"),
+            "sales": r.get("sales"),
+            "cpc": r.get("cpc"),
+            "ctr": r.get("ctr"),
+            "roas": r.get("roas"),
+            "cvr": r.get("cvr"),
+            "source_file": source_file,
+        }
+        item = (r.get("item_number") or r.get("item") or "").strip()
+        kw = (r.get("keyword") or "").strip()
+        name = r.get("item_name")
+        if item:
+            o["item"] = item
+        if name:
+            o["item_name"] = name
+        if kw:
+            o["keyword"] = kw
+        out.append({k: v for k, v in o.items() if v is not None and v != ""})
+    return out
+
+
+def collect_ads(
+    shop: str,
+    month: str,
+    *,
+    session: Path | None = None,
+    proxy: str | None = None,
+    headless: bool = True,
+    save_artifacts: bool = False,
+    state_dir: Path | None = None,
+    log: Callable[[str], None] | None = None,
+) -> list[dict]:
+    """拉某月 rpp_item + rpp_keyword，返回已塑形的 ingest 行。"""
+    _log = log or (lambda m: None)
+    shop_u = shop.upper()
+    if not re.fullmatch(r"\d{4}-\d{2}", month):
+        raise CollectError(f"month must be YYYY-MM, got {month!r}")
+    sess = session or session_path(shop_u, state_dir)
+    if not sess.is_file():
+        raise CollectError(f"missing session: {sess}")
+    parse_zip = _ensure_parse()
+    _prepare_browsers_env()
+    try:
+        from playwright.sync_api import sync_playwright
+    except ImportError as exc:
+        raise CollectError(
+            "playwright 未安装：pip3 install playwright && "
+            "python3 -m playwright install chromium"
+        ) from exc
+
+    px = proxy if proxy is not None else _proxy()
+    _log(f"ads collect shop={shop_u} month={month} proxy={px}")
+    out_rows: list[dict] = []
+    sdir = state_dir or STATE_DIR
+
+    with sync_playwright() as p:
+        launch_kwargs: dict = {
+            "headless": headless,
+            "args": ["--no-sandbox", "--disable-dev-shm-usage"],
+        }
+        ctx_kwargs: dict = {
+            "storage_state": str(sess),
+            "locale": "ja-JP",
+            "viewport": {"width": 1440, "height": 1000},
+        }
+        if px:
+            launch_kwargs["proxy"] = {"server": px}
+            ctx_kwargs["proxy"] = {"server": px}
+        browser = p.chromium.launch(**launch_kwargs)
+        try:
+            ctx = browser.new_context(**ctx_kwargs)
+            page = ctx.new_page()
+            page.goto(RPP_PAGE, wait_until="domcontentloaded", timeout=90000)
+            page.wait_for_timeout(3000)
+            url = page.url or ""
+            if "login" in url.lower() or "glogin" in url.lower():
+                raise SessionExpiredError(
+                    f"session expired (redirect {url[:120]})")
+            for ad_type in ("rpp_item", "rpp_keyword"):
+                _log(f"pulling {ad_type} …")
+                rows, filename, body = pull_one(
+                    page, shop_u, ad_type, month, parse_zip)
+                shaped = _serialize_rows(rows, filename)
+                out_rows.extend(shaped)
+                _log(f"{ad_type}: {len(shaped)} rows file={filename}")
+                if save_artifacts:
+                    sdir.mkdir(parents=True, exist_ok=True)
+                    tag = month.replace("-", "")
+                    zip_path = sdir / f"{shop_u}_{ad_type}_{tag}.zip"
+                    zip_path.write_bytes(body)
+                    zip_path.chmod(0o600)
+            ctx.close()
+        finally:
+            browser.close()
+    return out_rows
+
+
+def months_to_pull(today: date | None = None,
+                   last_ads_month: str | None = None) -> list[str]:
+    """日常调度：当月必拉；月初 1–3 日兼拉上月；同月已成功则仍返回当月（幂等）。"""
+    t = today or date.today()
+    cur = f"{t.year:04d}-{t.month:02d}"
+    months = [cur]
+    if t.day <= 3:
+        if t.month == 1:
+            prev = f"{t.year - 1:04d}-12"
+        else:
+            prev = f"{t.year:04d}-{t.month - 1:02d}"
+        months.insert(0, prev)
+    return months

package/ingest_util.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""组装 ingest payload、本地缓冲、推送中心。不记录 token。"""
+from __future__ import annotations
+
+import json
+import os
+import time
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Callable
+from zoneinfo import ZoneInfo
+
+import api_client
+
+HERE = Path(__file__).resolve().parent
+STATE_DIR = HERE / "state"
+BUFFER_DIR = STATE_DIR / "buffer"
+JST = ZoneInfo("Asia/Tokyo")
+
+
+def now_jst() -> datetime:
+    return datetime.now(JST)
+
+
+def build_payload(shop: str, source: str, rows: list[dict],
+                  tag: str | None = None) -> dict:
+    ts = now_jst()
+    tag = tag or ts.strftime("%Y%m%dT%H%M%S")
+    batch_id = f"agent-{shop.upper()}-{source}-{tag}"
+    if len(batch_id) > 128:
+        batch_id = batch_id[:128]
+    return {
+        "batch_id": batch_id,
+        "collected_at": ts.isoformat(timespec="seconds"),
+        "rows": rows,
+    }
+
+
+def buffer_path(source: str, batch_id: str) -> Path:
+    BUFFER_DIR.mkdir(parents=True, exist_ok=True)
+    safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in batch_id)[:100]
+    return BUFFER_DIR / f"{source}_{safe}.json"
+
+
+def save_buffer(source: str, body: dict) -> Path:
+    path = buffer_path(source, body.get("batch_id") or "unknown")
+    path.write_text(json.dumps(body, ensure_ascii=False), encoding="utf-8")
+    path.chmod(0o600)
+    return path
+
+
+def list_buffers() -> list[Path]:
+    if not BUFFER_DIR.is_dir():
+        return []
+    return sorted(BUFFER_DIR.glob("*.json"))
+
+
+def push_ingest(base: str, token: str, source: str, body: dict,
+                *, dry_run: bool = False,
+                buffer_on_fail: bool = True,
+                log: Callable[[str], None] | None = None) -> dict:
+    """推送；失败时可选写入 buffer。返回中心 JSON；附 _http 元数据不进缓冲。"""
+    _log = log or (lambda m: None)
+    n = len(body.get("rows") or [])
+    _log(f"ingest {source} batch={body.get('batch_id')} rows={n} dry_run={dry_run}")
+    try:
+        resp = api_client.ingest(base, token, source, body, dry_run=dry_run)
+        _log(f"ingest {source} ok received={resp.get('received')} "
+             f"inserted={resp.get('inserted')} duplicates={resp.get('duplicates')} "
+             f"dry_run={resp.get('dry_run')}")
+        return resp
+    except api_client.ApiError as exc:
+        _log(f"ingest {source} FAIL HTTP {exc.code}")
+        if buffer_on_fail and not dry_run:
+            path = save_buffer(source, body)
+            _log(f"buffered → {path.name}")
+        raise
+
+
+def flush_buffers(base: str, token: str,
+                  log: Callable[[str], None] | None = None) -> dict[str, Any]:
+    """重试本地缓冲；成功则删除文件。"""
+    _log = log or (lambda m: None)
+    summary = {"tried": 0, "ok": 0, "fail": 0}
+    for path in list_buffers():
+        summary["tried"] += 1
+        try:
+            body = json.loads(path.read_text(encoding="utf-8"))
+            # 文件名: {source}_{batch...}.json — source 取首段已知名
+            name = path.name
+            source = "rms_access" if name.startswith("rms_access") else (
+                "rms_ads" if name.startswith("rms_ads") else None)
+            if not source:
+                # 尝试从 batch_id
+                bid = body.get("batch_id") or ""
+                if "rms_access" in bid:
+                    source = "rms_access"
+                elif "rms_ads" in bid:
+                    source = "rms_ads"
+                else:
+                    _log(f"buffer skip unknown source: {path.name}")
+                    summary["fail"] += 1
+                    continue
+            api_client.ingest(base, token, source, body, dry_run=False)
+            path.unlink(missing_ok=True)
+            summary["ok"] += 1
+            _log(f"buffer flushed {path.name}")
+        except Exception as exc:  # noqa: BLE001
+            summary["fail"] += 1
+            _log(f"buffer flush fail {path.name}: {type(exc).__name__}")
+            time.sleep(0.5)
+    return summary
+
+
+def load_daemon_state(path: Path | None = None) -> dict:
+    p = path or (STATE_DIR / "daemon_state.json")
+    if not p.is_file():
+        return {}
+    try:
+        return json.loads(p.read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return {}
+
+
+def save_daemon_state(state: dict, path: Path | None = None) -> None:
+    p = path or (STATE_DIR / "daemon_state.json")
+    STATE_DIR.mkdir(parents=True, exist_ok=True)
+    # 永不写入 token / 密码
+    clean = {k: v for k, v in state.items()
+             if "token" not in k.lower() and "password" not in k.lower()}
+    tmp = p.with_suffix(".tmp")
+    tmp.write_text(json.dumps(clean, ensure_ascii=False, indent=2) + "\n",
+                   encoding="utf-8")
+    os.chmod(tmp, 0o600)
+    tmp.replace(p)