@genduo/desk 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +186 -2
- package/api_client.py +65 -0
- package/bin/genduo-desk.js +54 -0
- package/cli.py +632 -0
- package/collect_access.py +162 -0
- package/collect_ads.py +255 -0
- package/ingest_util.py +136 -0
- package/keychain_util.py +115 -0
- package/local_config.py +53 -0
- package/package.json +42 -4
- package/rms_ads_parse.py +99 -0
- package/rms_browser_login.py +213 -0
- package/run.py +303 -0
- package/scripts/postinstall.js +23 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""日次访问人数采集(店舗カルテ get-shop-access-general)。
|
|
4
|
+
|
|
5
|
+
Mac Agent 用:会话在 state/<SHOP>.json;不强制 VPS Playwright 路径;
|
|
6
|
+
默认经本地代理(HTTPS_PROXY 或 http://127.0.0.1:7897)。
|
|
7
|
+
会话过期(跳登录页)抛 SessionExpiredError,由上层记 last_error,不刷登录。
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import os
|
|
12
|
+
import time
|
|
13
|
+
from datetime import date, datetime, timedelta
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Callable
|
|
16
|
+
from urllib.parse import urlencode
|
|
17
|
+
|
|
18
|
+
HERE = Path(__file__).resolve().parent
|
|
19
|
+
STATE_DIR = HERE / "state"
|
|
20
|
+
DEFAULT_PROXY = "http://127.0.0.1:7897"
|
|
21
|
+
API = "https://datatool.rms.rakuten.co.jp/datatool/api/get-shop-access-general"
|
|
22
|
+
ACCESS_PAGE = "https://datatool.rms.rakuten.co.jp/datatool/access"
|
|
23
|
+
CHUNK_DAYS = 365
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class SessionExpiredError(RuntimeError):
|
|
27
|
+
"""storage_state 失效,被重定向到登录页。"""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CollectError(RuntimeError):
|
|
31
|
+
pass
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _proxy() -> str:
|
|
35
|
+
return (os.environ.get("HTTPS_PROXY")
|
|
36
|
+
or os.environ.get("https_proxy")
|
|
37
|
+
or os.environ.get("DATAANALYST_PROXY")
|
|
38
|
+
or DEFAULT_PROXY)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _prepare_browsers_env() -> None:
|
|
42
|
+
# 不强制 VPS 路径;仅当本机确有 /opt/ms-playwright 时才用
|
|
43
|
+
vps = Path("/opt/ms-playwright")
|
|
44
|
+
if "PLAYWRIGHT_BROWSERS_PATH" in os.environ:
|
|
45
|
+
if not Path(os.environ["PLAYWRIGHT_BROWSERS_PATH"]).is_dir():
|
|
46
|
+
os.environ.pop("PLAYWRIGHT_BROWSERS_PATH", None)
|
|
47
|
+
elif vps.is_dir():
|
|
48
|
+
os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(vps)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def session_path(shop: str, state_dir: Path | None = None) -> Path:
|
|
52
|
+
return (state_dir or STATE_DIR) / f"{shop.upper()}.json"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _fetch(pg, start: date, end: date) -> list[dict]:
|
|
56
|
+
query = urlencode({
|
|
57
|
+
"resourceType": "visit",
|
|
58
|
+
"period": "daily",
|
|
59
|
+
"startDate": start.strftime("%Y%m%d"),
|
|
60
|
+
"endDate": end.strftime("%Y%m%d"),
|
|
61
|
+
"axisType": "overall",
|
|
62
|
+
"latestDate": end.strftime("%Y%m%d"),
|
|
63
|
+
})
|
|
64
|
+
r = pg.request.get(f"{API}?{query}", timeout=60000)
|
|
65
|
+
ctype = (r.headers.get("content-type") or "").lower()
|
|
66
|
+
if r.status != 200 or "json" not in ctype:
|
|
67
|
+
raise CollectError(f"access API HTTP {r.status} ctype={ctype[:40]}")
|
|
68
|
+
data = r.json()
|
|
69
|
+
rows = []
|
|
70
|
+
for item in data.get("baseData") or []:
|
|
71
|
+
raw = str(item.get("date") or "")
|
|
72
|
+
if len(raw) != 8:
|
|
73
|
+
continue
|
|
74
|
+
d = datetime.strptime(raw, "%Y%m%d").date()
|
|
75
|
+
rows.append({"d": d, "access": int(item.get("metricTotal") or 0)})
|
|
76
|
+
return rows
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _collect_range(pg, shop: str, start: date, end: date) -> list[dict]:
|
|
80
|
+
rows: list[dict] = []
|
|
81
|
+
cursor = start
|
|
82
|
+
while cursor <= end:
|
|
83
|
+
stop = min(cursor + timedelta(days=CHUNK_DAYS - 1), end)
|
|
84
|
+
rows.extend(_fetch(pg, cursor, stop))
|
|
85
|
+
cursor = stop + timedelta(days=1)
|
|
86
|
+
time.sleep(0.4)
|
|
87
|
+
first = next((i for i, r in enumerate(rows) if r["access"] > 0), None)
|
|
88
|
+
rows = rows[first:] if first is not None else []
|
|
89
|
+
return [
|
|
90
|
+
{"shop": shop.upper(), "date": r["d"].isoformat(),
|
|
91
|
+
"access": r["access"], "source": "agent"}
|
|
92
|
+
for r in rows
|
|
93
|
+
]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def collect_access(
|
|
97
|
+
shop: str,
|
|
98
|
+
start: date,
|
|
99
|
+
end: date,
|
|
100
|
+
*,
|
|
101
|
+
session: Path | None = None,
|
|
102
|
+
proxy: str | None = None,
|
|
103
|
+
headless: bool = True,
|
|
104
|
+
log: Callable[[str], None] | None = None,
|
|
105
|
+
) -> list[dict]:
|
|
106
|
+
"""拉 [start, end] 日次访问人数。返回 ingest 友好行(date/access/source=agent)。"""
|
|
107
|
+
_log = log or (lambda m: None)
|
|
108
|
+
shop_u = shop.upper()
|
|
109
|
+
sess = session or session_path(shop_u)
|
|
110
|
+
if not sess.is_file():
|
|
111
|
+
raise CollectError(f"missing session: {sess}")
|
|
112
|
+
if start > end:
|
|
113
|
+
raise CollectError("start > end")
|
|
114
|
+
_prepare_browsers_env()
|
|
115
|
+
try:
|
|
116
|
+
from playwright.sync_api import sync_playwright
|
|
117
|
+
except ImportError as exc:
|
|
118
|
+
raise CollectError(
|
|
119
|
+
"playwright 未安装:pip3 install playwright && "
|
|
120
|
+
"python3 -m playwright install chromium"
|
|
121
|
+
) from exc
|
|
122
|
+
|
|
123
|
+
px = proxy if proxy is not None else _proxy()
|
|
124
|
+
_log(f"access collect shop={shop_u} {start}..{end} proxy={px}")
|
|
125
|
+
|
|
126
|
+
with sync_playwright() as p:
|
|
127
|
+
launch_kwargs: dict = {
|
|
128
|
+
"headless": headless,
|
|
129
|
+
"args": ["--no-sandbox", "--disable-dev-shm-usage"],
|
|
130
|
+
}
|
|
131
|
+
ctx_kwargs: dict = {
|
|
132
|
+
"storage_state": str(sess),
|
|
133
|
+
"locale": "ja-JP",
|
|
134
|
+
"viewport": {"width": 1440, "height": 1000},
|
|
135
|
+
}
|
|
136
|
+
if px:
|
|
137
|
+
launch_kwargs["proxy"] = {"server": px}
|
|
138
|
+
ctx_kwargs["proxy"] = {"server": px}
|
|
139
|
+
browser = p.chromium.launch(**launch_kwargs)
|
|
140
|
+
try:
|
|
141
|
+
ctx = browser.new_context(**ctx_kwargs)
|
|
142
|
+
pg = ctx.new_page()
|
|
143
|
+
pg.goto(ACCESS_PAGE, wait_until="domcontentloaded", timeout=90000)
|
|
144
|
+
pg.wait_for_timeout(1500)
|
|
145
|
+
url = pg.url or ""
|
|
146
|
+
if "login" in url.lower() or "glogin" in url.lower():
|
|
147
|
+
raise SessionExpiredError(
|
|
148
|
+
f"session expired (redirect {url[:120]})")
|
|
149
|
+
rows = _collect_range(pg, shop_u, start, end)
|
|
150
|
+
_log(f"access rows={len(rows)}")
|
|
151
|
+
ctx.close()
|
|
152
|
+
return rows
|
|
153
|
+
finally:
|
|
154
|
+
browser.close()
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def default_window(lookback_days: int = 14,
|
|
158
|
+
today: date | None = None) -> tuple[date, date]:
|
|
159
|
+
"""日常窗口:昨天往前 lookback_days 天(含昨天)。"""
|
|
160
|
+
end = (today or date.today()) - timedelta(days=1)
|
|
161
|
+
start = end - timedelta(days=max(1, lookback_days) - 1)
|
|
162
|
+
return start, end
|
package/collect_ads.py
ADDED
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""RPP 月报采集(商品别 + 关键词别)。
|
|
4
|
+
|
|
5
|
+
从 poc_pull_rpp 提炼;会话 state/<SHOP>.json;默认本地代理。
|
|
6
|
+
优先用包内 `rms_ads_parse.parse_zip`;找不到再回退 monorepo `pipeline.rms_ads.parse`。
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import calendar
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import sys
|
|
14
|
+
import time
|
|
15
|
+
from datetime import date
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Callable
|
|
18
|
+
|
|
19
|
+
HERE = Path(__file__).resolve().parent
|
|
20
|
+
ROOT = HERE.parents[1] # repo root (…/dataanalyst)
|
|
21
|
+
STATE_DIR = HERE / "state"
|
|
22
|
+
DEFAULT_PROXY = "http://127.0.0.1:7897"
|
|
23
|
+
REPORT_TYPE = {"rpp_item": 13, "rpp_keyword": 14}
|
|
24
|
+
RPP_PAGE = "https://ad.rms.rakuten.co.jp/rpp/reports"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class SessionExpiredError(RuntimeError):
|
|
28
|
+
pass
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class CollectError(RuntimeError):
|
|
32
|
+
pass
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _proxy() -> str:
|
|
36
|
+
return (os.environ.get("HTTPS_PROXY")
|
|
37
|
+
or os.environ.get("https_proxy")
|
|
38
|
+
or os.environ.get("DATAANALYST_PROXY")
|
|
39
|
+
or DEFAULT_PROXY)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _prepare_browsers_env() -> None:
|
|
43
|
+
vps = Path("/opt/ms-playwright")
|
|
44
|
+
if "PLAYWRIGHT_BROWSERS_PATH" in os.environ:
|
|
45
|
+
if not Path(os.environ["PLAYWRIGHT_BROWSERS_PATH"]).is_dir():
|
|
46
|
+
os.environ.pop("PLAYWRIGHT_BROWSERS_PATH", None)
|
|
47
|
+
elif vps.is_dir():
|
|
48
|
+
os.environ["PLAYWRIGHT_BROWSERS_PATH"] = str(vps)
|
|
49
|
+
# Mac:勿强制 VPS 路径
|
|
50
|
+
if not Path(os.environ.get("PLAYWRIGHT_BROWSERS_PATH", "")).is_dir():
|
|
51
|
+
os.environ.pop("PLAYWRIGHT_BROWSERS_PATH", None)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _ensure_parse():
|
|
55
|
+
# npm 包内自带;monorepo 机也可回退到 pipeline
|
|
56
|
+
try:
|
|
57
|
+
from rms_ads_parse import parse_zip # type: ignore
|
|
58
|
+
return parse_zip
|
|
59
|
+
except ImportError:
|
|
60
|
+
pass
|
|
61
|
+
if str(ROOT) not in sys.path:
|
|
62
|
+
sys.path.insert(0, str(ROOT))
|
|
63
|
+
try:
|
|
64
|
+
from pipeline.rms_ads.parse import parse_zip # noqa: WPS433
|
|
65
|
+
except ImportError as exc:
|
|
66
|
+
raise CollectError(
|
|
67
|
+
"无法导入 rms_ads_parse / pipeline.rms_ads.parse"
|
|
68
|
+
) from exc
|
|
69
|
+
return parse_zip
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def session_path(shop: str, state_dir: Path | None = None) -> Path:
|
|
73
|
+
return (state_dir or STATE_DIR) / f"{shop.upper()}.json"
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _wait_report(page, report_type: int, before_id: int, timeout: int = 120) -> int:
|
|
77
|
+
end = time.time() + timeout
|
|
78
|
+
while time.time() < end:
|
|
79
|
+
data = page.request.get(
|
|
80
|
+
"https://ad.rms.rakuten.co.jp/rpp/api/download/list").json()
|
|
81
|
+
for row in (data.get("data") or {}).get("userHistoryList") or []:
|
|
82
|
+
if (row.get("reportType") == report_type and row.get("id", 0) > before_id
|
|
83
|
+
and row.get("status") == 2):
|
|
84
|
+
return int(row["id"])
|
|
85
|
+
time.sleep(3)
|
|
86
|
+
raise CollectError(f"report timeout reportType={report_type}")
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _latest_id(page, report_type: int) -> int:
|
|
90
|
+
data = page.request.get(
|
|
91
|
+
"https://ad.rms.rakuten.co.jp/rpp/api/download/list").json()
|
|
92
|
+
ids = [r.get("id", 0) for r in (data.get("data") or {}).get("userHistoryList") or []
|
|
93
|
+
if r.get("reportType") == report_type]
|
|
94
|
+
return max(ids) if ids else 0
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _filename(resp, ad_type: str) -> str:
|
|
98
|
+
cd = resp.headers.get("content-disposition", "")
|
|
99
|
+
m = re.search(r'filename="?([^";]+)', cd)
|
|
100
|
+
return m.group(1) if m else f"{ad_type}.zip"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def pull_one(page, shop: str, ad_type: str, month: str, parse_zip) -> tuple[list[dict], str, bytes]:
|
|
104
|
+
radio = "#rdReportTypeItem" if ad_type == "rpp_item" else "#rdReportTypeKeyword"
|
|
105
|
+
page.check(radio)
|
|
106
|
+
page.check("#rdPeriodMonth")
|
|
107
|
+
y, m = int(month[:4]), int(month[5:7])
|
|
108
|
+
last = calendar.monthrange(y, m)[1]
|
|
109
|
+
dates = page.locator("input[type=text]")
|
|
110
|
+
dates.nth(0).fill(f"{y:04d}-{m:02d}-01")
|
|
111
|
+
dates.nth(1).fill(f"{y:04d}-{m:02d}-{last:02d}")
|
|
112
|
+
page.wait_for_timeout(500)
|
|
113
|
+
page.get_by_role("button", name="この条件で検索").click(timeout=15000)
|
|
114
|
+
page.wait_for_timeout(3000)
|
|
115
|
+
before = _latest_id(page, REPORT_TYPE[ad_type])
|
|
116
|
+
label = ("全商品レポートダウンロード" if ad_type == "rpp_item"
|
|
117
|
+
else "全キーワードレポートダウンロード")
|
|
118
|
+
page.get_by_role("button", name=label).click(timeout=15000)
|
|
119
|
+
download_id = _wait_report(page, REPORT_TYPE[ad_type], before)
|
|
120
|
+
resp = page.request.get(
|
|
121
|
+
f"https://ad.rms.rakuten.co.jp/rpp/api/download/report"
|
|
122
|
+
f"?downloadId={download_id}&reportType={REPORT_TYPE[ad_type]}")
|
|
123
|
+
if resp.status != 200:
|
|
124
|
+
raise CollectError(f"download HTTP {resp.status}")
|
|
125
|
+
body = resp.body()
|
|
126
|
+
filename = _filename(resp, ad_type)
|
|
127
|
+
rows = parse_zip(body, ad_type, shop.upper())
|
|
128
|
+
return rows, filename, body
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _serialize_rows(rows: list[dict], source_file: str) -> list[dict]:
|
|
132
|
+
out = []
|
|
133
|
+
for r in rows:
|
|
134
|
+
o = {
|
|
135
|
+
"shop": r["shop"],
|
|
136
|
+
"ad_type": r["ad_type"],
|
|
137
|
+
"period_type": r.get("period_type") or "monthly",
|
|
138
|
+
"period_start": (r["period_start"].isoformat()
|
|
139
|
+
if isinstance(r.get("period_start"), date)
|
|
140
|
+
else r.get("period_start")),
|
|
141
|
+
"period_end": (r["period_end"].isoformat()
|
|
142
|
+
if isinstance(r.get("period_end"), date)
|
|
143
|
+
else r.get("period_end")),
|
|
144
|
+
"cost": r.get("cost"),
|
|
145
|
+
"clicks": r.get("clicks"),
|
|
146
|
+
"orders": r.get("orders"),
|
|
147
|
+
"sales": r.get("sales"),
|
|
148
|
+
"cpc": r.get("cpc"),
|
|
149
|
+
"ctr": r.get("ctr"),
|
|
150
|
+
"roas": r.get("roas"),
|
|
151
|
+
"cvr": r.get("cvr"),
|
|
152
|
+
"source_file": source_file,
|
|
153
|
+
}
|
|
154
|
+
item = (r.get("item_number") or r.get("item") or "").strip()
|
|
155
|
+
kw = (r.get("keyword") or "").strip()
|
|
156
|
+
name = r.get("item_name")
|
|
157
|
+
if item:
|
|
158
|
+
o["item"] = item
|
|
159
|
+
if name:
|
|
160
|
+
o["item_name"] = name
|
|
161
|
+
if kw:
|
|
162
|
+
o["keyword"] = kw
|
|
163
|
+
out.append({k: v for k, v in o.items() if v is not None and v != ""})
|
|
164
|
+
return out
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def collect_ads(
|
|
168
|
+
shop: str,
|
|
169
|
+
month: str,
|
|
170
|
+
*,
|
|
171
|
+
session: Path | None = None,
|
|
172
|
+
proxy: str | None = None,
|
|
173
|
+
headless: bool = True,
|
|
174
|
+
save_artifacts: bool = False,
|
|
175
|
+
state_dir: Path | None = None,
|
|
176
|
+
log: Callable[[str], None] | None = None,
|
|
177
|
+
) -> list[dict]:
|
|
178
|
+
"""拉某月 rpp_item + rpp_keyword,返回已塑形的 ingest 行。"""
|
|
179
|
+
_log = log or (lambda m: None)
|
|
180
|
+
shop_u = shop.upper()
|
|
181
|
+
if not re.fullmatch(r"\d{4}-\d{2}", month):
|
|
182
|
+
raise CollectError(f"month must be YYYY-MM, got {month!r}")
|
|
183
|
+
sess = session or session_path(shop_u, state_dir)
|
|
184
|
+
if not sess.is_file():
|
|
185
|
+
raise CollectError(f"missing session: {sess}")
|
|
186
|
+
parse_zip = _ensure_parse()
|
|
187
|
+
_prepare_browsers_env()
|
|
188
|
+
try:
|
|
189
|
+
from playwright.sync_api import sync_playwright
|
|
190
|
+
except ImportError as exc:
|
|
191
|
+
raise CollectError(
|
|
192
|
+
"playwright 未安装:pip3 install playwright && "
|
|
193
|
+
"python3 -m playwright install chromium"
|
|
194
|
+
) from exc
|
|
195
|
+
|
|
196
|
+
px = proxy if proxy is not None else _proxy()
|
|
197
|
+
_log(f"ads collect shop={shop_u} month={month} proxy={px}")
|
|
198
|
+
out_rows: list[dict] = []
|
|
199
|
+
sdir = state_dir or STATE_DIR
|
|
200
|
+
|
|
201
|
+
with sync_playwright() as p:
|
|
202
|
+
launch_kwargs: dict = {
|
|
203
|
+
"headless": headless,
|
|
204
|
+
"args": ["--no-sandbox", "--disable-dev-shm-usage"],
|
|
205
|
+
}
|
|
206
|
+
ctx_kwargs: dict = {
|
|
207
|
+
"storage_state": str(sess),
|
|
208
|
+
"locale": "ja-JP",
|
|
209
|
+
"viewport": {"width": 1440, "height": 1000},
|
|
210
|
+
}
|
|
211
|
+
if px:
|
|
212
|
+
launch_kwargs["proxy"] = {"server": px}
|
|
213
|
+
ctx_kwargs["proxy"] = {"server": px}
|
|
214
|
+
browser = p.chromium.launch(**launch_kwargs)
|
|
215
|
+
try:
|
|
216
|
+
ctx = browser.new_context(**ctx_kwargs)
|
|
217
|
+
page = ctx.new_page()
|
|
218
|
+
page.goto(RPP_PAGE, wait_until="domcontentloaded", timeout=90000)
|
|
219
|
+
page.wait_for_timeout(3000)
|
|
220
|
+
url = page.url or ""
|
|
221
|
+
if "login" in url.lower() or "glogin" in url.lower():
|
|
222
|
+
raise SessionExpiredError(
|
|
223
|
+
f"session expired (redirect {url[:120]})")
|
|
224
|
+
for ad_type in ("rpp_item", "rpp_keyword"):
|
|
225
|
+
_log(f"pulling {ad_type} …")
|
|
226
|
+
rows, filename, body = pull_one(
|
|
227
|
+
page, shop_u, ad_type, month, parse_zip)
|
|
228
|
+
shaped = _serialize_rows(rows, filename)
|
|
229
|
+
out_rows.extend(shaped)
|
|
230
|
+
_log(f"{ad_type}: {len(shaped)} rows file={filename}")
|
|
231
|
+
if save_artifacts:
|
|
232
|
+
sdir.mkdir(parents=True, exist_ok=True)
|
|
233
|
+
tag = month.replace("-", "")
|
|
234
|
+
zip_path = sdir / f"{shop_u}_{ad_type}_{tag}.zip"
|
|
235
|
+
zip_path.write_bytes(body)
|
|
236
|
+
zip_path.chmod(0o600)
|
|
237
|
+
ctx.close()
|
|
238
|
+
finally:
|
|
239
|
+
browser.close()
|
|
240
|
+
return out_rows
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def months_to_pull(today: date | None = None,
|
|
244
|
+
last_ads_month: str | None = None) -> list[str]:
|
|
245
|
+
"""日常调度:当月必拉;月初 1–3 日兼拉上月;同月已成功则仍返回当月(幂等)。"""
|
|
246
|
+
t = today or date.today()
|
|
247
|
+
cur = f"{t.year:04d}-{t.month:02d}"
|
|
248
|
+
months = [cur]
|
|
249
|
+
if t.day <= 3:
|
|
250
|
+
if t.month == 1:
|
|
251
|
+
prev = f"{t.year - 1:04d}-12"
|
|
252
|
+
else:
|
|
253
|
+
prev = f"{t.year:04d}-{t.month - 1:02d}"
|
|
254
|
+
months.insert(0, prev)
|
|
255
|
+
return months
|
package/ingest_util.py
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""组装 ingest payload、本地缓冲、推送中心。不记录 token。"""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import time
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any, Callable
|
|
12
|
+
from zoneinfo import ZoneInfo
|
|
13
|
+
|
|
14
|
+
import api_client
|
|
15
|
+
|
|
16
|
+
HERE = Path(__file__).resolve().parent
|
|
17
|
+
STATE_DIR = HERE / "state"
|
|
18
|
+
BUFFER_DIR = STATE_DIR / "buffer"
|
|
19
|
+
JST = ZoneInfo("Asia/Tokyo")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def now_jst() -> datetime:
|
|
23
|
+
return datetime.now(JST)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def build_payload(shop: str, source: str, rows: list[dict],
|
|
27
|
+
tag: str | None = None) -> dict:
|
|
28
|
+
ts = now_jst()
|
|
29
|
+
tag = tag or ts.strftime("%Y%m%dT%H%M%S")
|
|
30
|
+
batch_id = f"agent-{shop.upper()}-{source}-{tag}"
|
|
31
|
+
if len(batch_id) > 128:
|
|
32
|
+
batch_id = batch_id[:128]
|
|
33
|
+
return {
|
|
34
|
+
"batch_id": batch_id,
|
|
35
|
+
"collected_at": ts.isoformat(timespec="seconds"),
|
|
36
|
+
"rows": rows,
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def buffer_path(source: str, batch_id: str) -> Path:
|
|
41
|
+
BUFFER_DIR.mkdir(parents=True, exist_ok=True)
|
|
42
|
+
safe = "".join(c if c.isalnum() or c in "-_" else "_" for c in batch_id)[:100]
|
|
43
|
+
return BUFFER_DIR / f"{source}_{safe}.json"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def save_buffer(source: str, body: dict) -> Path:
|
|
47
|
+
path = buffer_path(source, body.get("batch_id") or "unknown")
|
|
48
|
+
path.write_text(json.dumps(body, ensure_ascii=False), encoding="utf-8")
|
|
49
|
+
path.chmod(0o600)
|
|
50
|
+
return path
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def list_buffers() -> list[Path]:
|
|
54
|
+
if not BUFFER_DIR.is_dir():
|
|
55
|
+
return []
|
|
56
|
+
return sorted(BUFFER_DIR.glob("*.json"))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def push_ingest(base: str, token: str, source: str, body: dict,
|
|
60
|
+
*, dry_run: bool = False,
|
|
61
|
+
buffer_on_fail: bool = True,
|
|
62
|
+
log: Callable[[str], None] | None = None) -> dict:
|
|
63
|
+
"""推送;失败时可选写入 buffer。返回中心 JSON;附 _http 元数据不进缓冲。"""
|
|
64
|
+
_log = log or (lambda m: None)
|
|
65
|
+
n = len(body.get("rows") or [])
|
|
66
|
+
_log(f"ingest {source} batch={body.get('batch_id')} rows={n} dry_run={dry_run}")
|
|
67
|
+
try:
|
|
68
|
+
resp = api_client.ingest(base, token, source, body, dry_run=dry_run)
|
|
69
|
+
_log(f"ingest {source} ok received={resp.get('received')} "
|
|
70
|
+
f"inserted={resp.get('inserted')} duplicates={resp.get('duplicates')} "
|
|
71
|
+
f"dry_run={resp.get('dry_run')}")
|
|
72
|
+
return resp
|
|
73
|
+
except api_client.ApiError as exc:
|
|
74
|
+
_log(f"ingest {source} FAIL HTTP {exc.code}")
|
|
75
|
+
if buffer_on_fail and not dry_run:
|
|
76
|
+
path = save_buffer(source, body)
|
|
77
|
+
_log(f"buffered → {path.name}")
|
|
78
|
+
raise
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def flush_buffers(base: str, token: str,
|
|
82
|
+
log: Callable[[str], None] | None = None) -> dict[str, Any]:
|
|
83
|
+
"""重试本地缓冲;成功则删除文件。"""
|
|
84
|
+
_log = log or (lambda m: None)
|
|
85
|
+
summary = {"tried": 0, "ok": 0, "fail": 0}
|
|
86
|
+
for path in list_buffers():
|
|
87
|
+
summary["tried"] += 1
|
|
88
|
+
try:
|
|
89
|
+
body = json.loads(path.read_text(encoding="utf-8"))
|
|
90
|
+
# 文件名: {source}_{batch...}.json — source 取首段已知名
|
|
91
|
+
name = path.name
|
|
92
|
+
source = "rms_access" if name.startswith("rms_access") else (
|
|
93
|
+
"rms_ads" if name.startswith("rms_ads") else None)
|
|
94
|
+
if not source:
|
|
95
|
+
# 尝试从 batch_id
|
|
96
|
+
bid = body.get("batch_id") or ""
|
|
97
|
+
if "rms_access" in bid:
|
|
98
|
+
source = "rms_access"
|
|
99
|
+
elif "rms_ads" in bid:
|
|
100
|
+
source = "rms_ads"
|
|
101
|
+
else:
|
|
102
|
+
_log(f"buffer skip unknown source: {path.name}")
|
|
103
|
+
summary["fail"] += 1
|
|
104
|
+
continue
|
|
105
|
+
api_client.ingest(base, token, source, body, dry_run=False)
|
|
106
|
+
path.unlink(missing_ok=True)
|
|
107
|
+
summary["ok"] += 1
|
|
108
|
+
_log(f"buffer flushed {path.name}")
|
|
109
|
+
except Exception as exc: # noqa: BLE001
|
|
110
|
+
summary["fail"] += 1
|
|
111
|
+
_log(f"buffer flush fail {path.name}: {type(exc).__name__}")
|
|
112
|
+
time.sleep(0.5)
|
|
113
|
+
return summary
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def load_daemon_state(path: Path | None = None) -> dict:
|
|
117
|
+
p = path or (STATE_DIR / "daemon_state.json")
|
|
118
|
+
if not p.is_file():
|
|
119
|
+
return {}
|
|
120
|
+
try:
|
|
121
|
+
return json.loads(p.read_text(encoding="utf-8"))
|
|
122
|
+
except (OSError, json.JSONDecodeError):
|
|
123
|
+
return {}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def save_daemon_state(state: dict, path: Path | None = None) -> None:
|
|
127
|
+
p = path or (STATE_DIR / "daemon_state.json")
|
|
128
|
+
STATE_DIR.mkdir(parents=True, exist_ok=True)
|
|
129
|
+
# 永不写入 token / 密码
|
|
130
|
+
clean = {k: v for k, v in state.items()
|
|
131
|
+
if "token" not in k.lower() and "password" not in k.lower()}
|
|
132
|
+
tmp = p.with_suffix(".tmp")
|
|
133
|
+
tmp.write_text(json.dumps(clean, ensure_ascii=False, indent=2) + "\n",
|
|
134
|
+
encoding="utf-8")
|
|
135
|
+
os.chmod(tmp, 0o600)
|
|
136
|
+
tmp.replace(p)
|
package/keychain_util.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""macOS Keychain 读写(不打印 secret)。
|
|
4
|
+
|
|
5
|
+
与现有约定对齐:
|
|
6
|
+
· Agent token service = dataanalyst-agent-token account = $USER
|
|
7
|
+
· R-Login 第一组 service = dataanalyst/rms/<SHOP>-rlogin account = R-Login ID
|
|
8
|
+
· Rakuten SSO 第二组 service = dataanalyst/rms/<SHOP>-ai account = 邮箱
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
import subprocess
|
|
15
|
+
from typing import Optional
|
|
16
|
+
|
|
17
|
+
AGENT_TOKEN_SERVICE = "dataanalyst-agent-token"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def rms_rlogin_service(shop: str) -> str:
|
|
21
|
+
return f"dataanalyst/rms/{shop.upper()}-rlogin"
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def rms_ai_service(shop: str) -> str:
|
|
25
|
+
return f"dataanalyst/rms/{shop.upper()}-ai"
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _run(cmd: list[str], *, check: bool = True) -> subprocess.CompletedProcess:
|
|
29
|
+
return subprocess.run(cmd, capture_output=True, text=True, check=check)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def has_generic(service: str, account: Optional[str] = None) -> bool:
|
|
33
|
+
cmd = ["security", "find-generic-password", "-s", service]
|
|
34
|
+
if account:
|
|
35
|
+
cmd += ["-a", account]
|
|
36
|
+
r = _run(cmd, check=False)
|
|
37
|
+
return r.returncode == 0
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def get_password(service: str, account: Optional[str] = None) -> str:
|
|
41
|
+
"""读取密码;失败抛 RuntimeError。调用方不得打印返回值。"""
|
|
42
|
+
cmd = ["security", "find-generic-password", "-s", service, "-w"]
|
|
43
|
+
if account:
|
|
44
|
+
cmd += ["-a", account]
|
|
45
|
+
r = _run(cmd, check=False)
|
|
46
|
+
if r.returncode != 0:
|
|
47
|
+
raise RuntimeError(f"Keychain 缺少条目: {service}")
|
|
48
|
+
return (r.stdout or "").rstrip("\n")
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def get_account(service: str) -> str:
|
|
52
|
+
r = _run(["security", "find-generic-password", "-s", service], check=False)
|
|
53
|
+
if r.returncode != 0:
|
|
54
|
+
raise RuntimeError(f"Keychain 缺少条目: {service}")
|
|
55
|
+
m = re.search(r'"acct"<blob>="([^"]*)"', r.stdout or "")
|
|
56
|
+
if not m:
|
|
57
|
+
raise RuntimeError(f"Keychain 缺 account: {service}")
|
|
58
|
+
return m.group(1)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def set_generic(service: str, account: str, secret: str, *,
|
|
62
|
+
label: str = "") -> None:
|
|
63
|
+
"""写入/覆盖 generic password;secret 不进日志。"""
|
|
64
|
+
if not secret:
|
|
65
|
+
raise ValueError(f"拒绝写入空 secret: {service}")
|
|
66
|
+
cmd = ["security", "add-generic-password", "-U",
|
|
67
|
+
"-s", service, "-a", account, "-w", secret]
|
|
68
|
+
if label:
|
|
69
|
+
cmd += ["-l", label]
|
|
70
|
+
r = _run(cmd, check=False)
|
|
71
|
+
if r.returncode != 0:
|
|
72
|
+
err = (r.stderr or r.stdout or "").strip() or f"exit {r.returncode}"
|
|
73
|
+
# 不回显可能含参数的完整命令
|
|
74
|
+
raise RuntimeError(f"Keychain 写入失败 ({service}): {err[:200]}")
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def set_agent_token(token: str, *, account: Optional[str] = None) -> None:
|
|
78
|
+
acct = account or os.environ.get("USER", "")
|
|
79
|
+
if not acct:
|
|
80
|
+
raise RuntimeError("无法确定 Keychain account($USER 为空)")
|
|
81
|
+
if not token.startswith("da_"):
|
|
82
|
+
raise ValueError("Agent token 应以 da_ 开头")
|
|
83
|
+
set_generic(AGENT_TOKEN_SERVICE, acct, token,
|
|
84
|
+
label="dataanalyst agent token")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def get_agent_token() -> str:
|
|
88
|
+
env = os.environ.get("DATAANALYST_AGENT_TOKEN", "").strip()
|
|
89
|
+
if env:
|
|
90
|
+
return env
|
|
91
|
+
acct = os.environ.get("USER", "")
|
|
92
|
+
try:
|
|
93
|
+
return get_password(AGENT_TOKEN_SERVICE, acct or None)
|
|
94
|
+
except RuntimeError:
|
|
95
|
+
# 兼容早期未指定 -a 的写法
|
|
96
|
+
return get_password(AGENT_TOKEN_SERVICE)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def set_rms_creds(shop: str, *, rlogin_id: str, rlogin_password: str,
|
|
100
|
+
sso_user: str, sso_password: str) -> None:
|
|
101
|
+
shop_u = shop.upper()
|
|
102
|
+
set_generic(rms_rlogin_service(shop_u), rlogin_id, rlogin_password,
|
|
103
|
+
label=f"{shop_u} R-Login")
|
|
104
|
+
set_generic(rms_ai_service(shop_u), sso_user, sso_password,
|
|
105
|
+
label=f"{shop_u} Rakuten SSO")
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def rms_creds_present(shop: str) -> dict:
|
|
109
|
+
"""只返回是否存在,不读出密码。"""
|
|
110
|
+
shop_u = shop.upper()
|
|
111
|
+
return {
|
|
112
|
+
"rlogin": has_generic(rms_rlogin_service(shop_u)),
|
|
113
|
+
"ai": has_generic(rms_ai_service(shop_u)),
|
|
114
|
+
"agent_token": has_generic(AGENT_TOKEN_SERVICE),
|
|
115
|
+
}
|