@geoly-ai/social-hub-cli 0.3.14 → 0.3.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/admin-index-gates.test.js +0 -1
- package/dist/admin-index-gates.test.js.map +1 -1
- package/dist/client-delivery.test.js +40 -1
- package/dist/client-delivery.test.js.map +1 -1
- package/dist/cmd-manifest.json +8 -11
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +142 -113
- package/dist/index.js.map +1 -1
- package/dist/index.test.js +1 -18
- package/dist/index.test.js.map +1 -1
- package/dist/permissions-gates-admin.d.ts.map +1 -1
- package/dist/permissions-gates-admin.js +0 -5
- package/dist/permissions-gates-admin.js.map +1 -1
- package/package.json +2 -2
- package/skills/README.md +1 -1
- package/skills/manifest.json +2 -2
- package/skills/reddit-matrix.lock.json +1 -1
- package/skills/reddit-voc-volume/SKILL.md +54 -10
- package/skills/reddit-voc-volume/matrix-contract.md +9 -0
- package/skills/reddit-voc-volume/references/arctic-shift.md +38 -3
- package/skills/reddit-voc-volume/references/hub-first.md +9 -5
- package/skills/reddit-voc-volume/scripts/reddit_voc_volume.py +516 -3
- package/skills/reddit-voc-volume/tests/test_archived_comments.py +189 -0
- package/skills/social-hub-admin/SKILL.md +0 -1
- package/skills/social-hub-cli/SKILL.md +2 -2
- package/skills/social-hub-events-observability/SKILL.md +2 -11
- package/skills/social-hub-intelligence/SKILL.md +5 -0
- package/skills/social-hub-migration/SKILL.md +0 -9
- package/skills/social-hub-publishing/SKILL.md +1 -2
- package/skills/social-hub-subreddit-pools/SKILL.md +34 -13
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
"""
|
|
3
|
-
Reddit VOC volume sampler — keyword / subreddit / links / stickies modes.
|
|
3
|
+
Reddit VOC volume sampler — keyword / subreddit / links / comments / stickies modes.
|
|
4
4
|
Strict rate limits; browser-like JSON requests; no bulk crawling.
|
|
5
5
|
"""
|
|
6
6
|
from __future__ import annotations
|
|
@@ -42,6 +42,8 @@ RETRY_BACKOFF_SEC = (30.0, 120.0)
|
|
|
42
42
|
ARCTIC_BASE_URL = "https://arctic-shift.photon-reddit.com"
|
|
43
43
|
ARCTIC_MAX_LIMIT = 100 # 单次上限;文档另有 "auto",但会随服务器负载浮动,不用
|
|
44
44
|
ARCTIC_IDS_MAX = 500
|
|
45
|
+
ARCTIC_COMMENT_TREE_MAX = 25_000
|
|
46
|
+
ARCTIC_COMMENT_SEARCH_MAX = 100
|
|
45
47
|
ARCTIC_DEFAULT_TIMEOUT_SEC = 60.0
|
|
46
48
|
ARCTIC_MAX_RETRIES = 3
|
|
47
49
|
ARCTIC_MIN_SLEEP_SEC = 2.0 # 作者要求 "a couple requests per second" 以内,取更保守值
|
|
@@ -143,7 +145,7 @@ SOCIAL_HUB_REQUIRED_SKILLS = ("social-hub-cli", "social-hub-shared")
|
|
|
143
145
|
HUB_DEFAULT_POLL_TIMEOUT_SEC = 600.0
|
|
144
146
|
HUB_POLL_BACKOFF_SEC = (5.0, 10.0, 20.0, 30.0)
|
|
145
147
|
HUB_INTELLIGENCE_TIMEOUT_SEC = 30.0
|
|
146
|
-
COLLECTOR_VERSION = "2.12.
|
|
148
|
+
COLLECTOR_VERSION = "2.12.31"
|
|
147
149
|
ACQUISITION_POLICIES = ("hub-first", "hub-only", "self-first")
|
|
148
150
|
DEFAULT_FLAIR_FILTER_PAGES = ("", "hot", "new", "top")
|
|
149
151
|
|
|
@@ -3184,6 +3186,7 @@ def arctic_get(
|
|
|
3184
3186
|
user_agent: str,
|
|
3185
3187
|
timeout: float = ARCTIC_DEFAULT_TIMEOUT_SEC,
|
|
3186
3188
|
tracker: "RequestTracker | None" = None,
|
|
3189
|
+
keep_empty_params: set[str] | None = None,
|
|
3187
3190
|
) -> dict[str, Any]:
|
|
3188
3191
|
"""调用 Arctic Shift API,处理 429 限流与重查询超时。
|
|
3189
3192
|
|
|
@@ -3191,7 +3194,12 @@ def arctic_get(
|
|
|
3191
3194
|
`{"error": "Timeout. Maybe slow down a bit"}` —— 那是查询代价过高而非
|
|
3192
3195
|
限流,同样退避重试(实测跨 15 年的 aggregate 会稳定触发)。
|
|
3193
3196
|
"""
|
|
3194
|
-
|
|
3197
|
+
keep_empty_params = keep_empty_params or set()
|
|
3198
|
+
clean = {
|
|
3199
|
+
k: v
|
|
3200
|
+
for k, v in params.items()
|
|
3201
|
+
if v is not None and (v != "" or k in keep_empty_params)
|
|
3202
|
+
}
|
|
3195
3203
|
url = f"{ARCTIC_BASE_URL}{path}?{urllib.parse.urlencode(clean)}"
|
|
3196
3204
|
last_err = ""
|
|
3197
3205
|
# 预算按**逻辑请求**计一次,不按重试次数计:重试是同一次取数的补偿,
|
|
@@ -3277,6 +3285,306 @@ def arctic_search_posts(
|
|
|
3277
3285
|
return [arctic_post_item(r) for r in rows if isinstance(r, dict)]
|
|
3278
3286
|
|
|
3279
3287
|
|
|
3288
|
+
_ARCTIC_BASE36_ID_RE = re.compile(r"^[0-9a-z]+$", re.I)
|
|
3289
|
+
|
|
3290
|
+
|
|
3291
|
+
def normalize_reddit_post_id(target: str) -> str:
|
|
3292
|
+
"""Extract a bare base36 post ID from a Reddit URL, ``t3_`` ID, or bare ID."""
|
|
3293
|
+
raw = str(target or "").strip()
|
|
3294
|
+
if not raw:
|
|
3295
|
+
raise ValueError("post URL or post ID is required (unless --ids is used)")
|
|
3296
|
+
|
|
3297
|
+
# Full Reddit permalinks and redd.it short links. Reject lookalike hosts rather
|
|
3298
|
+
# than accepting any URL that happens to contain `/comments/<id>`.
|
|
3299
|
+
if "://" in raw:
|
|
3300
|
+
parsed = urllib.parse.urlparse(raw)
|
|
3301
|
+
host = (parsed.hostname or "").lower().rstrip(".")
|
|
3302
|
+
if host == "redd.it" or host.endswith(".redd.it"):
|
|
3303
|
+
candidate = parsed.path.strip("/").split("/", 1)[0]
|
|
3304
|
+
elif host == "reddit.com" or host.endswith(".reddit.com"):
|
|
3305
|
+
match = re.search(r"/comments/([0-9a-z]+)(?:/|$)", parsed.path, re.I)
|
|
3306
|
+
candidate = match.group(1) if match else ""
|
|
3307
|
+
else:
|
|
3308
|
+
raise ValueError(f"unsupported post URL host: {host or '<missing>'}")
|
|
3309
|
+
elif "/comments/" in raw:
|
|
3310
|
+
match = re.search(r"/comments/([0-9a-z]+)(?:/|$)", raw, re.I)
|
|
3311
|
+
candidate = match.group(1) if match else ""
|
|
3312
|
+
else:
|
|
3313
|
+
candidate = raw[3:] if raw.lower().startswith("t3_") else raw
|
|
3314
|
+
if raw.lower().startswith("t1_"):
|
|
3315
|
+
raise ValueError("expected a post ID (t3_), received a comment ID (t1_)")
|
|
3316
|
+
|
|
3317
|
+
candidate = candidate.strip().lower()
|
|
3318
|
+
if not candidate or not _ARCTIC_BASE36_ID_RE.fullmatch(candidate):
|
|
3319
|
+
raise ValueError(f"malformed Reddit post ID: {raw}")
|
|
3320
|
+
return candidate
|
|
3321
|
+
|
|
3322
|
+
|
|
3323
|
+
def normalize_reddit_comment_id(raw_id: str) -> str:
|
|
3324
|
+
"""Normalize one comment ID to its bare base36 representation."""
|
|
3325
|
+
raw = str(raw_id or "").strip()
|
|
3326
|
+
if raw.lower().startswith("t3_"):
|
|
3327
|
+
raise ValueError(f"expected a comment ID (t1_), received a post ID: {raw}")
|
|
3328
|
+
candidate = raw[3:] if raw.lower().startswith("t1_") else raw
|
|
3329
|
+
candidate = candidate.strip().lower()
|
|
3330
|
+
if not candidate or not _ARCTIC_BASE36_ID_RE.fullmatch(candidate):
|
|
3331
|
+
raise ValueError(f"malformed Reddit comment ID: {raw or '<empty>'}")
|
|
3332
|
+
return candidate
|
|
3333
|
+
|
|
3334
|
+
|
|
3335
|
+
def parse_reddit_comment_ids(values: list[str] | None) -> list[str]:
|
|
3336
|
+
"""Parse comma- or whitespace-separated comment IDs, preserving first-seen order."""
|
|
3337
|
+
ids: list[str] = []
|
|
3338
|
+
seen: set[str] = set()
|
|
3339
|
+
for value in values or []:
|
|
3340
|
+
for token in str(value).split(","):
|
|
3341
|
+
if not token.strip():
|
|
3342
|
+
raise ValueError("malformed Reddit comment ID: empty value")
|
|
3343
|
+
comment_id = normalize_reddit_comment_id(token)
|
|
3344
|
+
if comment_id not in seen:
|
|
3345
|
+
ids.append(comment_id)
|
|
3346
|
+
seen.add(comment_id)
|
|
3347
|
+
if len(ids) > ARCTIC_IDS_MAX:
|
|
3348
|
+
raise ValueError(f"too many comment IDs: {len(ids)} (maximum {ARCTIC_IDS_MAX})")
|
|
3349
|
+
return ids
|
|
3350
|
+
|
|
3351
|
+
|
|
3352
|
+
def normalize_comment_parent_id(parent_id: str | None, *, allow_top_level: bool = False) -> str | None:
|
|
3353
|
+
"""Normalize a parent comment ID; search uses an empty value for top-level comments."""
|
|
3354
|
+
if parent_id is None:
|
|
3355
|
+
return None
|
|
3356
|
+
raw = str(parent_id).strip()
|
|
3357
|
+
if raw.lower() in ("top", "root"):
|
|
3358
|
+
if allow_top_level:
|
|
3359
|
+
return ""
|
|
3360
|
+
raise ValueError("--parent-id top/root is only valid with --source search")
|
|
3361
|
+
return f"t1_{normalize_reddit_comment_id(raw)}"
|
|
3362
|
+
|
|
3363
|
+
|
|
3364
|
+
def arctic_comment_item(
|
|
3365
|
+
row: dict[str, Any],
|
|
3366
|
+
*,
|
|
3367
|
+
depth: int | None = None,
|
|
3368
|
+
tree_order: int | None = None,
|
|
3369
|
+
sibling_index: int | None = None,
|
|
3370
|
+
tree_path: list[int] | None = None,
|
|
3371
|
+
tree_parent_id: str | None = None,
|
|
3372
|
+
) -> dict[str, Any]:
|
|
3373
|
+
"""Map an Arctic comment row without truncating its archived body."""
|
|
3374
|
+
body = row.get("body")
|
|
3375
|
+
body_text = body if isinstance(body, str) else ("" if body is None else str(body))
|
|
3376
|
+
body_status = classify_body(body_text)
|
|
3377
|
+
measurable = body_status not in ("Removed", "Deleted")
|
|
3378
|
+
comment_id = str(row.get("id") or "").removeprefix("t1_").lower()
|
|
3379
|
+
permalink = str(row.get("permalink") or "")
|
|
3380
|
+
if permalink.startswith("/"):
|
|
3381
|
+
permalink = f"https://www.reddit.com{permalink}"
|
|
3382
|
+
item = {
|
|
3383
|
+
"item_type": "comment",
|
|
3384
|
+
"id": comment_id,
|
|
3385
|
+
"name": str(row.get("name") or (f"t1_{comment_id}" if comment_id else "")),
|
|
3386
|
+
"link_id": row.get("link_id"),
|
|
3387
|
+
# Preserve Arctic's original edge exactly. `tree_parent_id` separately records
|
|
3388
|
+
# the edge observed while traversing the nested tree, useful for consistency checks.
|
|
3389
|
+
"parent_id": row.get("parent_id"),
|
|
3390
|
+
"tree_parent_id": tree_parent_id,
|
|
3391
|
+
"author": row.get("author"),
|
|
3392
|
+
"created_utc": row.get("created_utc"),
|
|
3393
|
+
"score": row.get("score"),
|
|
3394
|
+
"score_status": "ArchivedSnapshot",
|
|
3395
|
+
"score_realtime": False,
|
|
3396
|
+
"subreddit": row.get("subreddit"),
|
|
3397
|
+
"body": body_text,
|
|
3398
|
+
"body_status": body_status,
|
|
3399
|
+
"body_chars": len(body_text) if measurable else 0,
|
|
3400
|
+
"body_words": len(re.findall(r"\S+", body_text)) if measurable else 0,
|
|
3401
|
+
"body_truncated": False if measurable else None,
|
|
3402
|
+
"permalink": permalink or None,
|
|
3403
|
+
"retrieved_on": row.get("retrieved_on"),
|
|
3404
|
+
"distinguished": row.get("distinguished"),
|
|
3405
|
+
"is_submitter": row.get("is_submitter"),
|
|
3406
|
+
"edited": row.get("edited"),
|
|
3407
|
+
"depth": depth,
|
|
3408
|
+
"tree_order": tree_order,
|
|
3409
|
+
"sibling_index": sibling_index,
|
|
3410
|
+
"tree_path": tree_path,
|
|
3411
|
+
"source_fetch": "arctic_shift",
|
|
3412
|
+
}
|
|
3413
|
+
return item
|
|
3414
|
+
|
|
3415
|
+
|
|
3416
|
+
def _arctic_tree_children(replies: Any) -> list[dict[str, Any]]:
|
|
3417
|
+
if replies in (None, ""):
|
|
3418
|
+
return []
|
|
3419
|
+
if not isinstance(replies, dict):
|
|
3420
|
+
raise ArcticError("arctic_comments_tree_malformed_payload: replies must be a listing")
|
|
3421
|
+
data = replies.get("data")
|
|
3422
|
+
if not isinstance(data, dict):
|
|
3423
|
+
raise ArcticError("arctic_comments_tree_malformed_payload: replies.data must be an object")
|
|
3424
|
+
children = data.get("children")
|
|
3425
|
+
if not isinstance(children, list) or any(not isinstance(child, dict) for child in children):
|
|
3426
|
+
raise ArcticError("arctic_comments_tree_malformed_payload: replies.data.children must be object[]")
|
|
3427
|
+
return children
|
|
3428
|
+
|
|
3429
|
+
|
|
3430
|
+
def flatten_arctic_comment_tree(payload: Any) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
3431
|
+
"""Flatten Arctic's Reddit-shaped comment tree while retaining exact tree coordinates."""
|
|
3432
|
+
if not isinstance(payload, dict) or not isinstance(payload.get("data"), list):
|
|
3433
|
+
raise ArcticError("arctic_comments_tree_malformed_payload: expected data[]")
|
|
3434
|
+
roots = payload.get("data")
|
|
3435
|
+
if any(not isinstance(node, dict) for node in roots):
|
|
3436
|
+
raise ArcticError("arctic_comments_tree_malformed_payload: data must contain object nodes")
|
|
3437
|
+
nodes = roots
|
|
3438
|
+
items: list[dict[str, Any]] = []
|
|
3439
|
+
collapsed: list[dict[str, Any]] = []
|
|
3440
|
+
order = 0
|
|
3441
|
+
|
|
3442
|
+
def walk(children: list[dict[str, Any]], *, depth: int, path: list[int], parent_id: str | None) -> None:
|
|
3443
|
+
nonlocal order
|
|
3444
|
+
for sibling_index, node in enumerate(children):
|
|
3445
|
+
kind = str(node.get("kind") or "")
|
|
3446
|
+
data = node.get("data")
|
|
3447
|
+
if not isinstance(data, dict):
|
|
3448
|
+
raise ArcticError("arctic_comments_tree_malformed_payload: node.data must be an object")
|
|
3449
|
+
node_path = [*path, sibling_index]
|
|
3450
|
+
node_order = order
|
|
3451
|
+
order += 1
|
|
3452
|
+
if kind == "more":
|
|
3453
|
+
child_ids = [
|
|
3454
|
+
normalize_reddit_comment_id(str(value))
|
|
3455
|
+
for value in (data.get("children") or [])
|
|
3456
|
+
if str(value or "").strip()
|
|
3457
|
+
]
|
|
3458
|
+
collapsed.append(
|
|
3459
|
+
{
|
|
3460
|
+
"kind": "more",
|
|
3461
|
+
"id": data.get("id"),
|
|
3462
|
+
"parent_id": data.get("parent_id") or parent_id,
|
|
3463
|
+
"count": data.get("count"),
|
|
3464
|
+
"children": child_ids,
|
|
3465
|
+
"depth": depth,
|
|
3466
|
+
"tree_order": node_order,
|
|
3467
|
+
"sibling_index": sibling_index,
|
|
3468
|
+
"tree_path": node_path,
|
|
3469
|
+
}
|
|
3470
|
+
)
|
|
3471
|
+
continue
|
|
3472
|
+
if kind != "t1":
|
|
3473
|
+
raise ArcticError(f"arctic_comments_tree_malformed_payload: unexpected kind={kind or '<empty>'}")
|
|
3474
|
+
|
|
3475
|
+
replies = _arctic_tree_children(data.get("replies"))
|
|
3476
|
+
item = arctic_comment_item(
|
|
3477
|
+
data,
|
|
3478
|
+
depth=depth,
|
|
3479
|
+
tree_order=node_order,
|
|
3480
|
+
sibling_index=sibling_index,
|
|
3481
|
+
tree_path=node_path,
|
|
3482
|
+
tree_parent_id=parent_id,
|
|
3483
|
+
)
|
|
3484
|
+
item["child_ids"] = [
|
|
3485
|
+
str(child.get("data", {}).get("id") or "").removeprefix("t1_").lower()
|
|
3486
|
+
for child in replies
|
|
3487
|
+
if child.get("kind") == "t1" and isinstance(child.get("data"), dict)
|
|
3488
|
+
]
|
|
3489
|
+
items.append(item)
|
|
3490
|
+
current_parent = str(data.get("name") or (f"t1_{item['id']}" if item.get("id") else "")) or None
|
|
3491
|
+
walk(replies, depth=depth + 1, path=node_path, parent_id=current_parent)
|
|
3492
|
+
|
|
3493
|
+
walk(nodes, depth=0, path=[], parent_id=None)
|
|
3494
|
+
return items, collapsed
|
|
3495
|
+
|
|
3496
|
+
|
|
3497
|
+
def arctic_comments_tree(
|
|
3498
|
+
post_id: str,
|
|
3499
|
+
*,
|
|
3500
|
+
parent_id: str | None = None,
|
|
3501
|
+
limit: int = ARCTIC_COMMENT_TREE_MAX,
|
|
3502
|
+
start_breadth: int | None = None,
|
|
3503
|
+
start_depth: int | None = None,
|
|
3504
|
+
user_agent: str,
|
|
3505
|
+
tracker: "RequestTracker | None" = None,
|
|
3506
|
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
3507
|
+
payload = arctic_get(
|
|
3508
|
+
"/api/comments/tree",
|
|
3509
|
+
{
|
|
3510
|
+
"link_id": f"t3_{post_id}",
|
|
3511
|
+
"parent_id": parent_id,
|
|
3512
|
+
"limit": limit,
|
|
3513
|
+
"start_breadth": start_breadth,
|
|
3514
|
+
"start_depth": start_depth,
|
|
3515
|
+
},
|
|
3516
|
+
user_agent=user_agent,
|
|
3517
|
+
tracker=tracker,
|
|
3518
|
+
)
|
|
3519
|
+
return flatten_arctic_comment_tree(payload)
|
|
3520
|
+
|
|
3521
|
+
|
|
3522
|
+
def arctic_search_comments(
|
|
3523
|
+
post_id: str,
|
|
3524
|
+
*,
|
|
3525
|
+
body: str | None = None,
|
|
3526
|
+
author: str | None = None,
|
|
3527
|
+
after: str | None = None,
|
|
3528
|
+
before: str | None = None,
|
|
3529
|
+
parent_id: str | None = None,
|
|
3530
|
+
limit: int = ARCTIC_COMMENT_SEARCH_MAX,
|
|
3531
|
+
sort: str = "asc",
|
|
3532
|
+
user_agent: str,
|
|
3533
|
+
tracker: "RequestTracker | None" = None,
|
|
3534
|
+
) -> list[dict[str, Any]]:
|
|
3535
|
+
payload = arctic_get(
|
|
3536
|
+
"/api/comments/search",
|
|
3537
|
+
{
|
|
3538
|
+
"link_id": post_id,
|
|
3539
|
+
"body": body,
|
|
3540
|
+
"author": author,
|
|
3541
|
+
"after": after,
|
|
3542
|
+
"before": before,
|
|
3543
|
+
"parent_id": parent_id,
|
|
3544
|
+
"limit": limit,
|
|
3545
|
+
"sort": sort,
|
|
3546
|
+
},
|
|
3547
|
+
user_agent=user_agent,
|
|
3548
|
+
tracker=tracker,
|
|
3549
|
+
keep_empty_params={"parent_id"} if parent_id == "" else None,
|
|
3550
|
+
)
|
|
3551
|
+
rows = payload.get("data")
|
|
3552
|
+
if not isinstance(rows, list):
|
|
3553
|
+
return []
|
|
3554
|
+
return [
|
|
3555
|
+
arctic_comment_item(row, tree_order=index)
|
|
3556
|
+
for index, row in enumerate(rows)
|
|
3557
|
+
if isinstance(row, dict)
|
|
3558
|
+
]
|
|
3559
|
+
|
|
3560
|
+
|
|
3561
|
+
def arctic_comments_by_ids(
|
|
3562
|
+
comment_ids: list[str],
|
|
3563
|
+
*,
|
|
3564
|
+
user_agent: str,
|
|
3565
|
+
tracker: "RequestTracker | None" = None,
|
|
3566
|
+
) -> tuple[list[dict[str, Any]], list[str]]:
|
|
3567
|
+
payload = arctic_get(
|
|
3568
|
+
"/api/comments/ids",
|
|
3569
|
+
{"ids": ",".join(comment_ids)},
|
|
3570
|
+
user_agent=user_agent,
|
|
3571
|
+
tracker=tracker,
|
|
3572
|
+
)
|
|
3573
|
+
rows = payload.get("data")
|
|
3574
|
+
by_id = {
|
|
3575
|
+
str(row.get("id") or "").removeprefix("t1_").lower(): row
|
|
3576
|
+
for row in (rows if isinstance(rows, list) else [])
|
|
3577
|
+
if isinstance(row, dict)
|
|
3578
|
+
}
|
|
3579
|
+
items = [
|
|
3580
|
+
arctic_comment_item(by_id[comment_id], tree_order=index)
|
|
3581
|
+
for index, comment_id in enumerate(comment_ids)
|
|
3582
|
+
if comment_id in by_id
|
|
3583
|
+
]
|
|
3584
|
+
missing = [comment_id for comment_id in comment_ids if comment_id not in by_id]
|
|
3585
|
+
return items, missing
|
|
3586
|
+
|
|
3587
|
+
|
|
3280
3588
|
def arctic_subreddit_rules(subreddits: list[str], *, user_agent: str) -> dict[str, list[dict[str, Any]]]:
|
|
3281
3589
|
"""替代 /about/rules.json(本机对 reddit.com 直连被拦时唯一可用途径)。"""
|
|
3282
3590
|
names = [_clean_subreddit_name(s) for s in subreddits if s]
|
|
@@ -4571,6 +4879,43 @@ def main() -> int:
|
|
|
4571
4879
|
)
|
|
4572
4880
|
pl.add_argument("--pretty", action="store_true")
|
|
4573
4881
|
|
|
4882
|
+
pc = sub.add_parser(
|
|
4883
|
+
"comments",
|
|
4884
|
+
help="读取帖子历史评论正文:默认 Arctic 评论树,也支持筛选搜索与评论 ID 批查",
|
|
4885
|
+
)
|
|
4886
|
+
pc.add_argument("target", nargs="?", help="Reddit post URL、t3_<post_id> 或裸 post ID;--ids 模式省略")
|
|
4887
|
+
pc.add_argument(
|
|
4888
|
+
"--source",
|
|
4889
|
+
choices=["tree", "search"],
|
|
4890
|
+
default="tree",
|
|
4891
|
+
help="tree=评论树(默认,最多 25,000);search=平面筛选(最多 100)",
|
|
4892
|
+
)
|
|
4893
|
+
pc.add_argument(
|
|
4894
|
+
"--ids",
|
|
4895
|
+
nargs="+",
|
|
4896
|
+
metavar="COMMENT_ID",
|
|
4897
|
+
help=f"按 t1_/裸评论 ID 批查,可用空格或逗号分隔,最多 {ARCTIC_IDS_MAX}",
|
|
4898
|
+
)
|
|
4899
|
+
pc.add_argument("--body", help="search only:评论正文关键词/FTS 查询")
|
|
4900
|
+
pc.add_argument("--author", help="search only:作者")
|
|
4901
|
+
pc.add_argument("--after", help="search only:起始时间(ISO 日期、epoch 或 Arctic offset)")
|
|
4902
|
+
pc.add_argument("--before", help="search only:结束时间(ISO 日期、epoch 或 Arctic offset)")
|
|
4903
|
+
pc.add_argument(
|
|
4904
|
+
"--parent-id",
|
|
4905
|
+
help="tree/search:父评论 ID;search 中 top/root 表示顶层评论",
|
|
4906
|
+
)
|
|
4907
|
+
pc.add_argument(
|
|
4908
|
+
"--limit",
|
|
4909
|
+
type=int,
|
|
4910
|
+
default=None,
|
|
4911
|
+
help="tree 默认/上限 25000;search 默认/上限 100;越界直接报错",
|
|
4912
|
+
)
|
|
4913
|
+
pc.add_argument("--sort", choices=["asc", "desc"], default=None, help="search only:按 created_utc 排序")
|
|
4914
|
+
pc.add_argument("--start-breadth", type=int, default=None, help="tree only:Arctic 折叠宽度,必须 >=0")
|
|
4915
|
+
pc.add_argument("--start-depth", type=int, default=None, help="tree only:Arctic 折叠深度,必须 >=0")
|
|
4916
|
+
pc.add_argument("--user-agent", default=DEFAULT_UA)
|
|
4917
|
+
pc.add_argument("--pretty", action="store_true")
|
|
4918
|
+
|
|
4574
4919
|
ph = sub.add_parser(
|
|
4575
4920
|
"history",
|
|
4576
4921
|
help="历史检索:按时间窗从 Arctic Shift 归档取帖(覆盖 2005-06 起至近实时)",
|
|
@@ -4601,6 +4946,174 @@ def main() -> int:
|
|
|
4601
4946
|
psh.add_argument("--pretty", action="store_true")
|
|
4602
4947
|
|
|
4603
4948
|
args = p.parse_args()
|
|
4949
|
+
if args.mode == "comments":
|
|
4950
|
+
try:
|
|
4951
|
+
comment_ids = parse_reddit_comment_ids(args.ids) if args.ids else []
|
|
4952
|
+
if comment_ids and args.target:
|
|
4953
|
+
raise ValueError("provide either a post target or --ids, not both")
|
|
4954
|
+
if comment_ids and args.source != "tree":
|
|
4955
|
+
raise ValueError("--ids cannot be combined with --source search")
|
|
4956
|
+
if not comment_ids and not args.target:
|
|
4957
|
+
raise ValueError("post URL or post ID is required (unless --ids is used)")
|
|
4958
|
+
|
|
4959
|
+
filter_values = (args.body, args.author, args.after, args.before, args.sort)
|
|
4960
|
+
if comment_ids and (any(value is not None for value in filter_values) or args.parent_id is not None):
|
|
4961
|
+
raise ValueError("--ids cannot be combined with post/search/tree filters")
|
|
4962
|
+
if comment_ids and (args.limit is not None or args.start_breadth is not None or args.start_depth is not None):
|
|
4963
|
+
raise ValueError("--ids cannot be combined with --limit/--start-breadth/--start-depth")
|
|
4964
|
+
if args.source == "tree" and any(value is not None for value in filter_values):
|
|
4965
|
+
raise ValueError("--body/--author/--after/--before/--sort require --source search")
|
|
4966
|
+
if args.source == "search" and (args.start_breadth is not None or args.start_depth is not None):
|
|
4967
|
+
raise ValueError("--start-breadth/--start-depth are only valid with --source tree")
|
|
4968
|
+
if args.start_breadth is not None and args.start_breadth < 0:
|
|
4969
|
+
raise ValueError("--start-breadth must be >= 0")
|
|
4970
|
+
if args.start_depth is not None and args.start_depth < 0:
|
|
4971
|
+
raise ValueError("--start-depth must be >= 0")
|
|
4972
|
+
|
|
4973
|
+
post_id = normalize_reddit_post_id(args.target) if args.target else None
|
|
4974
|
+
parent_id = (
|
|
4975
|
+
normalize_comment_parent_id(args.parent_id, allow_top_level=args.source == "search")
|
|
4976
|
+
if args.parent_id is not None and post_id is not None
|
|
4977
|
+
else None
|
|
4978
|
+
)
|
|
4979
|
+
retrieval_mode = "ids" if comment_ids else args.source
|
|
4980
|
+
maximum = ARCTIC_COMMENT_TREE_MAX if retrieval_mode == "tree" else ARCTIC_COMMENT_SEARCH_MAX
|
|
4981
|
+
default_limit = ARCTIC_COMMENT_TREE_MAX if retrieval_mode == "tree" else ARCTIC_COMMENT_SEARCH_MAX
|
|
4982
|
+
limit = default_limit if args.limit is None else args.limit
|
|
4983
|
+
if retrieval_mode != "ids" and not 1 <= limit <= maximum:
|
|
4984
|
+
raise ValueError(f"--limit must be between 1 and {maximum} for {retrieval_mode} mode")
|
|
4985
|
+
except ValueError as exc:
|
|
4986
|
+
pc.error(str(exc))
|
|
4987
|
+
|
|
4988
|
+
tracker = RequestTracker()
|
|
4989
|
+
errors: list[str] = []
|
|
4990
|
+
collapsed: list[dict[str, Any]] = []
|
|
4991
|
+
missing_ids: list[str] = []
|
|
4992
|
+
items: list[dict[str, Any]] = []
|
|
4993
|
+
tree_fetch_succeeded = False
|
|
4994
|
+
try:
|
|
4995
|
+
if retrieval_mode == "ids":
|
|
4996
|
+
items, missing_ids = arctic_comments_by_ids(
|
|
4997
|
+
comment_ids,
|
|
4998
|
+
user_agent=args.user_agent,
|
|
4999
|
+
tracker=tracker,
|
|
5000
|
+
)
|
|
5001
|
+
elif retrieval_mode == "search":
|
|
5002
|
+
items = arctic_search_comments(
|
|
5003
|
+
post_id,
|
|
5004
|
+
body=args.body,
|
|
5005
|
+
author=args.author,
|
|
5006
|
+
after=args.after,
|
|
5007
|
+
before=args.before,
|
|
5008
|
+
parent_id=parent_id,
|
|
5009
|
+
limit=limit,
|
|
5010
|
+
sort=args.sort or "asc",
|
|
5011
|
+
user_agent=args.user_agent,
|
|
5012
|
+
tracker=tracker,
|
|
5013
|
+
)
|
|
5014
|
+
else:
|
|
5015
|
+
items, collapsed = arctic_comments_tree(
|
|
5016
|
+
post_id,
|
|
5017
|
+
parent_id=parent_id,
|
|
5018
|
+
limit=limit,
|
|
5019
|
+
start_breadth=args.start_breadth,
|
|
5020
|
+
start_depth=args.start_depth,
|
|
5021
|
+
user_agent=args.user_agent,
|
|
5022
|
+
tracker=tracker,
|
|
5023
|
+
)
|
|
5024
|
+
tree_fetch_succeeded = True
|
|
5025
|
+
except ArcticError as exc:
|
|
5026
|
+
errors.append(f"arctic_comments_failed:{exc}"[:300])
|
|
5027
|
+
if missing_ids:
|
|
5028
|
+
errors.append(f"comment_ids_not_found:{','.join(missing_ids)}"[:300])
|
|
5029
|
+
|
|
5030
|
+
bodies = [item for item in items if item.get("body_status") == "Present"]
|
|
5031
|
+
removed = [item for item in items if item.get("body_status") in ("Removed", "Deleted")]
|
|
5032
|
+
notes = [
|
|
5033
|
+
"Arctic Shift 是第三方历史归档源,通常滞后约 2 小时,且不保证可用性。",
|
|
5034
|
+
"评论 body 适合历史 VOC;score 是归档快照而非实时值,不得用于当前热度判断。",
|
|
5035
|
+
]
|
|
5036
|
+
if collapsed:
|
|
5037
|
+
notes.append(
|
|
5038
|
+
f"评论树含 {len(collapsed)} 个 kind=more 折叠节点;"
|
|
5039
|
+
"tree_complete=false,collapsed[].children 保留尚未展开的评论 ID。"
|
|
5040
|
+
)
|
|
5041
|
+
fetched_at = utc_now_iso()
|
|
5042
|
+
canonical_source = f"https://www.reddit.com/comments/{post_id}/" if post_id else None
|
|
5043
|
+
fingerprint = hashlib.sha256(
|
|
5044
|
+
json.dumps(
|
|
5045
|
+
{
|
|
5046
|
+
"mode": "comments",
|
|
5047
|
+
"retrieval_mode": retrieval_mode,
|
|
5048
|
+
"post_id": post_id,
|
|
5049
|
+
"comment_ids": comment_ids,
|
|
5050
|
+
"body": args.body,
|
|
5051
|
+
"author": args.author,
|
|
5052
|
+
"after": args.after,
|
|
5053
|
+
"before": args.before,
|
|
5054
|
+
"parent_id": parent_id,
|
|
5055
|
+
"limit": None if retrieval_mode == "ids" else limit,
|
|
5056
|
+
"sort": args.sort or ("asc" if retrieval_mode == "search" else None),
|
|
5057
|
+
"start_breadth": args.start_breadth,
|
|
5058
|
+
"start_depth": args.start_depth,
|
|
5059
|
+
},
|
|
5060
|
+
ensure_ascii=False,
|
|
5061
|
+
sort_keys=True,
|
|
5062
|
+
).encode("utf-8")
|
|
5063
|
+
).hexdigest()
|
|
5064
|
+
out = {
|
|
5065
|
+
"ok": not errors,
|
|
5066
|
+
"mode": "comments",
|
|
5067
|
+
"retrieval_mode": retrieval_mode,
|
|
5068
|
+
"provenance": "arctic_shift",
|
|
5069
|
+
"source_endpoint": f"/api/comments/{'ids' if retrieval_mode == 'ids' else retrieval_mode}",
|
|
5070
|
+
"source_urls": [canonical_source] if canonical_source else [],
|
|
5071
|
+
"post_id": post_id,
|
|
5072
|
+
"link_id": f"t3_{post_id}" if post_id else None,
|
|
5073
|
+
"requested_comment_ids": comment_ids,
|
|
5074
|
+
"missing_comment_ids": missing_ids,
|
|
5075
|
+
"filters": {
|
|
5076
|
+
"body": args.body,
|
|
5077
|
+
"author": args.author,
|
|
5078
|
+
"after": args.after,
|
|
5079
|
+
"before": args.before,
|
|
5080
|
+
"parent_id": parent_id,
|
|
5081
|
+
"sort": args.sort or ("asc" if retrieval_mode == "search" else None),
|
|
5082
|
+
},
|
|
5083
|
+
"count": len(items),
|
|
5084
|
+
"tree_complete": tree_fetch_succeeded and not collapsed if retrieval_mode == "tree" else None,
|
|
5085
|
+
"collapsed_count": len(collapsed),
|
|
5086
|
+
"collapsed": collapsed,
|
|
5087
|
+
"body_stats": {
|
|
5088
|
+
"with_text": len(bodies),
|
|
5089
|
+
"removed_or_deleted": len(removed),
|
|
5090
|
+
"max_chars": max((int(item.get("body_chars") or 0) for item in bodies), default=0),
|
|
5091
|
+
},
|
|
5092
|
+
"items": items,
|
|
5093
|
+
"archive_caveats": {
|
|
5094
|
+
"estimated_lag": "about_2_hours",
|
|
5095
|
+
"uptime_guaranteed": False,
|
|
5096
|
+
"score_realtime": False,
|
|
5097
|
+
},
|
|
5098
|
+
"rate_limit": {
|
|
5099
|
+
"requested_limit": len(comment_ids) if retrieval_mode == "ids" else limit,
|
|
5100
|
+
"effective_limit": len(comment_ids) if retrieval_mode == "ids" else limit,
|
|
5101
|
+
"http_requests": tracker.count,
|
|
5102
|
+
"rate_limited": tracker.rate_limited,
|
|
5103
|
+
},
|
|
5104
|
+
"fetched_at": fetched_at,
|
|
5105
|
+
"fetchedAt": fetched_at,
|
|
5106
|
+
"fingerprint": fingerprint,
|
|
5107
|
+
"runId": None,
|
|
5108
|
+
"hubRunId": None,
|
|
5109
|
+
"fallbackReason": None,
|
|
5110
|
+
"writebackStatus": "not_applicable",
|
|
5111
|
+
"notes": notes,
|
|
5112
|
+
"errors": errors,
|
|
5113
|
+
}
|
|
5114
|
+
print(json.dumps(out, ensure_ascii=False, indent=2 if args.pretty else None))
|
|
5115
|
+
return 0 if out["ok"] else 1
|
|
5116
|
+
|
|
4604
5117
|
if args.mode == "history":
|
|
4605
5118
|
subs = [_clean_subreddit_name(s) for s in (args.subreddit or [])] or [None]
|
|
4606
5119
|
items: list[dict[str, Any]] = []
|