luckiest-co 1.0.47 → 1.0.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/package.json +1 -1
- package/skills/ab-testing/CHANGELOG.md +5 -0
- package/skills/ab-testing/SKILL.md +12 -9
- package/skills/ab-testing/evals/evals.json +18 -7
- package/skills/ab-testing/references/sample-size-guide.md +2 -2
- package/skills/ads/CHANGELOG.md +5 -0
- package/skills/ads/SKILL.md +4 -4
- package/skills/ai-seo/CHANGELOG.md +7 -0
- package/skills/ai-seo/SKILL.md +27 -20
- package/skills/analytics/CHANGELOG.md +6 -0
- package/skills/analytics/SKILL.md +4 -3
- package/skills/analytics/evals/evals.json +27 -0
- package/skills/analytics/references/ga4-implementation.md +7 -3
- package/skills/analytics/references/gtm-implementation.md +8 -2
- package/skills/aso/CHANGELOG.md +5 -0
- package/skills/aso/SKILL.md +4 -4
- package/skills/aso/evals/evals.json +15 -2
- package/skills/aso/references/benchmarks.md +4 -2
- package/skills/aso/references/google-play-specs.md +5 -1
- package/skills/brainstorm/SKILL.md +59 -16
- package/skills/brainstorm/builder-check-prompt.md +23 -0
- package/skills/churn-prevention/CHANGELOG.md +5 -0
- package/skills/churn-prevention/SKILL.md +3 -3
- package/skills/churn-prevention/evals/evals.json +14 -0
- package/skills/churn-prevention/references/dunning-playbook.md +6 -1
- package/skills/coder-brainstorming/CHANGELOG.md +10 -0
- package/skills/coder-brainstorming/SKILL.md +62 -20
- package/skills/coder-brainstorming/builder-check-prompt.md +23 -0
- package/skills/coder-brainstorming/scripts/server.cjs +1 -1
- package/skills/coder-debugging/CHANGELOG.md +4 -0
- package/skills/coder-debugging/SKILL.md +5 -5
- package/skills/coder-requesting-code-review/CHANGELOG.md +4 -0
- package/skills/coder-requesting-code-review/SKILL.md +4 -4
- package/skills/coder-subagent-driven-development/CHANGELOG.md +4 -0
- package/skills/coder-subagent-driven-development/SKILL.md +3 -3
- package/skills/coder-subagent-driven-development/scripts/sdd-workspace +3 -1
- package/skills/cold-email/CHANGELOG.md +7 -0
- package/skills/cold-email/SKILL.md +27 -7
- package/skills/cold-email/evals/evals.json +30 -0
- package/skills/cold-email/references/benchmarks.md +15 -11
- package/skills/cold-email/references/deliverability.md +110 -0
- package/skills/cold-email/references/linkedin-outreach.md +91 -0
- package/skills/cold-email/references/multichannel-cadence.md +79 -0
- package/skills/cold-email/references/outbound-plays.md +88 -0
- package/skills/cold-email/references/personalization.md +11 -0
- package/skills/cold-email/references/reply-handling.md +65 -0
- package/skills/competitor-profiling/CHANGELOG.md +5 -0
- package/skills/competitor-profiling/SKILL.md +4 -4
- package/skills/content-strategy/CHANGELOG.md +5 -0
- package/skills/content-strategy/SKILL.md +5 -3
- package/skills/copy-editing/CHANGELOG.md +5 -0
- package/skills/copy-editing/SKILL.md +5 -5
- package/skills/copy-editing/references/content-refresh.md +33 -0
- package/skills/cro/CHANGELOG.md +5 -0
- package/skills/cro/SKILL.md +14 -3
- package/skills/cro/references/form.md +1 -1
- package/skills/directory-submissions/CHANGELOG.md +5 -0
- package/skills/directory-submissions/SKILL.md +13 -4
- package/skills/directory-submissions/evals/evals.json +15 -1
- package/skills/directory-submissions/references/directory-list.md +2 -4
- package/skills/emails/CHANGELOG.md +5 -0
- package/skills/emails/SKILL.md +14 -4
- package/skills/launch/CHANGELOG.md +7 -0
- package/skills/launch/SKILL.md +8 -4
- package/skills/launch/evals/evals.json +29 -0
- package/skills/launch/references/shipped-changes.md +84 -0
- package/skills/launch/references/site-launch-qa.md +55 -0
- package/skills/lead-magnets/CHANGELOG.md +5 -0
- package/skills/lead-magnets/SKILL.md +6 -4
- package/skills/lead-magnets/evals/evals.json +18 -0
- package/skills/lead-magnets/references/content-to-skill-corpus-design.md +89 -0
- package/skills/lead-magnets/references/content-to-skill-release.md +71 -0
- package/skills/lead-magnets/references/content-to-skill.md +108 -0
- package/skills/marketing-plan/CHANGELOG.md +5 -0
- package/skills/marketing-plan/SKILL.md +3 -3
- package/skills/marketing-plan/evals/evals.json +13 -0
- package/skills/marketing-plan/references/budget-planning.md +26 -17
- package/skills/model-router/CHANGELOG.md +6 -0
- package/skills/model-router/SKILL.md +3 -3
- package/skills/model-router/scripts/route.py +1 -1
- package/skills/model-router/scripts/test_route.py +5 -0
- package/skills/pricing/CHANGELOG.md +5 -0
- package/skills/pricing/SKILL.md +4 -4
- package/skills/pricing/evals/evals.json +12 -0
- package/skills/pricing/references/research-methods.md +8 -4
- package/skills/programmatic-seo/CHANGELOG.md +5 -0
- package/skills/programmatic-seo/SKILL.md +8 -4
- package/skills/programmatic-seo/evals/evals.json +13 -0
- package/skills/programmatic-seo/references/implementation-platforms.md +37 -0
- package/skills/prospecting/CHANGELOG.md +7 -0
- package/skills/prospecting/SKILL.md +27 -13
- package/skills/prospecting/evals/evals.json +41 -0
- package/skills/prospecting/references/account-research.md +59 -0
- package/skills/prospecting/references/b2b-prospecting.md +1 -1
- package/skills/prospecting/references/compliance.md +44 -6
- package/skills/prospecting/references/data-sources.md +13 -15
- package/skills/prospecting/references/enrichment-playbook.md +61 -0
- package/skills/prospecting/references/saas-prospecting.md +1 -1
- package/skills/prospecting/references/signal-plays.md +99 -0
- package/skills/prospecting/references/sourcing-and-enrichment.md +89 -0
- package/skills/referrals/CHANGELOG.md +5 -0
- package/skills/referrals/SKILL.md +10 -4
- package/skills/referrals/evals/evals.json +99 -0
- package/skills/referrals/references/affiliate-operations.md +64 -0
- package/skills/referrals/references/affiliate-programs.md +2 -2
- package/skills/referrals/references/program-examples.md +14 -6
- package/skills/referrals/references/review-platform-rules.md +36 -0
- package/skills/referrals/references/review-program-operations.md +62 -0
- package/skills/referrals/references/review-programs.md +96 -0
- package/skills/revops/CHANGELOG.md +5 -0
- package/skills/revops/SKILL.md +10 -6
- package/skills/revops/evals/evals.json +13 -0
- package/skills/revops/references/automation-playbooks.md +1 -1
- package/skills/revops/references/inbound-routing.md +43 -0
- package/skills/revops/references/lifecycle-definitions.md +31 -0
- package/skills/sales-enablement/CHANGELOG.md +5 -0
- package/skills/sales-enablement/SKILL.md +6 -4
- package/skills/sales-enablement/evals/evals.json +14 -0
- package/skills/sales-enablement/references/cold-call-scripts.md +68 -0
- package/skills/schema/CHANGELOG.md +5 -0
- package/skills/schema/SKILL.md +20 -8
- package/skills/schema/evals/evals.json +40 -0
- package/skills/schema/references/schema-examples.md +9 -15
- package/skills/seo-audit/CHANGELOG.md +5 -0
- package/skills/seo-audit/SKILL.md +10 -16
- package/skills/seo-audit/evals/evals.json +68 -0
- package/skills/seo-audit/references/international-seo.md +9 -11
- package/skills/seo-audit/references/local-seo-pages-and-measurement.md +39 -0
- package/skills/seo-audit/references/local-seo-profile-and-citations.md +41 -0
- package/skills/seo-audit/references/local-seo.md +86 -0
- package/skills/seo-audit/references/rankings-push.md +62 -0
- package/skills/seo-audit/references/title-tags.md +89 -0
- package/skills/signup/CHANGELOG.md +5 -0
- package/skills/signup/SKILL.md +7 -3
- package/skills/site-architecture/CHANGELOG.md +5 -0
- package/skills/site-architecture/SKILL.md +5 -4
- package/skills/site-architecture/evals/evals.json +25 -0
- package/skills/site-architecture/references/platforms-and-migration.md +84 -0
- package/skills/social/CHANGELOG.md +5 -0
- package/skills/social/SKILL.md +4 -4
- package/skills/trends/.skillignore +3 -0
- package/skills/trends/CHANGELOG.md +20 -0
- package/skills/trends/SKILL.md +12 -8
- package/skills/trends/references/agent-mode.md +36 -0
- package/skills/trends/references/category-peers.md +38 -0
- package/skills/trends/references/comparison.md +227 -0
- package/skills/trends/references/competitors.md +29 -0
- package/skills/trends/references/discovery.md +90 -0
- package/skills/trends/references/engine-contract.md +111 -1889
- package/skills/trends/references/followups.md +112 -0
- package/skills/trends/references/grok-bot-x.md +55 -0
- package/skills/trends/references/hiring-signals.md +19 -0
- package/skills/trends/references/library-queue.md +33 -0
- package/skills/trends/references/recommendations.md +86 -0
- package/skills/trends/references/research-runbook.md +781 -0
- package/skills/trends/references/runtime.md +165 -0
- package/skills/trends/references/save-html-brief.md +39 -20
- package/skills/trends/references/setup-wizard.md +253 -0
- package/skills/trends/references/synthesis.md +289 -0
- package/skills/trends/scripts/box_chrome_login.py +209 -0
- package/skills/trends/scripts/briefing.py +9 -17
- package/skills/trends/scripts/last30days.py +3247 -280
- package/skills/trends/scripts/lib/agentcookie.py +196 -0
- package/skills/trends/scripts/lib/amazon.py +877 -0
- package/skills/trends/scripts/lib/arxiv.py +42 -8
- package/skills/trends/scripts/lib/backends.py +769 -0
- package/skills/trends/scripts/lib/bird_x.py +340 -57
- package/skills/trends/scripts/lib/bluesky.py +84 -10
- package/skills/trends/scripts/lib/bounded_get.py +156 -0
- package/skills/trends/scripts/lib/brightdata.py +291 -0
- package/skills/trends/scripts/lib/categories.py +7 -1
- package/skills/trends/scripts/lib/chrome_cdp.py +455 -0
- package/skills/trends/scripts/lib/chrome_cookies.py +132 -39
- package/skills/trends/scripts/lib/cluster.py +31 -54
- package/skills/trends/scripts/lib/competitors.py +10 -1
- package/skills/trends/scripts/lib/cookie_extract.py +221 -44
- package/skills/trends/scripts/lib/cookie_paths.py +27 -0
- package/skills/trends/scripts/lib/corpus.py +439 -0
- package/skills/trends/scripts/lib/devto.py +313 -0
- package/skills/trends/scripts/lib/discovery_handoff.py +988 -0
- package/skills/trends/scripts/lib/doctor.py +2166 -0
- package/skills/trends/scripts/lib/dripstack.py +203 -0
- package/skills/trends/scripts/lib/entity_extract.py +37 -0
- package/skills/trends/scripts/lib/env.py +1092 -138
- package/skills/trends/scripts/lib/fanout.py +11 -1
- package/skills/trends/scripts/lib/feed.py +61 -0
- package/skills/trends/scripts/lib/fetch_memo.py +135 -0
- package/skills/trends/scripts/lib/freshness.py +566 -0
- package/skills/trends/scripts/lib/fusion.py +314 -28
- package/skills/trends/scripts/lib/github.py +438 -24
- package/skills/trends/scripts/lib/grok_x.py +1237 -0
- package/skills/trends/scripts/lib/grounding.py +129 -11
- package/skills/trends/scripts/lib/hackernews.py +61 -4
- package/skills/trends/scripts/lib/health.py +373 -0
- package/skills/trends/scripts/lib/hosted.py +333 -0
- package/skills/trends/scripts/lib/html_publish.py +34 -0
- package/skills/trends/scripts/lib/html_render.py +169 -3
- package/skills/trends/scripts/lib/http.py +1215 -70
- package/skills/trends/scripts/lib/instagram.py +132 -9
- package/skills/trends/scripts/lib/jobs.py +4 -2
- package/skills/trends/scripts/lib/library.py +308 -0
- package/skills/trends/scripts/lib/library_index.py +477 -0
- package/skills/trends/scripts/lib/linkedin.py +271 -24
- package/skills/trends/scripts/lib/log.py +5 -2
- package/skills/trends/scripts/lib/meta_ads.py +982 -0
- package/skills/trends/scripts/lib/normalize.py +382 -28
- package/skills/trends/scripts/lib/parallel_mcp.py +234 -0
- package/skills/trends/scripts/lib/permission_preflight.py +42 -3
- package/skills/trends/scripts/lib/perplexity.py +1080 -468
- package/skills/trends/scripts/lib/pipeline.py +5079 -211
- package/skills/trends/scripts/lib/planner.py +396 -53
- package/skills/trends/scripts/lib/polymarket.py +285 -23
- package/skills/trends/scripts/lib/prescriptions.py +432 -0
- package/skills/trends/scripts/lib/private_files.py +21 -0
- package/skills/trends/scripts/lib/providers.py +231 -10
- package/skills/trends/scripts/lib/quality_nudge.py +122 -37
- package/skills/trends/scripts/lib/reddit.py +177 -12
- package/skills/trends/scripts/lib/reddit_arctic.py +146 -5
- package/skills/trends/scripts/lib/reddit_keyless.py +121 -60
- package/skills/trends/scripts/lib/reddit_listing.py +215 -14
- package/skills/trends/scripts/lib/reddit_public.py +12 -13
- package/skills/trends/scripts/lib/reddit_search.py +304 -0
- package/skills/trends/scripts/lib/reddit_shreddit.py +51 -7
- package/skills/trends/scripts/lib/registers.py +134 -0
- package/skills/trends/scripts/lib/relevance.py +17 -0
- package/skills/trends/scripts/lib/render.py +2161 -271
- package/skills/trends/scripts/lib/rerank.py +623 -55
- package/skills/trends/scripts/lib/resolve.py +50 -1
- package/skills/trends/scripts/lib/safari_cookies.py +42 -15
- package/skills/trends/scripts/lib/schema.py +782 -1
- package/skills/trends/scripts/lib/setup_wizard.py +779 -120
- package/skills/trends/scripts/lib/signals.py +206 -7
- package/skills/trends/scripts/lib/stocktwits.py +374 -0
- package/skills/trends/scripts/lib/subproc.py +307 -53
- package/skills/trends/scripts/lib/techmeme.py +89 -64
- package/skills/trends/scripts/lib/telegram.py +317 -0
- package/skills/trends/scripts/lib/tiktok.py +13 -9
- package/skills/trends/scripts/lib/topic_shape.py +340 -0
- package/skills/trends/scripts/lib/transcribe.py +12 -3
- package/skills/trends/scripts/lib/trustpilot.py +256 -5
- package/skills/trends/scripts/lib/truthsocial.py +9 -1
- package/skills/trends/scripts/lib/ui.py +6 -0
- package/skills/trends/scripts/lib/usage.py +132 -0
- package/skills/trends/scripts/lib/web_search_keyless.py +63 -4
- package/skills/trends/scripts/lib/windows_job.py +138 -0
- package/skills/trends/scripts/lib/x_api.py +824 -0
- package/skills/trends/scripts/lib/x_envelope.py +733 -0
- package/skills/trends/scripts/lib/x_judge.py +292 -0
- package/skills/trends/scripts/lib/xai_x.py +26 -3
- package/skills/trends/scripts/lib/xquik.py +89 -15
- package/skills/trends/scripts/lib/xurl_x.py +284 -91
- package/skills/trends/scripts/lib/youtube_yt.py +570 -119
- package/skills/trends/scripts/setup-keychain.sh +12 -4
- package/skills/trends/scripts/setup-pass.sh +3 -0
- package/skills/trends/scripts/store.py +444 -50
- package/skills/trends/scripts/watchlist.py +38 -11
- package/skills/trends/scripts/x_lane_parity.py +107 -0
- package/skills/video-studio/CHANGELOG.md +4 -0
- package/skills/video-studio/SKILL.md +4 -4
- package/skills/video-studio/references/styles/product-flight.md +45 -0
- package/skills/video-studio/references/styles.md +3 -1
- package/skills/video-studio-motion/ATTRIBUTION.md +1 -1
- package/skills/video-studio-motion/CHANGELOG.md +4 -0
- package/skills/video-studio-motion/SKILL.md +8 -5
- package/skills/youtube-planner/ATTRIBUTION.md +25 -0
- package/skills/youtube-planner/CHANGELOG.md +12 -0
- package/skills/youtube-planner/SKILL.md +191 -0
- package/skills/youtube-planner/references/plan-template.md +57 -0
- package/skills/youtube-planner/references/stages.md +119 -0
- package/skills/youtube-planner/scripts/floor_ceiling.py +132 -0
- package/skills/coder-brainstorming/spec-document-reviewer-prompt.md +0 -49
- package/skills/trends/scripts/lib/reddit_rss.py +0 -224
|
@@ -8,6 +8,7 @@ presents it), but this module provides the detection and setup actions.
|
|
|
8
8
|
import json
|
|
9
9
|
import logging
|
|
10
10
|
import os
|
|
11
|
+
import re
|
|
11
12
|
import shutil
|
|
12
13
|
import subprocess
|
|
13
14
|
import time
|
|
@@ -16,6 +17,9 @@ from typing import Any, Dict, Optional, Tuple
|
|
|
16
17
|
from urllib.error import HTTPError, URLError
|
|
17
18
|
from urllib.request import Request, urlopen
|
|
18
19
|
|
|
20
|
+
from . import brightdata
|
|
21
|
+
from .private_files import write_private_text
|
|
22
|
+
|
|
19
23
|
logger = logging.getLogger(__name__)
|
|
20
24
|
|
|
21
25
|
|
|
@@ -28,6 +32,42 @@ def is_first_run(config: Dict[str, Any]) -> bool:
|
|
|
28
32
|
return not config.get("SETUP_COMPLETE")
|
|
29
33
|
|
|
30
34
|
|
|
35
|
+
_WELCOME_TEXT = """Welcome to /last30days! I research any topic across Reddit, X, YouTube, TikTok, Digg, arXiv, Techmeme, HN, Polymarket & more - what people actually said in the last 30 days. Let's get you set up (~30s).
|
|
36
|
+
|
|
37
|
+
I synthesize what people are actually saying right now across social, news, and market sources.
|
|
38
|
+
|
|
39
|
+
Auto setup gives you the core sources free in about 30 seconds:
|
|
40
|
+
- Reddit with comments - free keyless discovery (Reddit search + shreddit), no API key needed.
|
|
41
|
+
- YouTube search + transcripts - installs yt-dlp (open source, 190K+ GitHub stars).
|
|
42
|
+
- Digg - trending news, GitHub stars, and pipeline feeds - installs the free, keyless Digg CLI.
|
|
43
|
+
- arXiv (papers) + Techmeme (tech-news) - install free, keyless Printing Press CLIs and run on any topic (arXiv is relevance + recency gated to research topics).
|
|
44
|
+
- StockTwits - retail trader sentiment - auto-on when your topic is a ticker or crypto (e.g. "$NVDA earnings", "bitcoin"), off for everything else.
|
|
45
|
+
- Trustpilot - brand/company review sentiment - opt-in (add trustpilot to INCLUDE_SOURCES), off by default.
|
|
46
|
+
- Hacker News + Polymarket + GitHub (auto-on if the gh CLI is installed) - always on, zero config.
|
|
47
|
+
- X/Twitter - optional. It stays available when you already configured it, or after you explicitly approve a browser-cookie read; skipping it never blocks research.
|
|
48
|
+
|
|
49
|
+
Want TikTok and Instagram too? ScrapeCreators adds those (10,000 free calls, scrapecreators.com). No kickbacks, no affiliation.
|
|
50
|
+
|
|
51
|
+
Power users can turn on more sources in the Manual Setup guide (LinkedIn, Bluesky, Perplexity, and others) - each needs its own credential, so they are off by default."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def render_welcome() -> str:
|
|
55
|
+
"""Return the first-run welcome text.
|
|
56
|
+
|
|
57
|
+
Owned by the engine (single source of truth) so the model relays it rather
|
|
58
|
+
than re-authoring it -- authored prose gets skipped, relayed command output
|
|
59
|
+
does not. Mirrors the SKILL.md welcome; keep in sync if the source set
|
|
60
|
+
changes.
|
|
61
|
+
"""
|
|
62
|
+
return _WELCOME_TEXT
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
# Neutral note recorded on an official-only host instead of a cookie scan.
|
|
66
|
+
# Deliberately names no cookie mechanism beyond the fact that none is used
|
|
67
|
+
# (the vocabulary rule for Grok Bot onboarding output).
|
|
68
|
+
OFFICIAL_HOST_COOKIE_NOTE = "browser sessions are not read on this host"
|
|
69
|
+
|
|
70
|
+
|
|
31
71
|
def run_auto_setup(config: Dict[str, Any], *, allow_browser_cookies: bool = False) -> Dict[str, Any]:
|
|
32
72
|
"""Perform the auto-setup actions.
|
|
33
73
|
|
|
@@ -40,6 +80,12 @@ def run_auto_setup(config: Dict[str, Any], *, allow_browser_cookies: bool = Fals
|
|
|
40
80
|
Returns:
|
|
41
81
|
Dict with keys:
|
|
42
82
|
cookies_found: {source_name: browser_name} for each source where cookies were found
|
|
83
|
+
browser_cookie_scan_attempted: bool (True only after explicit consent
|
|
84
|
+
and at least one browser is eligible for extraction)
|
|
85
|
+
x_cookie_access_denied: browser names denied during an attempted X
|
|
86
|
+
cookie read when no other browser supplied a complete X pair
|
|
87
|
+
cookie_note: present only on an official-only host, where the scan
|
|
88
|
+
is skipped for every domain (neutral, relayable text)
|
|
43
89
|
ytdlp_installed: bool
|
|
44
90
|
ytdlp_action: already_installed | installed | install_failed | no_homebrew
|
|
45
91
|
digg_installed: bool (True when the engine can resolve digg-pp-cli on PATH)
|
|
@@ -49,16 +95,35 @@ def run_auto_setup(config: Dict[str, Any], *, allow_browser_cookies: bool = Fals
|
|
|
49
95
|
digg_stderr: present when digg_action is install_failed
|
|
50
96
|
digg_path: present when digg_action is installed_off_path (binary on disk, not on PATH)
|
|
51
97
|
"""
|
|
52
|
-
from .env import COOKIE_DOMAINS, cookie_extraction_browsers
|
|
98
|
+
from .env import COOKIE_DOMAINS, cookie_extraction_browsers, x_policy
|
|
53
99
|
|
|
54
100
|
cookies_found: Dict[str, str] = {}
|
|
101
|
+
access_denied: dict[str, set[str]] = {}
|
|
102
|
+
cookie_note: Optional[str] = None
|
|
103
|
+
browsers: list[str] = []
|
|
104
|
+
|
|
105
|
+
# Official-only host (LAST30DAYS_HOST=grok-bot): the consented
|
|
106
|
+
# cookie scan is skipped for EVERY domain (X and Truth Social alike) and
|
|
107
|
+
# recorded as not attempted, with a neutral note the caller can relay.
|
|
108
|
+
# The free CLI installs below still run. A LAST30DAYS_X_BACKEND=bird pin
|
|
109
|
+
# is the one path that re-enables discovery, via x_policy.
|
|
110
|
+
if allow_browser_cookies and not x_policy(config).cookie_discovery:
|
|
111
|
+
allow_browser_cookies = False
|
|
112
|
+
cookie_note = OFFICIAL_HOST_COOKIE_NOTE
|
|
55
113
|
|
|
56
114
|
if allow_browser_cookies:
|
|
57
115
|
from . import cookie_extract
|
|
58
116
|
|
|
59
117
|
cookie_config = dict(config)
|
|
118
|
+
cookie_config["BROWSER_CONSENT"] = "true"
|
|
60
119
|
if not (cookie_config.get("FROM_BROWSER") or "").strip():
|
|
61
|
-
|
|
120
|
+
# Chromium-first avoids Safari's usual Full Disk Access prompt.
|
|
121
|
+
# Browser-data permissions can still deny Chromium DB reads before
|
|
122
|
+
# the Keychain step, so a failed scan must report that separately.
|
|
123
|
+
# firefox/safari stay as the silent fallbacks. Note: an explicit
|
|
124
|
+
# comma list preserves this order (cookie_extraction_browsers);
|
|
125
|
+
# "auto" would put the silent browsers first, so do not use it here.
|
|
126
|
+
cookie_config["FROM_BROWSER"] = "chrome,brave,edge,vivaldi,opera,arc,chromium,firefox,safari"
|
|
62
127
|
browsers = cookie_extraction_browsers(cookie_config)
|
|
63
128
|
|
|
64
129
|
for source_name, spec in COOKIE_DOMAINS.items():
|
|
@@ -68,18 +133,27 @@ def run_auto_setup(config: Dict[str, Any], *, allow_browser_cookies: bool = Fals
|
|
|
68
133
|
for browser in browsers:
|
|
69
134
|
try:
|
|
70
135
|
result = cookie_extract.extract_cookies_with_source(browser, domain, cookie_names)
|
|
136
|
+
except PermissionError:
|
|
137
|
+
access_denied.setdefault(source_name, set()).add(browser)
|
|
138
|
+
continue
|
|
71
139
|
except Exception as exc:
|
|
72
140
|
logger.debug("Cookie extraction failed for %s via %s: %s", source_name, browser, exc)
|
|
73
141
|
continue
|
|
74
|
-
if result is not None and result[0]:
|
|
142
|
+
if result is not None and cookie_extract.has_complete_pair(result[0], cookie_names):
|
|
75
143
|
cookies_found[source_name] = result[1]
|
|
76
|
-
break #
|
|
144
|
+
break # Complete pair found for this service, stop trying browsers
|
|
77
145
|
|
|
78
|
-
# Check yt-dlp availability and install via Homebrew if missing
|
|
146
|
+
# Check yt-dlp availability and install via Homebrew if missing. Windows
|
|
147
|
+
# has no Homebrew, and its working install path is `pip install yt-dlp`
|
|
148
|
+
# (see #904), so it gets its own no-op-install guidance branch instead of
|
|
149
|
+
# falling into the Homebrew-oriented no_homebrew outcome.
|
|
79
150
|
ytdlp_action: str
|
|
80
151
|
if shutil.which("yt-dlp") is not None:
|
|
81
152
|
ytdlp_installed = True
|
|
82
153
|
ytdlp_action = "already_installed"
|
|
154
|
+
elif os.name == "nt":
|
|
155
|
+
ytdlp_installed = False
|
|
156
|
+
ytdlp_action = "no_pip_windows"
|
|
83
157
|
elif shutil.which("brew") is not None:
|
|
84
158
|
brew_stderr = ""
|
|
85
159
|
try:
|
|
@@ -107,8 +181,14 @@ def run_auto_setup(config: Dict[str, Any], *, allow_browser_cookies: bool = Fals
|
|
|
107
181
|
digg_installed, digg_action, digg_stderr, digg_path = _install_digg_cli()
|
|
108
182
|
pp_sources = install_default_pp_sources()
|
|
109
183
|
|
|
184
|
+
scan_attempted = allow_browser_cookies and bool(browsers)
|
|
110
185
|
results: Dict[str, Any] = {
|
|
111
186
|
"cookies_found": cookies_found,
|
|
187
|
+
"x_cookie_access_denied": (
|
|
188
|
+
[browser for browser in browsers if browser in access_denied.get("x", set())]
|
|
189
|
+
if scan_attempted and "x" not in cookies_found else []
|
|
190
|
+
),
|
|
191
|
+
"browser_cookie_scan_attempted": scan_attempted,
|
|
112
192
|
"ytdlp_installed": ytdlp_installed,
|
|
113
193
|
"ytdlp_action": ytdlp_action,
|
|
114
194
|
"digg_installed": digg_installed,
|
|
@@ -116,8 +196,16 @@ def run_auto_setup(config: Dict[str, Any], *, allow_browser_cookies: bool = Fals
|
|
|
116
196
|
# Per-CLI status for the additional default-on Printing Press sources
|
|
117
197
|
# (arxiv, techmeme, trustpilot): {source: {installed, action, ...}}.
|
|
118
198
|
"pp_sources": pp_sources,
|
|
199
|
+
# Reported, never installed: this CLI spends the user's own metered
|
|
200
|
+
# credits, so acquiring it stays their decision. Passing
|
|
201
|
+
# config matters: a user whose key lives in a .env file or the
|
|
202
|
+
# keychain (rather than a `brightdata login` credentials file) is
|
|
203
|
+
# active in the engine, and setup must not tell them otherwise.
|
|
204
|
+
"brightdata": brightdata_status(config),
|
|
119
205
|
"env_written": False,
|
|
120
206
|
}
|
|
207
|
+
if cookie_note:
|
|
208
|
+
results["cookie_note"] = cookie_note
|
|
121
209
|
if ytdlp_action == "install_failed":
|
|
122
210
|
results["ytdlp_stderr"] = brew_stderr
|
|
123
211
|
if digg_action == "install_failed":
|
|
@@ -141,22 +229,22 @@ def _digg_bin_candidate_paths() -> list[Path]:
|
|
|
141
229
|
"""Known install locations for digg-pp-cli (Printing Press library defaults).
|
|
142
230
|
|
|
143
231
|
Order: current installer default (~/.local/bin), legacy Go bins, Windows
|
|
144
|
-
managed dir. ``
|
|
232
|
+
managed dir. The directory list is ``health.installer_bin_dirs()`` — the
|
|
233
|
+
shared single source — with the Digg filename variants appended (plain
|
|
234
|
+
name for Unix-style dirs, ``.exe`` in the Windows managed dir).
|
|
235
|
+
``pipeline.available_sources()`` only activates Digg when
|
|
145
236
|
``shutil.which`` resolves on PATH — probing these dirs is for setup
|
|
146
237
|
verification and honest off-PATH messaging, not engine activation.
|
|
147
238
|
"""
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
candidates.append(
|
|
158
|
-
Path(local_app) / "Programs" / "PrintingPress" / "bin" / f"{DIGG_CLI_BIN}.exe"
|
|
159
|
-
)
|
|
239
|
+
from . import health
|
|
240
|
+
|
|
241
|
+
win_dir = health.windows_printing_press_bin_dir()
|
|
242
|
+
candidates: list[Path] = []
|
|
243
|
+
for directory in health.installer_bin_dirs():
|
|
244
|
+
if win_dir is not None and directory == win_dir:
|
|
245
|
+
candidates.append(directory / f"{DIGG_CLI_BIN}.exe")
|
|
246
|
+
else:
|
|
247
|
+
candidates.append(directory / DIGG_CLI_BIN)
|
|
160
248
|
return candidates
|
|
161
249
|
|
|
162
250
|
|
|
@@ -189,6 +277,46 @@ def _digg_bin_dir_hint(digg_path: str) -> str:
|
|
|
189
277
|
return parent
|
|
190
278
|
|
|
191
279
|
|
|
280
|
+
def _run_npx_install(slug: str) -> Tuple[str, str]:
|
|
281
|
+
"""Resolve ``npx`` and run the Printing Press catalog install for ``slug``.
|
|
282
|
+
|
|
283
|
+
Shared by ``_install_digg_cli`` and ``_install_pp_cli`` -- this is only the
|
|
284
|
+
"resolve npx, run the install, interpret no_npx/exception/nonzero-rc"
|
|
285
|
+
slice; each caller keeps its own on-path/off-path re-verification
|
|
286
|
+
(``_digg_bin_candidate_paths`` vs ``_pp_bin_candidate_paths`` already use
|
|
287
|
+
different candidate-directory sources, so merging them here would change
|
|
288
|
+
off-path detection behavior beyond this fix's scope).
|
|
289
|
+
|
|
290
|
+
Fixes the Windows PATHEXT mismatch: ``shutil.which("npx")`` resolves
|
|
291
|
+
``npx.CMD`` via PATHEXT, but ``subprocess.run`` given the bare string
|
|
292
|
+
``"npx"`` as argv[0] does not do that resolution and fails with
|
|
293
|
+
``WinError 2``. Passing the resolved path is a no-op on macOS/Linux, where
|
|
294
|
+
``shutil.which`` already returns the exact path ``CreateProcess``/``execve``
|
|
295
|
+
would resolve.
|
|
296
|
+
|
|
297
|
+
Returns ``(action, stderr)``: ``action`` is ``"no_npx"``,
|
|
298
|
+
``"install_failed"``, or ``""`` when the subprocess ran and returned
|
|
299
|
+
``rc=0`` (in which case ``stderr`` carries any non-fatal stderr output for
|
|
300
|
+
the caller's own off-path logging).
|
|
301
|
+
"""
|
|
302
|
+
npx = shutil.which("npx")
|
|
303
|
+
if npx is None:
|
|
304
|
+
return "no_npx", ""
|
|
305
|
+
try:
|
|
306
|
+
proc = subprocess.run(
|
|
307
|
+
[npx, "-y", PRINTING_PRESS_NPM, "install", slug, "--cli-only"],
|
|
308
|
+
capture_output=True, text=True, timeout=DIGG_INSTALL_TIMEOUT,
|
|
309
|
+
)
|
|
310
|
+
except Exception as exc:
|
|
311
|
+
logger.warning("npx install %s exception: %s", slug, exc)
|
|
312
|
+
return "install_failed", str(exc)
|
|
313
|
+
if proc.returncode != 0:
|
|
314
|
+
stderr = proc.stderr or f"npx install {slug} exited {proc.returncode}"
|
|
315
|
+
logger.warning("npx install %s failed (rc=%s): %s", slug, proc.returncode, stderr)
|
|
316
|
+
return "install_failed", stderr
|
|
317
|
+
return "", (proc.stderr or "")
|
|
318
|
+
|
|
319
|
+
|
|
192
320
|
def _install_digg_cli() -> Tuple[bool, str, str, str]:
|
|
193
321
|
"""Best-effort install of the digg-pp-cli binary.
|
|
194
322
|
|
|
@@ -210,32 +338,21 @@ def _install_digg_cli() -> Tuple[bool, str, str, str]:
|
|
|
210
338
|
off_path = _digg_off_path_binary()
|
|
211
339
|
if off_path:
|
|
212
340
|
return False, "installed_off_path", "", off_path
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
proc = subprocess.run(
|
|
217
|
-
["npx", "-y", PRINTING_PRESS_NPM, "install", "digg", "--cli-only"],
|
|
218
|
-
capture_output=True, text=True, timeout=DIGG_INSTALL_TIMEOUT,
|
|
219
|
-
)
|
|
220
|
-
except Exception as exc:
|
|
221
|
-
logger.warning("npx install digg exception: %s", exc)
|
|
222
|
-
return False, "install_failed", str(exc), ""
|
|
223
|
-
if proc.returncode != 0:
|
|
224
|
-
stderr = proc.stderr or f"npx install digg exited {proc.returncode}"
|
|
225
|
-
logger.warning("npx install digg failed (rc=%s): %s", proc.returncode, stderr)
|
|
226
|
-
return False, "install_failed", stderr, ""
|
|
341
|
+
action, stderr = _run_npx_install("digg")
|
|
342
|
+
if action:
|
|
343
|
+
return False, action, stderr, ""
|
|
227
344
|
on_path = _digg_on_path()
|
|
228
345
|
if on_path:
|
|
229
346
|
return True, "installed", "", ""
|
|
230
347
|
off_path = _digg_off_path_binary()
|
|
231
348
|
if off_path:
|
|
232
|
-
combined =
|
|
349
|
+
combined = stderr.strip()
|
|
233
350
|
if combined:
|
|
234
351
|
logger.warning("digg-pp-cli installed off PATH: %s", combined)
|
|
235
352
|
return False, "installed_off_path", combined, off_path
|
|
236
|
-
|
|
237
|
-
logger.warning("npx install digg failed verification: %s",
|
|
238
|
-
return False, "install_failed",
|
|
353
|
+
stderr_msg = stderr or "install completed but digg-pp-cli was not found"
|
|
354
|
+
logger.warning("npx install digg failed verification: %s", stderr_msg)
|
|
355
|
+
return False, "install_failed", stderr_msg, ""
|
|
239
356
|
|
|
240
357
|
|
|
241
358
|
# Additional default-on Printing Press sources installed the same way as Digg:
|
|
@@ -250,6 +367,85 @@ PP_DEFAULT_SOURCES: list[tuple[str, str, str]] = [
|
|
|
250
367
|
("techmeme", "techmeme", "techmeme-pp-cli"),
|
|
251
368
|
]
|
|
252
369
|
|
|
370
|
+
# Bright Data is deliberately absent from PP_DEFAULT_SOURCES: it is not a
|
|
371
|
+
# Printing Press CLI, it is opt-in like Trustpilot, and it spends the user's
|
|
372
|
+
# own metered credits. Setup reports its state and never installs it.
|
|
373
|
+
BRIGHTDATA_BIN = "brightdata"
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _brightdata_off_path_binary() -> Optional[str]:
|
|
377
|
+
"""Locate a brightdata binary that exists on disk but not on PATH.
|
|
378
|
+
|
|
379
|
+
Covers the common npm global prefixes. The distinction matters because
|
|
380
|
+
Hermes and OpenClaw gateways routinely run the engine with a PATH that
|
|
381
|
+
excludes the user's npm bin directory, so "installed" and "the engine
|
|
382
|
+
can see it" are different questions.
|
|
383
|
+
"""
|
|
384
|
+
home = Path.home()
|
|
385
|
+
candidates = [
|
|
386
|
+
home / ".local" / "bin" / BRIGHTDATA_BIN,
|
|
387
|
+
home / ".npm-global" / "bin" / BRIGHTDATA_BIN,
|
|
388
|
+
Path("/opt/homebrew/bin") / BRIGHTDATA_BIN,
|
|
389
|
+
Path("/usr/local/bin") / BRIGHTDATA_BIN,
|
|
390
|
+
]
|
|
391
|
+
npm_prefix = os.environ.get("NPM_CONFIG_PREFIX")
|
|
392
|
+
if npm_prefix:
|
|
393
|
+
candidates.insert(0, Path(npm_prefix) / "bin" / BRIGHTDATA_BIN)
|
|
394
|
+
for candidate in candidates:
|
|
395
|
+
try:
|
|
396
|
+
if candidate.is_file() and os.access(candidate, os.X_OK):
|
|
397
|
+
return str(candidate)
|
|
398
|
+
except OSError:
|
|
399
|
+
continue
|
|
400
|
+
return None
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def brightdata_status(config: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
404
|
+
"""Report the Bright Data install and auth state honestly.
|
|
405
|
+
|
|
406
|
+
Deliberately never claims the source is active unless the engine's own
|
|
407
|
+
gate would pass -- ``brightdata.is_available`` is the single predicate,
|
|
408
|
+
so setup and the engine cannot drift apart. Three states matter:
|
|
409
|
+
|
|
410
|
+
* ``already_installed`` -- on PATH; ``authenticated`` says whether the
|
|
411
|
+
amazon lane will actually run.
|
|
412
|
+
* ``installed_off_path`` -- on disk but invisible to the engine, which
|
|
413
|
+
is the Hermes/OpenClaw failure mode. Carries the path so the user can
|
|
414
|
+
fix their PATH.
|
|
415
|
+
* ``not_installed`` -- nothing found. No auto-install: this CLI
|
|
416
|
+
spends the user's metered credits, so acquiring it is their call.
|
|
417
|
+
"""
|
|
418
|
+
installed = brightdata.is_installed()
|
|
419
|
+
authenticated = brightdata.has_credentials(config)
|
|
420
|
+
if installed:
|
|
421
|
+
action = "already_installed"
|
|
422
|
+
off_path = ""
|
|
423
|
+
else:
|
|
424
|
+
off_path = _brightdata_off_path_binary() or ""
|
|
425
|
+
action = "installed_off_path" if off_path else "not_installed"
|
|
426
|
+
|
|
427
|
+
status: Dict[str, Any] = {
|
|
428
|
+
"installed": installed,
|
|
429
|
+
"action": action,
|
|
430
|
+
"authenticated": installed and authenticated,
|
|
431
|
+
# The engine gate, verbatim. Never report active on anything else.
|
|
432
|
+
"engine_active": brightdata.is_available(config),
|
|
433
|
+
}
|
|
434
|
+
if off_path:
|
|
435
|
+
status["path"] = off_path
|
|
436
|
+
status["hint"] = (
|
|
437
|
+
f"brightdata found at {off_path} but not on PATH; add its directory "
|
|
438
|
+
"to PATH so the engine subprocess can see it"
|
|
439
|
+
)
|
|
440
|
+
elif installed and not authenticated:
|
|
441
|
+
status["hint"] = "run `brightdata login` to activate the amazon source"
|
|
442
|
+
elif not installed:
|
|
443
|
+
status["hint"] = (
|
|
444
|
+
"install with `npm i -g @brightdata/cli` then `brightdata login` "
|
|
445
|
+
"to enable the amazon source"
|
|
446
|
+
)
|
|
447
|
+
return status
|
|
448
|
+
|
|
253
449
|
|
|
254
450
|
def _pp_bin_candidate_paths(bin_name: str) -> list[Path]:
|
|
255
451
|
"""Known install locations for a Printing Press CLI binary (slug-parameterized
|
|
@@ -291,32 +487,21 @@ def _install_pp_cli(slug: str, bin_name: str) -> Tuple[bool, str, str, str]:
|
|
|
291
487
|
off_path = _pp_off_path_binary(bin_name)
|
|
292
488
|
if off_path:
|
|
293
489
|
return False, "installed_off_path", "", off_path
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
proc = subprocess.run(
|
|
298
|
-
["npx", "-y", PRINTING_PRESS_NPM, "install", slug, "--cli-only"],
|
|
299
|
-
capture_output=True, text=True, timeout=DIGG_INSTALL_TIMEOUT,
|
|
300
|
-
)
|
|
301
|
-
except Exception as exc:
|
|
302
|
-
logger.warning("npx install %s exception: %s", slug, exc)
|
|
303
|
-
return False, "install_failed", str(exc), ""
|
|
304
|
-
if proc.returncode != 0:
|
|
305
|
-
stderr = proc.stderr or f"npx install {slug} exited {proc.returncode}"
|
|
306
|
-
logger.warning("npx install %s failed (rc=%s): %s", slug, proc.returncode, stderr)
|
|
307
|
-
return False, "install_failed", stderr, ""
|
|
490
|
+
action, stderr = _run_npx_install(slug)
|
|
491
|
+
if action:
|
|
492
|
+
return False, action, stderr, ""
|
|
308
493
|
on_path = shutil.which(bin_name)
|
|
309
494
|
if on_path:
|
|
310
495
|
return True, "installed", "", ""
|
|
311
496
|
off_path = _pp_off_path_binary(bin_name)
|
|
312
497
|
if off_path:
|
|
313
|
-
combined =
|
|
498
|
+
combined = stderr.strip()
|
|
314
499
|
if combined:
|
|
315
500
|
logger.warning("%s installed off PATH: %s", bin_name, combined)
|
|
316
501
|
return False, "installed_off_path", combined, off_path
|
|
317
|
-
|
|
318
|
-
logger.warning("npx install %s failed verification: %s", slug,
|
|
319
|
-
return False, "install_failed",
|
|
502
|
+
stderr_msg = stderr or f"install completed but {bin_name} was not found"
|
|
503
|
+
logger.warning("npx install %s failed verification: %s", slug, stderr_msg)
|
|
504
|
+
return False, "install_failed", stderr_msg, ""
|
|
320
505
|
|
|
321
506
|
|
|
322
507
|
def install_default_pp_sources() -> Dict[str, Dict[str, Any]]:
|
|
@@ -355,6 +540,58 @@ def _open_secret_append(path: Path):
|
|
|
355
540
|
return os.fdopen(fd, "a", encoding="utf-8")
|
|
356
541
|
|
|
357
542
|
|
|
543
|
+
def _env_line_key(stripped: str) -> str:
|
|
544
|
+
"""Return the key a stripped ``KEY=value`` line sets, as the loader reads it."""
|
|
545
|
+
from . import env as _env
|
|
546
|
+
|
|
547
|
+
return _env.env_line_key(stripped.split("=", 1)[0])
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _replace_env_line(env_path: Path, content: str, key_name: str, value: str) -> bool:
|
|
551
|
+
"""Rewrite every ``key_name=`` line of ``content`` with ``value`` as a 0o600 secret.
|
|
552
|
+
|
|
553
|
+
The new file is written to a sibling temp path opened at 0o600 and moved
|
|
554
|
+
over the original, so the secret never has a readable window and a
|
|
555
|
+
crash mid-write leaves the old file intact.
|
|
556
|
+
"""
|
|
557
|
+
def _is_key(stripped: str) -> bool:
|
|
558
|
+
return bool(
|
|
559
|
+
stripped and not stripped.startswith("#") and "=" in stripped
|
|
560
|
+
and _env_line_key(stripped) == key_name
|
|
561
|
+
)
|
|
562
|
+
|
|
563
|
+
stripped_lines = [line.strip() for line in content.splitlines()]
|
|
564
|
+
# Keep a hand-written ``export`` -- on any of the duplicates being
|
|
565
|
+
# collapsed -- so a shell that sources the file still exports the key.
|
|
566
|
+
exported = any(
|
|
567
|
+
_is_key(s) and s.split("=", 1)[0].strip() != key_name for s in stripped_lines
|
|
568
|
+
)
|
|
569
|
+
new_line = f"{key_name}={_format_env_value(value)}"
|
|
570
|
+
if exported:
|
|
571
|
+
new_line = f"export {new_line}"
|
|
572
|
+
lines = []
|
|
573
|
+
replaced = False
|
|
574
|
+
for line, stripped in zip(content.splitlines(), stripped_lines):
|
|
575
|
+
if _is_key(stripped):
|
|
576
|
+
if not replaced:
|
|
577
|
+
lines.append(new_line)
|
|
578
|
+
replaced = True
|
|
579
|
+
continue
|
|
580
|
+
lines.append(line)
|
|
581
|
+
if not replaced:
|
|
582
|
+
lines.append(new_line)
|
|
583
|
+
tmp_path = env_path.with_name(env_path.name + ".tmp")
|
|
584
|
+
fd = os.open(tmp_path, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600)
|
|
585
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
586
|
+
f.write("\n".join(lines) + "\n")
|
|
587
|
+
os.replace(tmp_path, env_path)
|
|
588
|
+
try:
|
|
589
|
+
os.chmod(env_path, 0o600)
|
|
590
|
+
except OSError:
|
|
591
|
+
pass
|
|
592
|
+
return True
|
|
593
|
+
|
|
594
|
+
|
|
358
595
|
def _format_env_value(value: str) -> str:
|
|
359
596
|
"""Quote a value so it round-trips through env.load_env_file.
|
|
360
597
|
|
|
@@ -378,20 +615,30 @@ def _format_env_value(value: str) -> str:
|
|
|
378
615
|
return value
|
|
379
616
|
|
|
380
617
|
|
|
381
|
-
def write_setup_config(
|
|
382
|
-
|
|
618
|
+
def write_setup_config(
|
|
619
|
+
env_path: Path,
|
|
620
|
+
from_browser: str | None = None,
|
|
621
|
+
*,
|
|
622
|
+
browser_consent: bool | None = None,
|
|
623
|
+
x_cookie_access_denied: str | None = None,
|
|
624
|
+
) -> bool:
|
|
625
|
+
"""Write setup completion, browser selection, and consent to the .env file.
|
|
383
626
|
|
|
384
627
|
Creates the file and parent directories if needed.
|
|
385
|
-
Appends
|
|
628
|
+
Appends without overwriting existing keys, except for an explicit consent
|
|
629
|
+
decision.
|
|
386
630
|
|
|
387
631
|
Args:
|
|
388
632
|
env_path: Path to the .env file (e.g. ~/.config/last30days/.env)
|
|
389
|
-
from_browser: Browser
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
633
|
+
from_browser: Browser or comma-separated browser list that actually
|
|
634
|
+
yielded cookies after consent (e.g. "chrome,firefox"). Pass None
|
|
635
|
+
(default) to leave FROM_BROWSER unchanged; when unset, future runs
|
|
636
|
+
do not read native browser stores. Avoid "auto", which would also probe
|
|
637
|
+
browsers that did not supply cookies during setup.
|
|
638
|
+
browser_consent: Record the user's current cookie-access decision.
|
|
639
|
+
None preserves any previous decision.
|
|
640
|
+
x_cookie_access_denied: Setup's non-secret last X denial observation.
|
|
641
|
+
Pass "none" to clear a prior denial after a new setup run.
|
|
395
642
|
|
|
396
643
|
Returns:
|
|
397
644
|
True if config was written successfully, False on error.
|
|
@@ -399,6 +646,18 @@ def write_setup_config(env_path: Path, from_browser: str | None = None) -> bool:
|
|
|
399
646
|
try:
|
|
400
647
|
env_path = Path(env_path)
|
|
401
648
|
env_path.parent.mkdir(parents=True, exist_ok=True)
|
|
649
|
+
if browser_consent is not None:
|
|
650
|
+
if not write_api_key(
|
|
651
|
+
env_path, "true" if browser_consent else "false",
|
|
652
|
+
key_name="BROWSER_CONSENT", replace=True,
|
|
653
|
+
):
|
|
654
|
+
return False
|
|
655
|
+
if x_cookie_access_denied is not None:
|
|
656
|
+
if not write_api_key(
|
|
657
|
+
env_path, x_cookie_access_denied,
|
|
658
|
+
key_name="LAST30DAYS_X_COOKIE_ACCESS_DENIED", replace=True,
|
|
659
|
+
):
|
|
660
|
+
return False
|
|
402
661
|
|
|
403
662
|
# Read existing content to avoid overwriting keys
|
|
404
663
|
existing_keys: set = set()
|
|
@@ -408,7 +667,7 @@ def write_setup_config(env_path: Path, from_browser: str | None = None) -> bool:
|
|
|
408
667
|
for line in existing_content.splitlines():
|
|
409
668
|
stripped = line.strip()
|
|
410
669
|
if stripped and not stripped.startswith("#") and "=" in stripped:
|
|
411
|
-
key = stripped
|
|
670
|
+
key = _env_line_key(stripped)
|
|
412
671
|
existing_keys.add(key)
|
|
413
672
|
|
|
414
673
|
lines_to_add = []
|
|
@@ -434,19 +693,28 @@ def write_setup_config(env_path: Path, from_browser: str | None = None) -> bool:
|
|
|
434
693
|
return False
|
|
435
694
|
|
|
436
695
|
|
|
437
|
-
def write_api_key(
|
|
696
|
+
def write_api_key(
|
|
697
|
+
env_path: Path,
|
|
698
|
+
api_key: str,
|
|
699
|
+
key_name: str = "SCRAPECREATORS_API_KEY",
|
|
700
|
+
*,
|
|
701
|
+
replace: bool = False,
|
|
702
|
+
) -> bool:
|
|
438
703
|
"""Append an API key to the .env file as a 0o600 secret.
|
|
439
704
|
|
|
440
705
|
Reuses the same secret-safe write path as ``write_setup_config`` so the
|
|
441
706
|
value lands with restrictive permissions and round-trips through
|
|
442
|
-
``env.load_env_file``. Idempotent: if ``key_name`` is already
|
|
443
|
-
the file, nothing is written and the existing value is
|
|
444
|
-
clobber a key the user may have set by hand).
|
|
707
|
+
``env.load_env_file``. Idempotent by default: if ``key_name`` is already
|
|
708
|
+
present in the file, nothing is written and the existing value is
|
|
709
|
+
preserved (we never clobber a key the user may have set by hand). With
|
|
710
|
+
``replace=True`` an existing line is rewritten in place instead, so an
|
|
711
|
+
explicit ``setup --store-key`` can rotate a rejected credential.
|
|
445
712
|
|
|
446
713
|
Args:
|
|
447
714
|
env_path: Path to the .env file (e.g. ~/.config/last30days/.env).
|
|
448
715
|
api_key: The raw key value to persist.
|
|
449
716
|
key_name: The env var name to write (default SCRAPECREATORS_API_KEY).
|
|
717
|
+
replace: Rewrite an existing ``key_name`` line instead of keeping it.
|
|
450
718
|
|
|
451
719
|
Returns:
|
|
452
720
|
True if the key was written or already present, False on error or when
|
|
@@ -464,7 +732,9 @@ def write_api_key(env_path: Path, api_key: str, key_name: str = "SCRAPECREATORS_
|
|
|
464
732
|
for line in existing_content.splitlines():
|
|
465
733
|
stripped = line.strip()
|
|
466
734
|
if stripped and not stripped.startswith("#") and "=" in stripped:
|
|
467
|
-
if stripped
|
|
735
|
+
if _env_line_key(stripped) == key_name:
|
|
736
|
+
if replace:
|
|
737
|
+
return _replace_env_line(env_path, existing_content, key_name, api_key)
|
|
468
738
|
return True # Already configured; do not duplicate
|
|
469
739
|
|
|
470
740
|
line = f"{key_name}={_format_env_value(api_key)}\n"
|
|
@@ -500,16 +770,25 @@ def get_setup_status_text(results: Dict[str, Any]) -> str:
|
|
|
500
770
|
Returns:
|
|
501
771
|
Multi-line status text.
|
|
502
772
|
"""
|
|
773
|
+
from .env import X_COOKIE_ACCESS_FIX
|
|
774
|
+
|
|
503
775
|
lines = []
|
|
504
776
|
lines.append("Setup complete! Here's what I found:")
|
|
505
777
|
lines.append("")
|
|
506
778
|
|
|
507
779
|
cookies_found = results.get("cookies_found", {})
|
|
508
|
-
if cookies_found:
|
|
780
|
+
if results.get("browser_cookie_scan_attempted") and cookies_found:
|
|
509
781
|
for source, browser in cookies_found.items():
|
|
510
782
|
lines.append(f" - {source.upper()} cookies found in {browser}")
|
|
511
|
-
|
|
512
|
-
|
|
783
|
+
denied_browsers = (
|
|
784
|
+
results.get("x_cookie_access_denied") or []
|
|
785
|
+
if results.get("browser_cookie_scan_attempted") else []
|
|
786
|
+
)
|
|
787
|
+
if denied_browsers:
|
|
788
|
+
lines.append(
|
|
789
|
+
" - X cookie access was permission denied in "
|
|
790
|
+
f"{', '.join(denied_browsers)}. {X_COOKIE_ACCESS_FIX}"
|
|
791
|
+
)
|
|
513
792
|
|
|
514
793
|
ytdlp_action = results.get("ytdlp_action", "")
|
|
515
794
|
if ytdlp_action == "installed":
|
|
@@ -518,6 +797,11 @@ def get_setup_status_text(results: Dict[str, Any]) -> str:
|
|
|
518
797
|
lines.append(" - yt-dlp install failed \u2014 run `brew install yt-dlp` manually")
|
|
519
798
|
elif ytdlp_action == "no_homebrew":
|
|
520
799
|
lines.append(" - yt-dlp not found. Install Homebrew first, then: brew install yt-dlp")
|
|
800
|
+
elif ytdlp_action == "no_pip_windows":
|
|
801
|
+
lines.append(
|
|
802
|
+
" - yt-dlp not found. Install with: pip install yt-dlp "
|
|
803
|
+
"(it may install to a Scripts directory not on PATH -- add it to PATH if YouTube search stays inactive)"
|
|
804
|
+
)
|
|
521
805
|
elif ytdlp_action == "already_installed":
|
|
522
806
|
lines.append(" - yt-dlp already installed")
|
|
523
807
|
elif results.get("ytdlp_installed", False):
|
|
@@ -553,10 +837,82 @@ def get_setup_status_text(results: Dict[str, Any]) -> str:
|
|
|
553
837
|
f"{DIGG_INSTALL_CMD}"
|
|
554
838
|
)
|
|
555
839
|
|
|
840
|
+
pp_sources = results.get("pp_sources", {})
|
|
841
|
+
pp_name: dict[str, str] = {"arxiv": "arXiv", "techmeme": "Techmeme"}
|
|
842
|
+
for source_key, entry in sorted(pp_sources.items()):
|
|
843
|
+
name = pp_name.get(source_key, source_key.title())
|
|
844
|
+
action = entry.get("action", "")
|
|
845
|
+
if action == "installed":
|
|
846
|
+
lines.append(f" - Installed {name} CLI ({name} source now active)")
|
|
847
|
+
elif action == "already_installed":
|
|
848
|
+
lines.append(f" - {name} CLI already installed ({name} active)")
|
|
849
|
+
elif action == "installed_off_path":
|
|
850
|
+
path = entry.get("path", "")
|
|
851
|
+
if path:
|
|
852
|
+
lines.append(
|
|
853
|
+
f" - {name} CLI at {path} but not on PATH — add "
|
|
854
|
+
f"{os.path.dirname(os.path.expanduser(path))} to PATH and "
|
|
855
|
+
f"restart your agent session/gateway for {name} to activate"
|
|
856
|
+
)
|
|
857
|
+
else:
|
|
858
|
+
lines.append(
|
|
859
|
+
f" - {name} CLI installed but not on PATH — add its install "
|
|
860
|
+
"directory to PATH and restart your agent session/gateway for "
|
|
861
|
+
f"{name} to activate"
|
|
862
|
+
)
|
|
863
|
+
elif action == "install_failed":
|
|
864
|
+
lines.append(
|
|
865
|
+
f" - {name} CLI install failed — run "
|
|
866
|
+
f"`npx -y {PRINTING_PRESS_NPM} install {source_key} --cli-only` manually"
|
|
867
|
+
)
|
|
868
|
+
elif action == "no_npx":
|
|
869
|
+
lines.append(
|
|
870
|
+
f" - {name} CLI not installed (free, optional). Install Node/npx, "
|
|
871
|
+
f"then: `npx -y {PRINTING_PRESS_NPM} install {source_key} --cli-only`"
|
|
872
|
+
)
|
|
873
|
+
|
|
874
|
+
# Bright Data / Amazon. Reported but never installed (it spends the user's
|
|
875
|
+
# own metered credits), so the only useful thing setup can do is say
|
|
876
|
+
# precisely why the lane is or is not active -- the three states below are
|
|
877
|
+
# otherwise invisible, since SKILL.md tells the model not to raise the
|
|
878
|
+
# subject mid-run.
|
|
879
|
+
brightdata_status_entry = results.get("brightdata") or {}
|
|
880
|
+
bd_action = brightdata_status_entry.get("action", "")
|
|
881
|
+
if brightdata_status_entry.get("engine_active"):
|
|
882
|
+
lines.append(" - Bright Data CLI ready (Amazon buyer signals available)")
|
|
883
|
+
elif bd_action == "already_installed":
|
|
884
|
+
lines.append(
|
|
885
|
+
" - Bright Data CLI installed but not logged in — run "
|
|
886
|
+
"`brightdata login` to enable Amazon buyer signals (optional)"
|
|
887
|
+
)
|
|
888
|
+
elif bd_action == "installed_off_path":
|
|
889
|
+
bd_path = brightdata_status_entry.get("path", "")
|
|
890
|
+
lines.append(
|
|
891
|
+
f" - Bright Data CLI found at {bd_path} but not on PATH — add "
|
|
892
|
+
f"{os.path.dirname(os.path.expanduser(bd_path))} to PATH and restart "
|
|
893
|
+
"your agent session/gateway for Amazon buyer signals to activate"
|
|
894
|
+
)
|
|
895
|
+
elif bd_action == "not_installed":
|
|
896
|
+
lines.append(
|
|
897
|
+
" - Amazon buyer signals not installed (optional; 5,000 free "
|
|
898
|
+
"requests/month). Install with: npm i -g @brightdata/cli && brightdata login"
|
|
899
|
+
)
|
|
900
|
+
|
|
901
|
+
cookie_note = results.get("cookie_note")
|
|
902
|
+
if cookie_note:
|
|
903
|
+
# Official-only host: relay the neutral note; say nothing about
|
|
904
|
+
# browsers. The scan was not attempted, so nothing is "found".
|
|
905
|
+
lines.append(f" - {cookie_note}")
|
|
906
|
+
|
|
556
907
|
env_written = results.get("env_written", False)
|
|
557
908
|
if env_written:
|
|
558
909
|
lines.append("")
|
|
559
|
-
|
|
910
|
+
if cookie_note:
|
|
911
|
+
lines.append("Configuration saved.")
|
|
912
|
+
elif denied_browsers:
|
|
913
|
+
lines.append("Configuration saved. X browser cookie access remains unverified.")
|
|
914
|
+
else:
|
|
915
|
+
lines.append("Configuration saved. Future runs will auto-detect your browsers.")
|
|
560
916
|
|
|
561
917
|
return "\n".join(lines)
|
|
562
918
|
|
|
@@ -625,6 +981,39 @@ def run_openclaw_setup(config: Dict[str, Any]) -> Dict[str, Any]:
|
|
|
625
981
|
|
|
626
982
|
_DEVICE_BASE = "https://api.scrapecreators.com/v1/github/device"
|
|
627
983
|
|
|
984
|
+
# A GitHub device code is always XXXX-XXXX (uppercase alphanumerics). We validate
|
|
985
|
+
# user_code against this before copying, labeling, or emitting it so a malformed
|
|
986
|
+
# or key-shaped value (e.g. a returning-account server response) is never
|
|
987
|
+
# mislabeled as a device code or leaked to stdout/clipboard.
|
|
988
|
+
_DEVICE_CODE_RE = re.compile(r"^[0-9A-Z]{4}-[0-9A-Z]{4}$")
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
def _existing_scrapecreators_key() -> Optional[str]:
|
|
992
|
+
"""Return the SCRAPECREATORS_API_KEY already saved in the .env, if any."""
|
|
993
|
+
try:
|
|
994
|
+
from . import env as _env
|
|
995
|
+
|
|
996
|
+
if _env.CONFIG_FILE and _env.CONFIG_FILE.exists():
|
|
997
|
+
return _env.load_env_file(_env.CONFIG_FILE).get("SCRAPECREATORS_API_KEY") or None
|
|
998
|
+
except Exception as exc: # never let a config-read failure block auth
|
|
999
|
+
logger.debug("Could not read existing ScrapeCreators key: %s", exc)
|
|
1000
|
+
return None
|
|
1001
|
+
|
|
1002
|
+
|
|
1003
|
+
def _clamp_device_interval(interval: Any) -> int:
|
|
1004
|
+
"""Clamp a server-provided device-flow poll interval to [1, 30] seconds.
|
|
1005
|
+
|
|
1006
|
+
The ``interval`` comes from the server (device/code response or the
|
|
1007
|
+
persisted poll handle), so 0 would hot-loop ``time.sleep``, a negative
|
|
1008
|
+
would crash it, a huge value would sail past the poll timeout, and a
|
|
1009
|
+
non-numeric value would raise. Defaults to 5 on missing/garbled input.
|
|
1010
|
+
"""
|
|
1011
|
+
try:
|
|
1012
|
+
value = int(interval or 5)
|
|
1013
|
+
except (TypeError, ValueError):
|
|
1014
|
+
return 5
|
|
1015
|
+
return min(max(value, 1), 30)
|
|
1016
|
+
|
|
628
1017
|
|
|
629
1018
|
def run_device_auth() -> Optional[Tuple[str, str, str, int]]:
|
|
630
1019
|
"""Start the device authorization flow.
|
|
@@ -648,10 +1037,14 @@ def run_device_auth() -> Optional[Tuple[str, str, str, int]]:
|
|
|
648
1037
|
device_code = data.get("device_code")
|
|
649
1038
|
user_code = data.get("user_code")
|
|
650
1039
|
verification_uri = data.get("verification_uri")
|
|
651
|
-
interval = data.get("interval", 5)
|
|
1040
|
+
interval = _clamp_device_interval(data.get("interval", 5))
|
|
652
1041
|
|
|
653
1042
|
if not device_code or not user_code:
|
|
654
|
-
|
|
1043
|
+
# Log only the response's key names, never its values — a returning
|
|
1044
|
+
# account's response could carry a raw API key we must not write to logs.
|
|
1045
|
+
logger.warning(
|
|
1046
|
+
"Device auth returned incomplete response (keys: %s)", sorted(data.keys())
|
|
1047
|
+
)
|
|
655
1048
|
return None
|
|
656
1049
|
|
|
657
1050
|
return (device_code, user_code, verification_uri or "", interval)
|
|
@@ -678,6 +1071,7 @@ def poll_device_auth(
|
|
|
678
1071
|
"""
|
|
679
1072
|
import sys
|
|
680
1073
|
|
|
1074
|
+
interval = _clamp_device_interval(interval)
|
|
681
1075
|
started_at = time.time()
|
|
682
1076
|
deadline = started_at + timeout
|
|
683
1077
|
last_reminder = started_at
|
|
@@ -733,62 +1127,211 @@ def poll_device_auth(
|
|
|
733
1127
|
return None
|
|
734
1128
|
|
|
735
1129
|
|
|
736
|
-
|
|
1130
|
+
# Bounded retries for transient ScrapeCreators /profile 5xx (see #882).
|
|
1131
|
+
_PROFILE_FETCH_ATTEMPTS = 3
|
|
1132
|
+
_PROFILE_FETCH_RETRY_SLEEP_S = 1.0
|
|
1133
|
+
_PROFILE_ERROR_BODY_LIMIT = 200
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
def _http_error_detail(exc: HTTPError, *, body_limit: int = _PROFILE_ERROR_BODY_LIMIT) -> str:
|
|
1137
|
+
"""Build a diagnosable HTTPError string including a truncated body.
|
|
1138
|
+
|
|
1139
|
+
The body is capped and never treated as a secret source of truth; callers
|
|
1140
|
+
still must not log bearer tokens. Used so a 5xx is not a black box.
|
|
1141
|
+
"""
|
|
1142
|
+
base = f"HTTP Error {exc.code}: {getattr(exc, 'reason', '') or ''}".rstrip(": ")
|
|
1143
|
+
try:
|
|
1144
|
+
raw = exc.read() or b""
|
|
1145
|
+
except Exception:
|
|
1146
|
+
return base
|
|
1147
|
+
if not raw:
|
|
1148
|
+
return base
|
|
1149
|
+
text = raw.decode("utf-8", errors="replace").strip()
|
|
1150
|
+
if not text:
|
|
1151
|
+
return base
|
|
1152
|
+
if len(text) > body_limit:
|
|
1153
|
+
text = text[:body_limit] + "…"
|
|
1154
|
+
return f"{base} body={text!r}"
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
def fetch_api_key(access_token: str) -> Dict[str, Any]:
|
|
737
1158
|
"""Fetch the ScrapeCreators API key using the GitHub access token.
|
|
738
1159
|
|
|
739
|
-
GETs the device/profile endpoint with Bearer auth.
|
|
1160
|
+
GETs the device/profile endpoint with Bearer auth. Distinguishes outcomes
|
|
1161
|
+
so callers (and SKILL.md) do not collapse a server 5xx into the
|
|
1162
|
+
already-linked guidance (#882).
|
|
1163
|
+
|
|
1164
|
+
Returns a result dict:
|
|
1165
|
+
- ``{"ok": True, "api_key": str}`` on success
|
|
1166
|
+
- ``{"ok": False, "reason": "no_api_key"}`` when /profile is 2xx but has
|
|
1167
|
+
no ``api_key`` field (typical already-linked account)
|
|
1168
|
+
- ``{"ok": False, "reason": "upstream_error", "http_status": int,
|
|
1169
|
+
"detail": str}`` on 5xx after bounded retries
|
|
1170
|
+
- ``{"ok": False, "reason": "http_error", "http_status": int,
|
|
1171
|
+
"detail": str}`` on other HTTP errors (e.g. 401)
|
|
1172
|
+
- ``{"ok": False, "reason": "request_failed", "detail": str}`` on
|
|
1173
|
+
network/OS failures
|
|
1174
|
+
"""
|
|
1175
|
+
data: Optional[Dict[str, Any]] = None
|
|
1176
|
+
last_upstream: Optional[Dict[str, Any]] = None
|
|
740
1177
|
|
|
741
|
-
|
|
742
|
-
|
|
1178
|
+
for attempt in range(_PROFILE_FETCH_ATTEMPTS):
|
|
1179
|
+
try:
|
|
1180
|
+
req = Request(f"{_DEVICE_BASE}/profile")
|
|
1181
|
+
req.add_header("Authorization", f"Bearer {access_token}")
|
|
1182
|
+
with urlopen(req, timeout=15) as resp:
|
|
1183
|
+
data = json.loads(resp.read())
|
|
1184
|
+
break
|
|
1185
|
+
except HTTPError as exc:
|
|
1186
|
+
detail = _http_error_detail(exc)
|
|
1187
|
+
if 500 <= int(exc.code) <= 599:
|
|
1188
|
+
logger.warning(
|
|
1189
|
+
"Device auth /profile upstream error (attempt %s/%s): %s",
|
|
1190
|
+
attempt + 1,
|
|
1191
|
+
_PROFILE_FETCH_ATTEMPTS,
|
|
1192
|
+
detail,
|
|
1193
|
+
)
|
|
1194
|
+
last_upstream = {
|
|
1195
|
+
"ok": False,
|
|
1196
|
+
"reason": "upstream_error",
|
|
1197
|
+
"http_status": int(exc.code),
|
|
1198
|
+
"detail": detail,
|
|
1199
|
+
}
|
|
1200
|
+
if attempt + 1 < _PROFILE_FETCH_ATTEMPTS:
|
|
1201
|
+
time.sleep(_PROFILE_FETCH_RETRY_SLEEP_S)
|
|
1202
|
+
continue
|
|
1203
|
+
return last_upstream
|
|
1204
|
+
logger.warning("Failed to fetch API key: %s", detail)
|
|
1205
|
+
return {
|
|
1206
|
+
"ok": False,
|
|
1207
|
+
"reason": "http_error",
|
|
1208
|
+
"http_status": int(exc.code),
|
|
1209
|
+
"detail": detail,
|
|
1210
|
+
}
|
|
1211
|
+
except (URLError, OSError) as exc:
|
|
1212
|
+
logger.warning("Failed to fetch API key: %s", exc)
|
|
1213
|
+
return {"ok": False, "reason": "request_failed", "detail": str(exc)}
|
|
1214
|
+
|
|
1215
|
+
if data is None:
|
|
1216
|
+
# Defensive: loop exited without success or an explicit return.
|
|
1217
|
+
return last_upstream or {
|
|
1218
|
+
"ok": False,
|
|
1219
|
+
"reason": "request_failed",
|
|
1220
|
+
"detail": "No profile response",
|
|
1221
|
+
}
|
|
1222
|
+
|
|
1223
|
+
api_key = data.get("api_key")
|
|
1224
|
+
if not api_key:
|
|
1225
|
+
# The /profile response parsed but carried no api_key — the common case
|
|
1226
|
+
# for a GitHub account already linked to a ScrapeCreators account. Log
|
|
1227
|
+
# the response's FIELD NAMES only (never values — the body may contain a
|
|
1228
|
+
# key under a different field) so the already-registered response shape
|
|
1229
|
+
# can be handled in a follow-up (see plan OQ1).
|
|
1230
|
+
logger.warning(
|
|
1231
|
+
"Device auth /profile returned no api_key (fields: %s)", sorted(data.keys())
|
|
1232
|
+
)
|
|
1233
|
+
return {"ok": False, "reason": "no_api_key"}
|
|
1234
|
+
return {"ok": True, "api_key": api_key}
|
|
1235
|
+
|
|
1236
|
+
|
|
1237
|
+
def _device_handle_path() -> Path:
|
|
1238
|
+
"""Where run_github_start persists the device_code/interval for run_github_poll.
|
|
1239
|
+
|
|
1240
|
+
Kept next to the .env in the config dir; falls back to the OS temp dir when
|
|
1241
|
+
no config dir is resolvable (clean/no-config mode).
|
|
743
1242
|
"""
|
|
744
1243
|
try:
|
|
745
|
-
|
|
746
|
-
req.add_header("Authorization", f"Bearer {access_token}")
|
|
747
|
-
with urlopen(req, timeout=15) as resp:
|
|
748
|
-
data = json.loads(resp.read())
|
|
749
|
-
except (HTTPError, URLError, OSError) as exc:
|
|
750
|
-
logger.warning("Failed to fetch API key: %s", exc)
|
|
751
|
-
return None
|
|
1244
|
+
from . import env as _env
|
|
752
1245
|
|
|
753
|
-
|
|
1246
|
+
if _env.CONFIG_FILE:
|
|
1247
|
+
return _env.CONFIG_FILE.parent / ".github-device-handle.json"
|
|
1248
|
+
except Exception:
|
|
1249
|
+
pass
|
|
1250
|
+
import tempfile
|
|
754
1251
|
|
|
1252
|
+
return Path(tempfile.gettempdir()) / "last30days-github-device-handle.json"
|
|
755
1253
|
|
|
756
|
-
def run_full_device_auth(timeout: int = 300) -> Dict[str, Any]:
|
|
757
|
-
"""Run the complete GitHub device auth flow and return JSON-serializable result.
|
|
758
1254
|
|
|
759
|
-
|
|
760
|
-
|
|
1255
|
+
def _start_device_flow() -> "Tuple[Dict[str, Any], Optional[Dict[str, Any]]]":
|
|
1256
|
+
"""Submit the GitHub device flow and surface the code, without polling.
|
|
761
1257
|
|
|
762
|
-
Returns
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
1258
|
+
Returns ``(public_result, handle)``. ``handle`` is None for the
|
|
1259
|
+
already-registered and error cases (nothing to poll); otherwise it carries
|
|
1260
|
+
the private poll state (``device_code``/``interval``/``user_code``/
|
|
1261
|
+
``clipboard_ok``) that never belongs in the public, stdout-printed result.
|
|
1262
|
+
Callers either persist the handle to a file (``run_github_start``, for a
|
|
1263
|
+
separate poll process) or hand it straight to ``run_github_poll`` in-memory
|
|
1264
|
+
(``run_full_device_auth``, so a failed file write can't strand the one-shot).
|
|
768
1265
|
"""
|
|
1266
|
+
import sys
|
|
769
1267
|
import webbrowser
|
|
770
1268
|
|
|
771
|
-
#
|
|
1269
|
+
# Already-registered short-circuit: a saved key means no device dance. The
|
|
1270
|
+
# key is returned raw here and masked at the CLI boundary before print.
|
|
1271
|
+
existing = _existing_scrapecreators_key()
|
|
1272
|
+
if existing:
|
|
1273
|
+
return (
|
|
1274
|
+
{
|
|
1275
|
+
"status": "already_registered",
|
|
1276
|
+
"method": "existing",
|
|
1277
|
+
"api_key": existing,
|
|
1278
|
+
"persisted": True,
|
|
1279
|
+
},
|
|
1280
|
+
None,
|
|
1281
|
+
)
|
|
1282
|
+
|
|
772
1283
|
result = run_device_auth()
|
|
773
1284
|
if result is None:
|
|
774
|
-
return {"status": "error", "message": "Failed to start device auth flow"}
|
|
1285
|
+
return ({"status": "error", "message": "Failed to start device auth flow"}, None)
|
|
775
1286
|
|
|
776
1287
|
device_code, user_code, verification_uri, interval = result
|
|
777
1288
|
|
|
778
|
-
|
|
1289
|
+
# Validate the code shape BEFORE copying, labeling, or emitting it. A
|
|
1290
|
+
# non-conforming user_code (e.g. a key-shaped value) is never surfaced as a
|
|
1291
|
+
# GitHub device code; we stop rather than instruct the user to paste garbage.
|
|
1292
|
+
if not _DEVICE_CODE_RE.match(user_code):
|
|
1293
|
+
logger.warning("Device auth returned a non-device-shaped user_code; aborting.")
|
|
1294
|
+
return (
|
|
1295
|
+
{
|
|
1296
|
+
"status": "error",
|
|
1297
|
+
"message": "ScrapeCreators returned an unexpected device-code format.",
|
|
1298
|
+
},
|
|
1299
|
+
None,
|
|
1300
|
+
)
|
|
1301
|
+
|
|
1302
|
+
# Structured stdout line for machine consumers.
|
|
1303
|
+
print(
|
|
1304
|
+
json.dumps(
|
|
1305
|
+
{
|
|
1306
|
+
"event": "device_code_ready",
|
|
1307
|
+
"user_code": user_code,
|
|
1308
|
+
"verification_uri": verification_uri,
|
|
1309
|
+
}
|
|
1310
|
+
),
|
|
1311
|
+
flush=True,
|
|
1312
|
+
)
|
|
779
1313
|
|
|
780
|
-
#
|
|
1314
|
+
# Copy the code to the clipboard BEFORE opening the browser.
|
|
781
1315
|
clipboard_ok = False
|
|
782
1316
|
if sys.platform == "darwin":
|
|
783
1317
|
try:
|
|
784
|
-
subprocess.run(
|
|
785
|
-
["pbcopy"], input=user_code.encode(), check=True, timeout=5,
|
|
786
|
-
)
|
|
1318
|
+
subprocess.run(["pbcopy"], input=user_code.encode(), check=True, timeout=5)
|
|
787
1319
|
clipboard_ok = True
|
|
788
1320
|
except Exception:
|
|
789
1321
|
pass # pbcopy unavailable or failed, fall through
|
|
790
1322
|
|
|
791
|
-
#
|
|
1323
|
+
# Print the code as a plain HUMAN line on stdout too, so a foreground caller
|
|
1324
|
+
# sees it in the returned output even without reading the JSON. The clipboard
|
|
1325
|
+
# claim is only made when pbcopy actually succeeded (else: type it).
|
|
1326
|
+
if clipboard_ok:
|
|
1327
|
+
print(
|
|
1328
|
+
f"Your GitHub code: {user_code} (already on your clipboard - just paste it, Cmd+V)",
|
|
1329
|
+
flush=True,
|
|
1330
|
+
)
|
|
1331
|
+
else:
|
|
1332
|
+
print(f"Your GitHub code: {user_code} (type it on the GitHub page)", flush=True)
|
|
1333
|
+
|
|
1334
|
+
# Human box on stderr for direct-terminal users.
|
|
792
1335
|
clipboard_hint = " (copied to clipboard)" if clipboard_ok else ""
|
|
793
1336
|
code_line = f" Your code: {user_code}{clipboard_hint}"
|
|
794
1337
|
action_line = " Paste it on the GitHub page that just opened"
|
|
@@ -805,26 +1348,143 @@ def run_full_device_auth(timeout: int = 300) -> Dict[str, Any]:
|
|
|
805
1348
|
except Exception:
|
|
806
1349
|
print(f"Open: {verification_uri}", file=sys.stderr)
|
|
807
1350
|
|
|
808
|
-
|
|
1351
|
+
public = {
|
|
1352
|
+
"status": "awaiting_authorization",
|
|
1353
|
+
"user_code": user_code,
|
|
1354
|
+
"verification_uri": verification_uri,
|
|
1355
|
+
"clipboard_ok": clipboard_ok,
|
|
1356
|
+
}
|
|
1357
|
+
handle = {
|
|
1358
|
+
"device_code": device_code,
|
|
1359
|
+
"interval": interval,
|
|
1360
|
+
"user_code": user_code,
|
|
1361
|
+
"clipboard_ok": clipboard_ok,
|
|
1362
|
+
}
|
|
1363
|
+
return (public, handle)
|
|
1364
|
+
|
|
1365
|
+
|
|
1366
|
+
def run_github_start() -> Dict[str, Any]:
|
|
1367
|
+
"""Start the device flow and persist the poll handle for a later
|
|
1368
|
+
``run_github_poll`` process. Returns the public result (never the private
|
|
1369
|
+
device_code). See ``_start_device_flow`` for the returned statuses."""
|
|
1370
|
+
public, handle = _start_device_flow()
|
|
1371
|
+
if handle is not None:
|
|
1372
|
+
# Persist the poll handle (0o600) so a separate --github-poll process can
|
|
1373
|
+
# resume it. Best-effort: the in-memory one-shot path does not depend on
|
|
1374
|
+
# this write succeeding.
|
|
1375
|
+
path = _device_handle_path()
|
|
1376
|
+
try:
|
|
1377
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1378
|
+
write_private_text(path, json.dumps(handle), encoding="utf-8")
|
|
1379
|
+
if os.name != "posix":
|
|
1380
|
+
os.chmod(path, 0o600)
|
|
1381
|
+
except Exception as exc:
|
|
1382
|
+
logger.warning("Could not persist device handle: %s", exc)
|
|
1383
|
+
return public
|
|
1384
|
+
|
|
1385
|
+
|
|
1386
|
+
def run_github_poll(timeout: int = 300, *, _handle: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
1387
|
+
"""Poll for authorization using the handle from start.
|
|
1388
|
+
|
|
1389
|
+
``_handle`` (in-memory, from the one-shot) takes precedence over the
|
|
1390
|
+
persisted handle file. Returns success (with the fetched key), timeout, or
|
|
1391
|
+
the honest "Authorized but failed to fetch API key" branch. Deletes the
|
|
1392
|
+
persisted handle when the flow terminates.
|
|
1393
|
+
"""
|
|
1394
|
+
import sys
|
|
809
1395
|
|
|
810
|
-
|
|
1396
|
+
if _handle is not None:
|
|
1397
|
+
data = _handle
|
|
1398
|
+
else:
|
|
1399
|
+
try:
|
|
1400
|
+
data = json.loads(_device_handle_path().read_text(encoding="utf-8"))
|
|
1401
|
+
except Exception:
|
|
1402
|
+
return {
|
|
1403
|
+
"status": "error",
|
|
1404
|
+
"message": "No pending GitHub device flow; run setup --github-start first.",
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
device_code = data["device_code"]
|
|
1408
|
+
interval = _clamp_device_interval(data.get("interval", 5))
|
|
1409
|
+
user_code = data.get("user_code", "")
|
|
1410
|
+
# Read the real clipboard state so the polling reminder never falsely claims
|
|
1411
|
+
# the code is on the clipboard (non-macOS, or a failed pbcopy). Missing key
|
|
1412
|
+
# (older handle) defaults to False -- don't overstate.
|
|
1413
|
+
clipboard_ok = bool(data.get("clipboard_ok", False))
|
|
1414
|
+
|
|
1415
|
+
print("Waiting for authorization...", file=sys.stderr, flush=True)
|
|
811
1416
|
access_token = poll_device_auth(
|
|
812
|
-
device_code, interval, timeout=timeout,
|
|
813
|
-
user_code=user_code, clipboard_ok=clipboard_ok,
|
|
1417
|
+
device_code, interval, timeout=timeout, user_code=user_code, clipboard_ok=clipboard_ok
|
|
814
1418
|
)
|
|
1419
|
+
|
|
1420
|
+
def _cleanup() -> None:
|
|
1421
|
+
try:
|
|
1422
|
+
_device_handle_path().unlink()
|
|
1423
|
+
except Exception:
|
|
1424
|
+
pass
|
|
1425
|
+
|
|
815
1426
|
if access_token is None:
|
|
816
|
-
|
|
1427
|
+
_cleanup()
|
|
1428
|
+
return {"status": "timeout", "user_code": user_code}
|
|
1429
|
+
|
|
1430
|
+
fetched = fetch_api_key(access_token)
|
|
1431
|
+
_cleanup()
|
|
1432
|
+
if fetched.get("ok") and fetched.get("api_key"):
|
|
1433
|
+
return {
|
|
1434
|
+
"status": "success",
|
|
1435
|
+
"method": "device",
|
|
1436
|
+
"api_key": fetched["api_key"],
|
|
1437
|
+
"user_code": user_code,
|
|
1438
|
+
}
|
|
817
1439
|
|
|
818
|
-
#
|
|
819
|
-
|
|
820
|
-
|
|
1440
|
+
# Discriminate failure modes so SKILL.md does not misdiagnose a 5xx as
|
|
1441
|
+
# "GitHub already linked" (#882). Keep the historical message only for the
|
|
1442
|
+
# 2xx-without-api_key / already-linked case.
|
|
1443
|
+
reason = fetched.get("reason") or "request_failed"
|
|
1444
|
+
if reason == "no_api_key":
|
|
821
1445
|
return {
|
|
822
1446
|
"status": "error",
|
|
823
1447
|
"message": "Authorized but failed to fetch API key",
|
|
824
|
-
"
|
|
1448
|
+
"reason": "no_api_key",
|
|
825
1449
|
}
|
|
1450
|
+
if reason == "upstream_error":
|
|
1451
|
+
http_status = fetched.get("http_status")
|
|
1452
|
+
return {
|
|
1453
|
+
"status": "error",
|
|
1454
|
+
"message": f"Authorized but ScrapeCreators profile failed (HTTP {http_status})",
|
|
1455
|
+
"reason": "upstream_error",
|
|
1456
|
+
"http_status": http_status,
|
|
1457
|
+
"detail": fetched.get("detail"),
|
|
1458
|
+
}
|
|
1459
|
+
if reason == "http_error":
|
|
1460
|
+
http_status = fetched.get("http_status")
|
|
1461
|
+
return {
|
|
1462
|
+
"status": "error",
|
|
1463
|
+
"message": f"Authorized but failed to fetch API key (HTTP {http_status})",
|
|
1464
|
+
"reason": "http_error",
|
|
1465
|
+
"http_status": http_status,
|
|
1466
|
+
"detail": fetched.get("detail"),
|
|
1467
|
+
}
|
|
1468
|
+
return {
|
|
1469
|
+
"status": "error",
|
|
1470
|
+
"message": "Authorized but failed to fetch API key",
|
|
1471
|
+
"reason": reason,
|
|
1472
|
+
"detail": fetched.get("detail"),
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1475
|
+
|
|
1476
|
+
def run_full_device_auth(timeout: int = 300) -> Dict[str, Any]:
|
|
1477
|
+
"""Back-compat one-shot: start the device flow, then poll to completion.
|
|
826
1478
|
|
|
827
|
-
|
|
1479
|
+
Passes the poll handle to ``run_github_poll`` IN MEMORY, so a failed handle-
|
|
1480
|
+
file write can't strand the one-shot. Kept so callers of ``setup --github`` /
|
|
1481
|
+
``--device-auth`` still work; the model-driven wizard uses the two-command
|
|
1482
|
+
split (start then poll) instead.
|
|
1483
|
+
"""
|
|
1484
|
+
public, handle = _start_device_flow()
|
|
1485
|
+
if handle is None:
|
|
1486
|
+
return public # already_registered or error
|
|
1487
|
+
return run_github_poll(timeout=timeout, _handle=handle)
|
|
828
1488
|
|
|
829
1489
|
|
|
830
1490
|
# ---------------------------------------------------------------------------
|
|
@@ -833,11 +1493,10 @@ def run_full_device_auth(timeout: int = 300) -> Dict[str, Any]:
|
|
|
833
1493
|
|
|
834
1494
|
|
|
835
1495
|
def run_github_auth(timeout: int = 300) -> Dict[str, Any]:
|
|
836
|
-
"""Run the --github setup path via device auth
|
|
837
|
-
|
|
838
|
-
Kept as the semantic entry point for GitHub-backed setup; this path must
|
|
839
|
-
not read or forward local GitHub CLI tokens.
|
|
1496
|
+
"""Run the --github setup path via device auth (one-shot, back-compat).
|
|
840
1497
|
|
|
841
|
-
|
|
1498
|
+
The existing-key short-circuit now lives in run_github_start; this delegates
|
|
1499
|
+
to the start+poll chain. This path must not read or forward local GitHub CLI
|
|
1500
|
+
tokens.
|
|
842
1501
|
"""
|
|
843
1502
|
return run_full_device_auth(timeout=timeout)
|