@geoly-ai/social-hub-cli 0.3.43 → 0.3.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/cli-error-channels.test.js +40 -0
- package/dist/cli-error-channels.test.js.map +1 -1
- package/dist/client.d.ts +7 -1
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +19 -2
- package/dist/client.js.map +1 -1
- package/dist/cmd-manifest.json +12 -2
- package/dist/cmd-manifest.test.js +13 -0
- package/dist/cmd-manifest.test.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -3
- package/dist/index.js.map +1 -1
- package/dist/output.d.ts +12 -0
- package/dist/output.d.ts.map +1 -1
- package/dist/output.js +42 -0
- package/dist/output.js.map +1 -1
- package/dist/permissions-gates-restrictions.d.ts +10 -0
- package/dist/permissions-gates-restrictions.d.ts.map +1 -0
- package/dist/permissions-gates-restrictions.js +53 -0
- package/dist/permissions-gates-restrictions.js.map +1 -0
- package/dist/permissions-gates-restrictions.test.d.ts +2 -0
- package/dist/permissions-gates-restrictions.test.d.ts.map +1 -0
- package/dist/permissions-gates-restrictions.test.js +37 -0
- package/dist/permissions-gates-restrictions.test.js.map +1 -0
- package/dist/permissions.d.ts.map +1 -1
- package/dist/permissions.js +2 -0
- package/dist/permissions.js.map +1 -1
- package/dist/register-restrictions.d.ts +64 -0
- package/dist/register-restrictions.d.ts.map +1 -0
- package/dist/register-restrictions.js +431 -0
- package/dist/register-restrictions.js.map +1 -0
- package/dist/register-restrictions.test.d.ts +2 -0
- package/dist/register-restrictions.test.d.ts.map +1 -0
- package/dist/register-restrictions.test.js +315 -0
- package/dist/register-restrictions.test.js.map +1 -0
- package/package.json +3 -3
- package/skills/integrity.json +26 -25
- package/skills/manifest.json +2 -2
- package/skills/reddit-answers-research/SKILL.md +1 -1
- package/skills/reddit-matrix.lock.json +5 -5
- package/skills/reddit-phase-brief/references/phase4-brief-workflow.md +2 -2
- package/skills/reddit-phase-research/references/phase01-research.md +2 -2
- package/skills/reddit-strategy-shared/references/version-manifest.md +2 -2
- package/skills/reddit-voc-volume/SKILL.md +35 -28
- package/skills/reddit-voc-volume/matrix-contract.md +1 -1
- package/skills/reddit-voc-volume/references/arctic-shift.md +8 -7
- package/skills/reddit-voc-volume/references/browser-fallback.md +9 -4
- package/skills/reddit-voc-volume/requirements.txt +3 -1
- package/skills/reddit-voc-volume/scripts/reddit_voc_volume.py +3 -20
- package/skills/reddit-voc-volume/services/firecrawl/examples/o1_web_crawler/o1_web_crawler.py +152 -152
- package/skills/reddit-voc-volume/tests/test_links_fallback.py +87 -0
- package/skills/social-hub-admin/SKILL.md +33 -0
- package/skills/social-hub-calendar-directives/SKILL.md +14 -1
- package/skills/social-hub-calendar-jobs/SKILL.md +14 -0
- package/skills/social-hub-cli/SKILL.md +2 -0
- package/skills/social-hub-content-classification/SKILL.md +4 -0
- package/skills/social-hub-graph-compliance/SKILL.md +42 -0
- package/skills/social-hub-intelligence/SKILL.md +24 -1
- package/skills/social-hub-openclaw-context/SKILL.md +21 -16
- package/skills/social-hub-ops-runtime/SKILL.md +44 -0
- package/skills/social-hub-publishing/SKILL.md +12 -0
- package/skills/social-hub-scrape/SKILL.md +18 -16
package/skills/reddit-voc-volume/services/firecrawl/examples/o1_web_crawler/o1_web_crawler.py
CHANGED
|
@@ -1,152 +1,152 @@
|
|
|
1
|
-
import os
|
|
2
|
-
from firecrawl import FirecrawlApp
|
|
3
|
-
import json
|
|
4
|
-
from dotenv import load_dotenv
|
|
5
|
-
from openai import OpenAI
|
|
6
|
-
|
|
7
|
-
# ANSI color codes
|
|
8
|
-
class Colors:
|
|
9
|
-
CYAN = '\033[96m'
|
|
10
|
-
YELLOW = '\033[93m'
|
|
11
|
-
GREEN = '\033[92m'
|
|
12
|
-
RED = '\033[91m'
|
|
13
|
-
MAGENTA = '\033[95m'
|
|
14
|
-
BLUE = '\033[94m'
|
|
15
|
-
RESET = '\033[0m'
|
|
16
|
-
|
|
17
|
-
# Load environment variables
|
|
18
|
-
load_dotenv()
|
|
19
|
-
|
|
20
|
-
# Retrieve API keys from environment variables
|
|
21
|
-
firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY")
|
|
22
|
-
openai_api_key = os.getenv("OPENAI_API_KEY")
|
|
23
|
-
|
|
24
|
-
# Initialize the FirecrawlApp and OpenAI client
|
|
25
|
-
app = FirecrawlApp(api_key=firecrawl_api_key)
|
|
26
|
-
client = OpenAI(api_key=openai_api_key)
|
|
27
|
-
|
|
28
|
-
# Find the page that most likely contains the objective
|
|
29
|
-
def find_relevant_page_via_map(objective, url, app, client):
|
|
30
|
-
try:
|
|
31
|
-
print(f"{Colors.CYAN}Understood. The objective is: {objective}{Colors.RESET}")
|
|
32
|
-
print(f"{Colors.CYAN}Initiating search on the website: {url}{Colors.RESET}")
|
|
33
|
-
|
|
34
|
-
map_prompt = f"""
|
|
35
|
-
The map function generates a list of URLs from a website and it accepts a search parameter. Based on the objective of: {objective}, come up with a 1-2 word search parameter that will help us find the information we need. Only respond with 1-2 words nothing else.
|
|
36
|
-
"""
|
|
37
|
-
|
|
38
|
-
print(f"{Colors.YELLOW}Analyzing objective to determine optimal search parameter...{Colors.RESET}")
|
|
39
|
-
completion = client.chat.completions.create(
|
|
40
|
-
model="o1-preview",
|
|
41
|
-
messages=[
|
|
42
|
-
{
|
|
43
|
-
"role": "user",
|
|
44
|
-
"content": [
|
|
45
|
-
{
|
|
46
|
-
"type": "text",
|
|
47
|
-
"text": map_prompt
|
|
48
|
-
}
|
|
49
|
-
]
|
|
50
|
-
}
|
|
51
|
-
]
|
|
52
|
-
)
|
|
53
|
-
|
|
54
|
-
map_search_parameter = completion.choices[0].message.content
|
|
55
|
-
print(f"{Colors.GREEN}Optimal search parameter identified: {map_search_parameter}{Colors.RESET}")
|
|
56
|
-
|
|
57
|
-
print(f"{Colors.YELLOW}Mapping website using the identified search parameter...{Colors.RESET}")
|
|
58
|
-
map_website = app.map_url(url, params={"search": map_search_parameter})
|
|
59
|
-
print(f"{Colors.GREEN}Website mapping completed successfully.{Colors.RESET}")
|
|
60
|
-
print(f"{Colors.GREEN}Located {len(map_website)} relevant links.{Colors.RESET}")
|
|
61
|
-
return map_website
|
|
62
|
-
except Exception as e:
|
|
63
|
-
print(f"{Colors.RED}Error encountered during relevant page identification: {str(e)}{Colors.RESET}")
|
|
64
|
-
return None
|
|
65
|
-
|
|
66
|
-
# Scrape the top 3 pages and see if the objective is met, if so return in json format else return None
|
|
67
|
-
def find_objective_in_top_pages(map_website, objective, app, client):
|
|
68
|
-
try:
|
|
69
|
-
# Get top 3 links from the map result
|
|
70
|
-
top_links = map_website[:3] if isinstance(map_website, list) else []
|
|
71
|
-
print(f"{Colors.CYAN}Proceeding to analyze top {len(top_links)} links: {top_links}{Colors.RESET}")
|
|
72
|
-
|
|
73
|
-
for link in top_links:
|
|
74
|
-
print(f"{Colors.YELLOW}Initiating scrape of page: {link}{Colors.RESET}")
|
|
75
|
-
# Scrape the page
|
|
76
|
-
scrape_result = app.scrape_url(link, params={'formats': ['markdown']})
|
|
77
|
-
print(f"{Colors.GREEN}Page scraping completed successfully.{Colors.RESET}")
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
# Check if objective is met
|
|
81
|
-
check_prompt = f"""
|
|
82
|
-
Given the following scraped content and objective, determine if the objective is met.
|
|
83
|
-
If it is, extract the relevant information in a simple and concise JSON format. Use only the necessary fields and avoid nested structures if possible.
|
|
84
|
-
If the objective is not met with confidence, respond with 'Objective not met'.
|
|
85
|
-
|
|
86
|
-
Objective: {objective}
|
|
87
|
-
Scraped content: {scrape_result['markdown']}
|
|
88
|
-
|
|
89
|
-
Remember:
|
|
90
|
-
1. Only return JSON if you are confident the objective is fully met.
|
|
91
|
-
2. Keep the JSON structure as simple and flat as possible.
|
|
92
|
-
3. Do not include any explanations or markdown formatting in your response.
|
|
93
|
-
"""
|
|
94
|
-
|
|
95
|
-
completion = client.chat.completions.create(
|
|
96
|
-
model="o1-preview",
|
|
97
|
-
messages=[
|
|
98
|
-
{
|
|
99
|
-
"role": "user",
|
|
100
|
-
"content": [
|
|
101
|
-
{
|
|
102
|
-
"type": "text",
|
|
103
|
-
"text": check_prompt
|
|
104
|
-
}
|
|
105
|
-
]
|
|
106
|
-
}
|
|
107
|
-
]
|
|
108
|
-
)
|
|
109
|
-
|
|
110
|
-
result = completion.choices[0].message.content
|
|
111
|
-
|
|
112
|
-
if result != "Objective not met":
|
|
113
|
-
print(f"{Colors.GREEN}Objective potentially fulfilled. Relevant information identified.{Colors.RESET}")
|
|
114
|
-
try:
|
|
115
|
-
return json.loads(result)
|
|
116
|
-
except json.JSONDecodeError:
|
|
117
|
-
print(f"{Colors.RED}Error in parsing response. Proceeding to next page...{Colors.RESET}")
|
|
118
|
-
else:
|
|
119
|
-
print(f"{Colors.YELLOW}Objective not met on this page. Proceeding to next link...{Colors.RESET}")
|
|
120
|
-
|
|
121
|
-
print(f"{Colors.RED}All available pages analyzed. Objective not fulfilled in examined content.{Colors.RESET}")
|
|
122
|
-
return None
|
|
123
|
-
|
|
124
|
-
except Exception as e:
|
|
125
|
-
print(f"{Colors.RED}Error encountered during page analysis: {str(e)}{Colors.RESET}")
|
|
126
|
-
return None
|
|
127
|
-
|
|
128
|
-
# Main function to execute the process
|
|
129
|
-
def main():
|
|
130
|
-
# Get user input
|
|
131
|
-
url = input(f"{Colors.BLUE}Enter the website to crawl: {Colors.RESET}")
|
|
132
|
-
objective = input(f"{Colors.BLUE}Enter your objective: {Colors.RESET}")
|
|
133
|
-
|
|
134
|
-
print(f"{Colors.YELLOW}Initiating web crawling process...{Colors.RESET}")
|
|
135
|
-
# Find the relevant page
|
|
136
|
-
map_website = find_relevant_page_via_map(objective, url, app, client)
|
|
137
|
-
|
|
138
|
-
if map_website:
|
|
139
|
-
print(f"{Colors.GREEN}Relevant pages identified. Proceeding with detailed analysis...{Colors.RESET}")
|
|
140
|
-
# Find objective in top pages
|
|
141
|
-
result = find_objective_in_top_pages(map_website, objective, app, client)
|
|
142
|
-
|
|
143
|
-
if result:
|
|
144
|
-
print(f"{Colors.GREEN}Objective successfully fulfilled. Extracted information:{Colors.RESET}")
|
|
145
|
-
print(f"{Colors.MAGENTA}{json.dumps(result, indent=2)}{Colors.RESET}")
|
|
146
|
-
else:
|
|
147
|
-
print(f"{Colors.RED}Unable to fulfill the objective with the available content.{Colors.RESET}")
|
|
148
|
-
else:
|
|
149
|
-
print(f"{Colors.RED}No relevant pages identified. Consider refining the search parameters or trying a different website.{Colors.RESET}")
|
|
150
|
-
|
|
151
|
-
if __name__ == "__main__":
|
|
152
|
-
main()
|
|
1
|
+
import os
|
|
2
|
+
from firecrawl import FirecrawlApp
|
|
3
|
+
import json
|
|
4
|
+
from dotenv import load_dotenv
|
|
5
|
+
from openai import OpenAI
|
|
6
|
+
|
|
7
|
+
# ANSI color codes
|
|
8
|
+
class Colors:
|
|
9
|
+
CYAN = '\033[96m'
|
|
10
|
+
YELLOW = '\033[93m'
|
|
11
|
+
GREEN = '\033[92m'
|
|
12
|
+
RED = '\033[91m'
|
|
13
|
+
MAGENTA = '\033[95m'
|
|
14
|
+
BLUE = '\033[94m'
|
|
15
|
+
RESET = '\033[0m'
|
|
16
|
+
|
|
17
|
+
# Load environment variables
|
|
18
|
+
load_dotenv()
|
|
19
|
+
|
|
20
|
+
# Retrieve API keys from environment variables
|
|
21
|
+
firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY")
|
|
22
|
+
openai_api_key = os.getenv("OPENAI_API_KEY")
|
|
23
|
+
|
|
24
|
+
# Initialize the FirecrawlApp and OpenAI client
|
|
25
|
+
app = FirecrawlApp(api_key=firecrawl_api_key)
|
|
26
|
+
client = OpenAI(api_key=openai_api_key)
|
|
27
|
+
|
|
28
|
+
# Find the page that most likely contains the objective
|
|
29
|
+
def find_relevant_page_via_map(objective, url, app, client):
|
|
30
|
+
try:
|
|
31
|
+
print(f"{Colors.CYAN}Understood. The objective is: {objective}{Colors.RESET}")
|
|
32
|
+
print(f"{Colors.CYAN}Initiating search on the website: {url}{Colors.RESET}")
|
|
33
|
+
|
|
34
|
+
map_prompt = f"""
|
|
35
|
+
The map function generates a list of URLs from a website and it accepts a search parameter. Based on the objective of: {objective}, come up with a 1-2 word search parameter that will help us find the information we need. Only respond with 1-2 words nothing else.
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
print(f"{Colors.YELLOW}Analyzing objective to determine optimal search parameter...{Colors.RESET}")
|
|
39
|
+
completion = client.chat.completions.create(
|
|
40
|
+
model="o1-preview",
|
|
41
|
+
messages=[
|
|
42
|
+
{
|
|
43
|
+
"role": "user",
|
|
44
|
+
"content": [
|
|
45
|
+
{
|
|
46
|
+
"type": "text",
|
|
47
|
+
"text": map_prompt
|
|
48
|
+
}
|
|
49
|
+
]
|
|
50
|
+
}
|
|
51
|
+
]
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
map_search_parameter = completion.choices[0].message.content
|
|
55
|
+
print(f"{Colors.GREEN}Optimal search parameter identified: {map_search_parameter}{Colors.RESET}")
|
|
56
|
+
|
|
57
|
+
print(f"{Colors.YELLOW}Mapping website using the identified search parameter...{Colors.RESET}")
|
|
58
|
+
map_website = app.map_url(url, params={"search": map_search_parameter})
|
|
59
|
+
print(f"{Colors.GREEN}Website mapping completed successfully.{Colors.RESET}")
|
|
60
|
+
print(f"{Colors.GREEN}Located {len(map_website)} relevant links.{Colors.RESET}")
|
|
61
|
+
return map_website
|
|
62
|
+
except Exception as e:
|
|
63
|
+
print(f"{Colors.RED}Error encountered during relevant page identification: {str(e)}{Colors.RESET}")
|
|
64
|
+
return None
|
|
65
|
+
|
|
66
|
+
# Scrape the top 3 pages and see if the objective is met, if so return in json format else return None
|
|
67
|
+
def find_objective_in_top_pages(map_website, objective, app, client):
|
|
68
|
+
try:
|
|
69
|
+
# Get top 3 links from the map result
|
|
70
|
+
top_links = map_website[:3] if isinstance(map_website, list) else []
|
|
71
|
+
print(f"{Colors.CYAN}Proceeding to analyze top {len(top_links)} links: {top_links}{Colors.RESET}")
|
|
72
|
+
|
|
73
|
+
for link in top_links:
|
|
74
|
+
print(f"{Colors.YELLOW}Initiating scrape of page: {link}{Colors.RESET}")
|
|
75
|
+
# Scrape the page
|
|
76
|
+
scrape_result = app.scrape_url(link, params={'formats': ['markdown']})
|
|
77
|
+
print(f"{Colors.GREEN}Page scraping completed successfully.{Colors.RESET}")
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# Check if objective is met
|
|
81
|
+
check_prompt = f"""
|
|
82
|
+
Given the following scraped content and objective, determine if the objective is met.
|
|
83
|
+
If it is, extract the relevant information in a simple and concise JSON format. Use only the necessary fields and avoid nested structures if possible.
|
|
84
|
+
If the objective is not met with confidence, respond with 'Objective not met'.
|
|
85
|
+
|
|
86
|
+
Objective: {objective}
|
|
87
|
+
Scraped content: {scrape_result['markdown']}
|
|
88
|
+
|
|
89
|
+
Remember:
|
|
90
|
+
1. Only return JSON if you are confident the objective is fully met.
|
|
91
|
+
2. Keep the JSON structure as simple and flat as possible.
|
|
92
|
+
3. Do not include any explanations or markdown formatting in your response.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
completion = client.chat.completions.create(
|
|
96
|
+
model="o1-preview",
|
|
97
|
+
messages=[
|
|
98
|
+
{
|
|
99
|
+
"role": "user",
|
|
100
|
+
"content": [
|
|
101
|
+
{
|
|
102
|
+
"type": "text",
|
|
103
|
+
"text": check_prompt
|
|
104
|
+
}
|
|
105
|
+
]
|
|
106
|
+
}
|
|
107
|
+
]
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
result = completion.choices[0].message.content
|
|
111
|
+
|
|
112
|
+
if result != "Objective not met":
|
|
113
|
+
print(f"{Colors.GREEN}Objective potentially fulfilled. Relevant information identified.{Colors.RESET}")
|
|
114
|
+
try:
|
|
115
|
+
return json.loads(result)
|
|
116
|
+
except json.JSONDecodeError:
|
|
117
|
+
print(f"{Colors.RED}Error in parsing response. Proceeding to next page...{Colors.RESET}")
|
|
118
|
+
else:
|
|
119
|
+
print(f"{Colors.YELLOW}Objective not met on this page. Proceeding to next link...{Colors.RESET}")
|
|
120
|
+
|
|
121
|
+
print(f"{Colors.RED}All available pages analyzed. Objective not fulfilled in examined content.{Colors.RESET}")
|
|
122
|
+
return None
|
|
123
|
+
|
|
124
|
+
except Exception as e:
|
|
125
|
+
print(f"{Colors.RED}Error encountered during page analysis: {str(e)}{Colors.RESET}")
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
# Main function to execute the process
|
|
129
|
+
def main():
|
|
130
|
+
# Get user input
|
|
131
|
+
url = input(f"{Colors.BLUE}Enter the website to crawl: {Colors.RESET}")
|
|
132
|
+
objective = input(f"{Colors.BLUE}Enter your objective: {Colors.RESET}")
|
|
133
|
+
|
|
134
|
+
print(f"{Colors.YELLOW}Initiating web crawling process...{Colors.RESET}")
|
|
135
|
+
# Find the relevant page
|
|
136
|
+
map_website = find_relevant_page_via_map(objective, url, app, client)
|
|
137
|
+
|
|
138
|
+
if map_website:
|
|
139
|
+
print(f"{Colors.GREEN}Relevant pages identified. Proceeding with detailed analysis...{Colors.RESET}")
|
|
140
|
+
# Find objective in top pages
|
|
141
|
+
result = find_objective_in_top_pages(map_website, objective, app, client)
|
|
142
|
+
|
|
143
|
+
if result:
|
|
144
|
+
print(f"{Colors.GREEN}Objective successfully fulfilled. Extracted information:{Colors.RESET}")
|
|
145
|
+
print(f"{Colors.MAGENTA}{json.dumps(result, indent=2)}{Colors.RESET}")
|
|
146
|
+
else:
|
|
147
|
+
print(f"{Colors.RED}Unable to fulfill the objective with the available content.{Colors.RESET}")
|
|
148
|
+
else:
|
|
149
|
+
print(f"{Colors.RED}No relevant pages identified. Consider refining the search parameters or trying a different website.{Colors.RESET}")
|
|
150
|
+
|
|
151
|
+
if __name__ == "__main__":
|
|
152
|
+
main()
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import contextlib
|
|
4
|
+
import importlib.util
|
|
5
|
+
import io
|
|
6
|
+
import json
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
import sys
|
|
9
|
+
import unittest
|
|
10
|
+
from unittest import mock
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "reddit_voc_volume.py"
|
|
14
|
+
SPEC = importlib.util.spec_from_file_location("reddit_voc_links_under_test", SCRIPT)
|
|
15
|
+
assert SPEC is not None and SPEC.loader is not None
|
|
16
|
+
voc = importlib.util.module_from_spec(SPEC)
|
|
17
|
+
SPEC.loader.exec_module(voc)
|
|
18
|
+
POST_URL = "https://www.reddit.com/r/example/comments/abc123/slug/"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class LinksFallbackTests(unittest.TestCase):
|
|
22
|
+
def run_main(self, *args: str) -> tuple[int, dict]:
|
|
23
|
+
stdout = io.StringIO()
|
|
24
|
+
with mock.patch.object(sys, "argv", [str(SCRIPT), *args]), contextlib.redirect_stdout(stdout):
|
|
25
|
+
code = voc.main()
|
|
26
|
+
return code, json.loads(stdout.getvalue())
|
|
27
|
+
|
|
28
|
+
def test_arctic_miss_uses_rendered_page_without_loading_browser_cookies(self) -> None:
|
|
29
|
+
with (
|
|
30
|
+
mock.patch.object(voc.CookieStore, "get", side_effect=AssertionError("browser cookies loaded")) as cookies,
|
|
31
|
+
mock.patch.object(voc, "arctic_post_by_permalink", return_value=None) as archive,
|
|
32
|
+
mock.patch.object(voc, "fetch_page_scrape", return_value={"markdown": "Public page"}) as page,
|
|
33
|
+
mock.patch.object(voc, "collect_subreddit_metrics", return_value=[]),
|
|
34
|
+
mock.patch.object(voc, "hub_writeback_items", return_value={"status": "pending"}),
|
|
35
|
+
mock.patch.object(voc, "sleep_with_jitter", return_value=0.0),
|
|
36
|
+
):
|
|
37
|
+
code, out = self.run_main("links", POST_URL, "--fallback", "auto", "--sleep-sec", "0")
|
|
38
|
+
|
|
39
|
+
self.assertEqual(code, 0)
|
|
40
|
+
self.assertTrue(out["ok"])
|
|
41
|
+
self.assertEqual(out["items"][0]["item_type"], "firecrawl_page")
|
|
42
|
+
self.assertEqual(out["errors"], [])
|
|
43
|
+
archive.assert_called_once()
|
|
44
|
+
self.assertEqual(page.call_args.kwargs["provider"], "social-hub")
|
|
45
|
+
cookies.assert_not_called()
|
|
46
|
+
|
|
47
|
+
def test_arctic_miss_without_fallback_reports_archive_not_json(self) -> None:
|
|
48
|
+
with (
|
|
49
|
+
mock.patch.object(voc.CookieStore, "get", side_effect=AssertionError("browser cookies loaded")) as cookies,
|
|
50
|
+
mock.patch.object(voc, "arctic_post_by_permalink", side_effect=RuntimeError("archive unavailable")),
|
|
51
|
+
mock.patch.object(voc, "collect_subreddit_metrics", return_value=[]),
|
|
52
|
+
mock.patch.object(voc, "hub_writeback_items", return_value={"status": "not_applicable"}),
|
|
53
|
+
mock.patch.object(voc, "sleep_with_jitter", return_value=0.0),
|
|
54
|
+
):
|
|
55
|
+
code, out = self.run_main("links", POST_URL, "--fallback", "none", "--sleep-sec", "0")
|
|
56
|
+
|
|
57
|
+
self.assertEqual(code, 1)
|
|
58
|
+
self.assertFalse(out["ok"])
|
|
59
|
+
self.assertTrue(any("archive unavailable" in error for error in out["errors"]))
|
|
60
|
+
self.assertTrue(any("Arctic lookup failed" in note for note in out["notes"]))
|
|
61
|
+
cookies.assert_not_called()
|
|
62
|
+
|
|
63
|
+
def test_auto_fallback_tries_firecrawl_after_social_hub_failure(self) -> None:
|
|
64
|
+
def page_result(_url: str, *, provider: str, **_kwargs: object) -> dict:
|
|
65
|
+
if provider == "social-hub":
|
|
66
|
+
raise RuntimeError("render unavailable")
|
|
67
|
+
return {"markdown": "Public page"}
|
|
68
|
+
|
|
69
|
+
with (
|
|
70
|
+
mock.patch.object(voc.CookieStore, "get", side_effect=AssertionError("browser cookies loaded")) as cookies,
|
|
71
|
+
mock.patch.object(voc, "arctic_post_by_permalink", return_value=None),
|
|
72
|
+
mock.patch.object(voc, "fetch_page_scrape", side_effect=page_result) as page,
|
|
73
|
+
mock.patch.object(voc, "social_hub_unavailable_hint", return_value="Hub unavailable"),
|
|
74
|
+
mock.patch.object(voc, "collect_subreddit_metrics", return_value=[]),
|
|
75
|
+
mock.patch.object(voc, "hub_writeback_items", return_value={"status": "pending"}),
|
|
76
|
+
mock.patch.object(voc, "sleep_with_jitter", return_value=0.0),
|
|
77
|
+
):
|
|
78
|
+
code, out = self.run_main("links", POST_URL, "--fallback", "auto", "--sleep-sec", "0")
|
|
79
|
+
|
|
80
|
+
self.assertEqual(code, 0)
|
|
81
|
+
self.assertEqual(out["items"][0]["page_provider"], "firecrawl")
|
|
82
|
+
self.assertEqual([call.kwargs["provider"] for call in page.call_args_list], ["social-hub", "firecrawl"])
|
|
83
|
+
cookies.assert_not_called()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
if __name__ == "__main__":
|
|
87
|
+
unittest.main()
|
|
@@ -155,6 +155,39 @@ social-hub brand-auth revoke --account <ref> --brand <brandKey> [-t <team-id>] [
|
|
|
155
155
|
|
|
156
156
|
发布前三点 gate 语义见 [`../../docs/brand-mention-gate-operations.md`](../../docs/brand-mention-gate-operations.md)。
|
|
157
157
|
|
|
158
|
+
## 板块限制(restrictions,系统级整板限制)
|
|
159
|
+
|
|
160
|
+
整板 banned / private / quarantined / unavailable 的系统级限制。**一条限制会让全部 team 在该板块的写稿、排期、
|
|
161
|
+
发布同时停下**,所以管理面只给 **admin**,并且只接受**不受限的登录**(`auth login` 的 CLI token 或网页 session):
|
|
162
|
+
API key、绑定 job/account 或带动作白名单的 CLI token **读写都 403**(权限资源 `systemSubredditRestriction`,
|
|
163
|
+
override 只能收紧,manager 拿不到任何一格)。
|
|
164
|
+
|
|
165
|
+
```bash
|
|
166
|
+
# 读(系统级 admin 视图:list / fanout 必须显式 --system;不带 --system 走的是 team 只读路由,见下)
|
|
167
|
+
social-hub restrictions list --system [--state provisional,verification_overdue] [--updated-since <iso>] [--subreddit <name>] [--limit 50] [--offset 0]
|
|
168
|
+
social-hub restrictions get <id> # 含全量证据;list 只回证据摘要 evidenceSummary
|
|
169
|
+
social-hub restrictions fanout <id> --system [--version <n>] [--outcome blocked] [--team-id <uuid>]
|
|
170
|
+
# 全部 team 的扇出汇总计数 + complete + 回执明细。阶段 B(扇出 worker)上线前恒为空、complete:false —— 不是故障。
|
|
171
|
+
|
|
172
|
+
# 写:一律 --dry-run 预览 / --apply 执行
|
|
173
|
+
social-hub restrictions create --subreddit r/<name> --kind banned --evidence-level strong \
|
|
174
|
+
--reason "..." [--state provisional --expires-at <iso>] [--evidence '<json 数组>'] --apply
|
|
175
|
+
# 默认直接 confirmed;provisional 必须带 --expires-at。已有活跃限制 → 409 details.code=RESTRICTION_EXISTS
|
|
176
|
+
social-hub restrictions confirm <id> [--expected-version <n>] --apply # provisional / verification_overdue → confirmed
|
|
177
|
+
social-hub restrictions revoke <id> --reason "..." [--expected-version <n>] --apply # 唯一的解封路径
|
|
178
|
+
# 版本不符 → 409 VERSION_MISMATCH;状态不对 → 409 INVALID_STATE(都在 error.details.code)
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
状态机:`provisional`(已上报待复核,**照样拦**)→ 复核期限过了无人确认 → `verification_overdue`(**继续拦**);
|
|
182
|
+
admin `confirm` → `confirmed`;admin `revoke` → `revoked`(唯一不拦的状态,可再建新限制)。
|
|
183
|
+
revoke **不会**恢复已经被拦成 `blocked` 的 job,需要人重新排期。
|
|
184
|
+
|
|
185
|
+
**运营侧 agent 拉取状态(再去私信 admin)不走这里**:用 team 只读路由
|
|
186
|
+
`restrictions list -t <team-id>` / `restrictions fanout <id> -t <team-id>`(`compliance:read`,manager 可用,
|
|
187
|
+
受限 token 也放行;fanout 只含本 team 的回执),见 `social-hub-graph-compliance`。
|
|
188
|
+
runner 侧的 `restrictions check` / `restrictions report` 同样见 `social-hub-graph-compliance`;
|
|
189
|
+
执行链路上 409 `TARGET_BLOCKED` / `JOB_BLOCKED` 的识别与 agent 停止规则见 `social-hub-ops-runtime`。
|
|
190
|
+
|
|
158
191
|
## Agent teams
|
|
159
192
|
|
|
160
193
|
```bash
|
|
@@ -421,10 +421,23 @@ PATCH { "status": "succeeded", "permalink": "…", "executionToken": "<刚才那
|
|
|
421
421
|
| `execution_token_not_expected` | 初次抢占时带了 token | 代码 bug:抢占不该带 token |
|
|
422
422
|
| `expected_revision_not_applicable` | 终态回写带了 `expectedRevision` | 代码 bug:去掉它 |
|
|
423
423
|
| `revision_stale` / `revision_unknown` | 抢占时的版本钉子对不上 | 同 preflight 同名 reason 的处置 |
|
|
424
|
+
| `target_blocked` | 目标板块/账号受限(见下) | 删 cron、安静退出。**不重试**、不回写终态 |
|
|
424
425
|
|
|
425
|
-
⚠️ 这些都走 **HTTP 409**、`code = "EXECUTION_NOT_ACQUIRED"
|
|
426
|
+
⚠️ 这些都走 **HTTP 409**、`details.reason` 取上表的值;前六个的 `code = "EXECUTION_NOT_ACQUIRED"`。
|
|
426
427
|
**按 reason 编程分流,别拿人类文案做子串匹配。**
|
|
427
428
|
|
|
429
|
+
#### `target_blocked`:板块受限
|
|
430
|
+
|
|
431
|
+
抢 `scheduled → running` 时,若目标板块已被系统级限制(整板 banned / private / quarantined / unavailable,
|
|
432
|
+
含待复核的 provisional 与逾期未复核的 verification_overdue)或该账号在该板块有生效 sanction / bot_bouncer 名单,
|
|
433
|
+
服务端拒绝抢占:HTTP 409,`error.code = "CONFLICT"`,
|
|
434
|
+
`details = { code: "TARGET_BLOCKED", reason: "target_blocked", source, id, kind?, subreddit }`。
|
|
435
|
+
|
|
436
|
+
- 识别看 `details.reason === "target_blocked"`(或 `details.code === "TARGET_BLOCKED"`),不看 message。
|
|
437
|
+
- 你**没有**拿到 token,这一轮什么都没发生:删本机 cron、退出。**不要重试**——只有 admin 撤销限制才会解除。
|
|
438
|
+
- 已经持有 token 的在途执行不受影响:终态回写照常放行(服务端只在抢占那一步判定)。
|
|
439
|
+
- 条目随后会由服务端以 `cancel`(cancel reason `target_blocked`)广播撤销,按常规 cancel 处置即可。
|
|
440
|
+
|
|
428
441
|
## ack 时机(本 source 局部改写通用规则)
|
|
429
442
|
|
|
430
443
|
通用 notifications skill 说"副作用完成后才 ack"——这条仍然成立,但**这个 source 的副作用是
|
|
@@ -43,6 +43,20 @@ social-hub jobs retry -t <team-id> -i <jobId>
|
|
|
43
43
|
social-hub jobs cancel -t <team-id> -i <jobId>
|
|
44
44
|
```
|
|
45
45
|
|
|
46
|
+
## 🔴 板块受限:建/改排期被拒(409 `TARGET_BLOCKED`)
|
|
47
|
+
|
|
48
|
+
目标板块受系统级限制,或该账号在该板块有生效 sanction / bot_bouncer 名单时,
|
|
49
|
+
`jobs create` / `jobs batch` / `jobs retry`(以及建计划/排期、改期、切 delivery mode)会被拒:
|
|
50
|
+
HTTP **409**,`error.code = "CONFLICT"`,`error.details = { code: "TARGET_BLOCKED", reason: "target_blocked",
|
|
51
|
+
source, id, kind?, subreddit }`。
|
|
52
|
+
|
|
53
|
+
- **按 `error.details.code` 识别,不要匹配 message。**
|
|
54
|
+
- **不要重试、不要换个 runAt 再建**:重试只会拿到同一个 409。换一个目标板块,或跳过。解除路径看 `details.source`:`restriction` 只有 admin `restrictions revoke`;`sanction`(团队/账号级)由有权限的人把该 sanction 改为 lifted;`blocklist` 由 admin 维护(不是 agent 的事,报告即可)。
|
|
55
|
+
- 已被拦下的 job 状态是 `blocked`(`lastError` 形如 `target_blocked:<source>:<id>`),是**永久终态**:
|
|
56
|
+
`jobs retry` 只接受 failed / cancelled,解封后要**新建** job。
|
|
57
|
+
- 执行侧(heartbeat / complete / fail / skip)的受限语义与停止规则见 `social-hub-ops-runtime`;
|
|
58
|
+
自查与上报见 `social-hub-graph-compliance` 的 `restrictions check` / `restrictions report`。
|
|
59
|
+
|
|
46
60
|
## OpenClaw / Agent 运行时
|
|
47
61
|
|
|
48
62
|
优先 ops 链路(见 `social-hub-ops-runtime`):
|
|
@@ -46,6 +46,8 @@ social-hub version --json
|
|
|
46
46
|
| 帖子发布前审核/优化(post-review) | `social-hub-post-review` |
|
|
47
47
|
| 事件 / dashboard / audit | `social-hub-events-observability` |
|
|
48
48
|
| 图谱 / 合规 / 风控 | `social-hub-graph-compliance` |
|
|
49
|
+
| 板块限制拉取/自查/上报 restrictions list/fanout/check -t | `social-hub-graph-compliance` |
|
|
50
|
+
| 板块限制管理 restrictions create/revoke/--system | `social-hub-admin` |
|
|
49
51
|
| settings / invites / members / api-keys | `social-hub-admin` |
|
|
50
52
|
| brand-registry / brand-fit / trigger 注册表四眼 / brand-auth | `social-hub-admin` |
|
|
51
53
|
| 飞书身份映射同步(告警 @ 谁) | `social-hub-admin` |
|
|
@@ -42,6 +42,10 @@ worker(单实例 cron)按 active spec 对缺当前 revision 成功 run 的
|
|
|
42
42
|
这些 run 可能对应帖子的**旧 revision**。要判断某帖「当前」被分成什么,只能用
|
|
43
43
|
`get-subject`(精确唯一键查询,proposed / missing 二态)。**不要拿 `recent` 的
|
|
44
44
|
第一条当 latest 用。**
|
|
45
|
+
- **`recent` 没有分页,返回的是截断后的一页**:`count` / `returned` 是**本页条数,不是总数**;
|
|
46
|
+
`truncated: true` 表示还有更多(服务端多取一条判定,精确),`selectionLimit` 是本次生效上限。
|
|
47
|
+
`--limit` 未到 200 时可以调大(上限 200);**已是 200 仍 `truncated: true`,说明这个 recent 视图枚举不完余项**,
|
|
48
|
+
**不要自己拼 offset**——这个端点根本不接受分页参数。
|
|
45
49
|
|
|
46
50
|
## 读面
|
|
47
51
|
|
|
@@ -73,6 +73,48 @@ social-hub compliance rule-caches-upsert -t <team-id> --sub example -j '{
|
|
|
73
73
|
}'
|
|
74
74
|
```
|
|
75
75
|
|
|
76
|
+
## 板块限制:runner 自查与上报(`restrictions check` / `restrictions report`)
|
|
77
|
+
|
|
78
|
+
整板限制(banned / private / quarantined / unavailable)存在系统级 `subreddit_restrictions`,
|
|
79
|
+
**只有 admin 能直接建 / 确认 / 撤销**(见 `social-hub-admin`)。agent / runner 用下面这些 **team 级**入口:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
# 拉取限制状态(compliance:read;manager 可用,受限 token 也放行)。Hub 不推送、不发群:
|
|
83
|
+
# agent 按 --updated-since 增量拉取,自行私信 admin。
|
|
84
|
+
social-hub restrictions list -t <team-id> [--state provisional,verification_overdue] [--updated-since <iso>] [--subreddit <name>] [--limit 50] [--offset 0]
|
|
85
|
+
# → {items,total,limit,offset,hasMore};每项带 state / expiresAt / lastVerifiedAt / createdBy / updatedAt / evidenceSummary(不含全量证据)
|
|
86
|
+
social-hub restrictions fanout <restriction-id> -t <team-id> [--version <n>] [--outcome blocked]
|
|
87
|
+
# → 只统计、只返回**本 team** 的回执(counts / receipts);complete 按全局扇出判断;
|
|
88
|
+
# 阶段 B(扇出 worker)上线前恒为空、complete:false —— 不是故障。
|
|
89
|
+
# 缺省 -t 用当前 context 的 team;两者都没有会报错(不会改走 admin 面)。--system 是 admin 专用的系统级视图。
|
|
90
|
+
|
|
91
|
+
# 动手前自查(compliance:read)。不带 --account 只判系统级;带上才同时判该账号的 sanction。
|
|
92
|
+
social-hub restrictions check -t <team-id> --subreddit <name> [--account <accountId>]
|
|
93
|
+
# → {"allowed":true,...} 或 {"allowed":false,"scope":"system|account","source":"restriction|blocklist|sanction",
|
|
94
|
+
# "id":"...","kind":"...","state":"provisional|confirmed|verification_overdue|null","expiresAt":...}
|
|
95
|
+
# allowed:false = 本轮跳过这个目标,不要重试。
|
|
96
|
+
|
|
97
|
+
# 发现受限时上报(compliance:create;写命令,需 --apply,可先 --dry-run)
|
|
98
|
+
social-hub restrictions report -t <team-id> --subreddit <name> --account <accountId> \
|
|
99
|
+
--kind banned --evidence '[{"type":"banned_page","url":"https://www.reddit.com/r/<name>","observedAt":"<iso>"}]' \
|
|
100
|
+
--evidence-level strong --reason "banner: r/<name> has been banned" \
|
|
101
|
+
--expires-at <复核期限 iso> [--scope system] --apply
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
`report` 的落点(第一版刻意保守,**服务端封禁页解析器在阶段 C 才做**,现在信的是你的声明):
|
|
105
|
+
|
|
106
|
+
| 你传的 | 实际写入 | 响应 |
|
|
107
|
+
| -------------------------------------------- | ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------ |
|
|
108
|
+
| `--scope system` + `--evidence-level strong` | 系统级 **provisional**(立即对全部 team 生效,要求 manager/admin;supervisor → 403) | `target:"system"`,`created:false` = 板块已有限制,原样返回现行行摘要 |
|
|
109
|
+
| `--scope system` + `weak` | 降级为该 team+账号的 sanction | `target:"account"`,`downgraded:true`,`downgradeReason:"weak_evidence"` |
|
|
110
|
+
| 缺省 / `--scope account` | 该 team+账号的 sanction(banned → `banned`,其余 → `restricted`) | `target:"account"`,`downgraded:false` |
|
|
111
|
+
|
|
112
|
+
- 证据至少一条、`--expires-at` 必填(复核期限)。provisional 过期**不会**解封,而是转 `verification_overdue` 继续拦,
|
|
113
|
+
只有 admin `restrictions revoke` 才解封。
|
|
114
|
+
- 落 sanction 时**不会**把复核期限写成 sanction 过期(那会自动解封);期限记在 reason 前缀和事件里。
|
|
115
|
+
- 只有「整板对所有人都不可用」才用 `--scope system`。账号自己被 ban(「you've been banned from r/X」)用默认 account。
|
|
116
|
+
- 执行链路上被拦(409 `TARGET_BLOCKED` / `JOB_BLOCKED`)的识别与停止规则见 `social-hub-ops-runtime`。
|
|
117
|
+
|
|
76
118
|
## 系统级板块黑名单(Bot Bouncer 等)
|
|
77
119
|
|
|
78
120
|
与 **per-account** `sanctions-*` 不同:名单为 **系统级**(`/v1/system/subreddit-blocklists`),全 team 共享,表示 Reddit 平台客观事实(如 bot-bouncer 安装 sub),无需 `-t`。
|
|
@@ -259,11 +259,32 @@ watchlist 须显式传 `brandIds`。
|
|
|
259
259
|
```bash
|
|
260
260
|
social-hub intelligence monitors-list [--enabled true|false] [--watchlist <uuid>]
|
|
261
261
|
social-hub intelligence monitors-enable --watchlist <uuid> [--fetch-limit 100]
|
|
262
|
+
social-hub intelligence monitors-update --watchlist <uuid> --fetch-limit 50 # 只改配置,不碰启停
|
|
262
263
|
social-hub intelligence monitors-disable --watchlist <uuid> [--purge]
|
|
263
264
|
social-hub intelligence monitors-run # 手工触发一轮(异步入队)
|
|
264
265
|
social-hub intelligence monitor-runs-list [--status degraded] [-n 20]
|
|
265
266
|
```
|
|
266
267
|
|
|
268
|
+
### 🔴 改抓取上限用 `monitors-update`,⛔ 不要用 `monitors-enable`
|
|
269
|
+
|
|
270
|
+
`monitors-enable` 是个 **upsert**,它的 `DO UPDATE` 里写死了 `enabled = true` ——
|
|
271
|
+
所以拿它去改一个**已停止**监听的 `--fetch-limit`,会把那个监听**静默重新启用**,
|
|
272
|
+
该板块的新帖会立刻重新推给负责人。
|
|
273
|
+
|
|
274
|
+
`monitors-update` 背后的 SQL 里**根本不出现 `enabled` 列**,所以它没有能力改启停:
|
|
275
|
+
停止状态下改完仍是停止,启用状态下改完仍是启用。
|
|
276
|
+
|
|
277
|
+
| 我要做的事 | 用哪个命令 |
|
|
278
|
+
| ------------------- | ----------------------------------- |
|
|
279
|
+
| 加入监听 / 重新启用 | `monitors-enable` |
|
|
280
|
+
| **只改抓取上限** | `monitors-update`(启停状态原样不动) |
|
|
281
|
+
| 停止监听(软停) | `monitors-disable` |
|
|
282
|
+
|
|
283
|
+
⚠️ `monitors-update` **不会创建**监听:该 watchlist 从没配置过监听时返回 404
|
|
284
|
+
(要新建请显式用 `monitors-enable`)。
|
|
285
|
+
⚠️ `--fetch-limit` 两个命令同一套边界:整数 1–100。
|
|
286
|
+
⚠️ `sort` **没有任何写入面**,两个命令都不收 —— DDL 的 CHECK 只允许 `'new'` 一个值。
|
|
287
|
+
|
|
267
288
|
**停止监听 = 软停**(`enabled=false`),保留观察 ledger,重启后已报告过的帖子不会重复报告。
|
|
268
289
|
`--purge` 是另一个动作:**连历史观察一起硬删**,只有要删掉整个 watchlist 时才需要
|
|
269
290
|
(monitor 配置对 watchlist 的外键是 RESTRICT,不先 purge 就删不掉)——purge 后重新监听
|
|
@@ -604,4 +625,6 @@ WebSocket `runs/ws` 为 watch 能力,不适合普通 CLI 批处理。
|
|
|
604
625
|
|
|
605
626
|
## 通用网页抓取
|
|
606
627
|
|
|
607
|
-
本 skill 只覆盖**系统内部**的 Reddit 情报采集(落库 +
|
|
628
|
+
本 skill 只覆盖**系统内部**的 Reddit 情报采集(落库 + 聚合)。worker 侧仍先试 `.json`(curl_cffi + 住宅代理),**但 `.json` 已被 Reddit 全面拦截**(2026-09-18 实测住宅代理同样 403)——真正出数的是封禁熔断器之后的 **Firecrawl 渲染兜底**(需 worker 配 `FIRECRAWL_BASE_URL`,`REDDIT_FIRECRAWL_FALLBACK_ENABLED` 不显式关即生效)。
|
|
629
|
+
|
|
630
|
+
若要**按任意 URL 临时抓取非 Reddit 页面**(HTML/普通网页走 Firecrawl),用独立的 **`social-hub-scrape`** skill(`social-hub scrape url …`)。⛔ **不要用它抓 Reddit**:该端点的 `.json` 直连与 Firecrawl 渲染两条路都被拦(见该 skill 顶部红字)。
|