@geoly-ai/social-hub-cli 0.3.46 → 0.3.48

package/skills/reddit-voc-volume/services/firecrawl/examples/o1_web_crawler/o1_web_crawler.py

@@ -1,152 +1,152 @@
-import os
-from firecrawl import FirecrawlApp
-import json
-from dotenv import load_dotenv
-from openai import OpenAI
-
-# ANSI color codes
-class Colors:
-    CYAN = '\033[96m'
-    YELLOW = '\033[93m'
-    GREEN = '\033[92m'
-    RED = '\033[91m'
-    MAGENTA = '\033[95m'
-    BLUE = '\033[94m'
-    RESET = '\033[0m'
-
-# Load environment variables
-load_dotenv()
-
-# Retrieve API keys from environment variables
-firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY")
-openai_api_key = os.getenv("OPENAI_API_KEY")
-
-# Initialize the FirecrawlApp and OpenAI client
-app = FirecrawlApp(api_key=firecrawl_api_key)
-client = OpenAI(api_key=openai_api_key)
-
-# Find the page that most likely contains the objective
-def find_relevant_page_via_map(objective, url, app, client):
-    try:
-        print(f"{Colors.CYAN}Understood. The objective is: {objective}{Colors.RESET}")
-        print(f"{Colors.CYAN}Initiating search on the website: {url}{Colors.RESET}")
-        
-        map_prompt = f"""
-        The map function generates a list of URLs from a website and it accepts a search parameter. Based on the objective of: {objective}, come up with a 1-2 word search parameter that will help us find the information we need. Only respond with 1-2 words nothing else.
-        """
-
-        print(f"{Colors.YELLOW}Analyzing objective to determine optimal search parameter...{Colors.RESET}")
-        completion = client.chat.completions.create(
-            model="o1-preview",
-            messages=[
-                {
-                    "role": "user",
-                    "content": [
-                        {
-                            "type": "text",
-                            "text": map_prompt
-                        }
-                    ]
-                }
-            ]
-        )
-
-        map_search_parameter = completion.choices[0].message.content
-        print(f"{Colors.GREEN}Optimal search parameter identified: {map_search_parameter}{Colors.RESET}")
-
-        print(f"{Colors.YELLOW}Mapping website using the identified search parameter...{Colors.RESET}")
-        map_website = app.map_url(url, params={"search": map_search_parameter})
-        print(f"{Colors.GREEN}Website mapping completed successfully.{Colors.RESET}")
-        print(f"{Colors.GREEN}Located {len(map_website)} relevant links.{Colors.RESET}")
-        return map_website
-    except Exception as e:
-        print(f"{Colors.RED}Error encountered during relevant page identification: {str(e)}{Colors.RESET}")
-        return None
-    
-# Scrape the top 3 pages and see if the objective is met, if so return in json format else return None
-def find_objective_in_top_pages(map_website, objective, app, client):
-    try:
-        # Get top 3 links from the map result
-        top_links = map_website[:3] if isinstance(map_website, list) else []
-        print(f"{Colors.CYAN}Proceeding to analyze top {len(top_links)} links: {top_links}{Colors.RESET}")
-        
-        for link in top_links:
-            print(f"{Colors.YELLOW}Initiating scrape of page: {link}{Colors.RESET}")
-            # Scrape the page
-            scrape_result = app.scrape_url(link, params={'formats': ['markdown']})
-            print(f"{Colors.GREEN}Page scraping completed successfully.{Colors.RESET}")
-     
-            
-            # Check if objective is met
-            check_prompt = f"""
-            Given the following scraped content and objective, determine if the objective is met.
-            If it is, extract the relevant information in a simple and concise JSON format. Use only the necessary fields and avoid nested structures if possible.
-            If the objective is not met with confidence, respond with 'Objective not met'.
-
-            Objective: {objective}
-            Scraped content: {scrape_result['markdown']}
-
-            Remember:
-            1. Only return JSON if you are confident the objective is fully met.
-            2. Keep the JSON structure as simple and flat as possible.
-            3. Do not include any explanations or markdown formatting in your response.
-            """
-        
-            completion = client.chat.completions.create(
-            model="o1-preview",
-            messages=[
-                {
-                    "role": "user",
-                    "content": [
-                        {
-                            "type": "text",
-                            "text": check_prompt
-                        }
-                    ]
-                }
-                ]
-            )
-            
-            result = completion.choices[0].message.content
-            
-            if result != "Objective not met":
-                print(f"{Colors.GREEN}Objective potentially fulfilled. Relevant information identified.{Colors.RESET}")
-                try:
-                    return json.loads(result)
-                except json.JSONDecodeError:
-                    print(f"{Colors.RED}Error in parsing response. Proceeding to next page...{Colors.RESET}")
-            else:
-                print(f"{Colors.YELLOW}Objective not met on this page. Proceeding to next link...{Colors.RESET}")
-        
-        print(f"{Colors.RED}All available pages analyzed. Objective not fulfilled in examined content.{Colors.RESET}")
-        return None
-    
-    except Exception as e:
-        print(f"{Colors.RED}Error encountered during page analysis: {str(e)}{Colors.RESET}")
-        return None
-
-# Main function to execute the process
-def main():
-    # Get user input
-    url = input(f"{Colors.BLUE}Enter the website to crawl: {Colors.RESET}")
-    objective = input(f"{Colors.BLUE}Enter your objective: {Colors.RESET}")
-    
-    print(f"{Colors.YELLOW}Initiating web crawling process...{Colors.RESET}")
-    # Find the relevant page
-    map_website = find_relevant_page_via_map(objective, url, app, client)
-    
-    if map_website:
-        print(f"{Colors.GREEN}Relevant pages identified. Proceeding with detailed analysis...{Colors.RESET}")
-        # Find objective in top pages
-        result = find_objective_in_top_pages(map_website, objective, app, client)
-        
-        if result:
-            print(f"{Colors.GREEN}Objective successfully fulfilled. Extracted information:{Colors.RESET}")
-            print(f"{Colors.MAGENTA}{json.dumps(result, indent=2)}{Colors.RESET}")
-        else:
-            print(f"{Colors.RED}Unable to fulfill the objective with the available content.{Colors.RESET}")
-    else:
-        print(f"{Colors.RED}No relevant pages identified. Consider refining the search parameters or trying a different website.{Colors.RESET}")
-
-if __name__ == "__main__":
-    main()
+import os
+from firecrawl import FirecrawlApp
+import json
+from dotenv import load_dotenv
+from openai import OpenAI
+
+# ANSI color codes
+class Colors:
+    CYAN = '\033[96m'
+    YELLOW = '\033[93m'
+    GREEN = '\033[92m'
+    RED = '\033[91m'
+    MAGENTA = '\033[95m'
+    BLUE = '\033[94m'
+    RESET = '\033[0m'
+
+# Load environment variables
+load_dotenv()
+
+# Retrieve API keys from environment variables
+firecrawl_api_key = os.getenv("FIRECRAWL_API_KEY")
+openai_api_key = os.getenv("OPENAI_API_KEY")
+
+# Initialize the FirecrawlApp and OpenAI client
+app = FirecrawlApp(api_key=firecrawl_api_key)
+client = OpenAI(api_key=openai_api_key)
+
+# Find the page that most likely contains the objective
+def find_relevant_page_via_map(objective, url, app, client):
+    try:
+        print(f"{Colors.CYAN}Understood. The objective is: {objective}{Colors.RESET}")
+        print(f"{Colors.CYAN}Initiating search on the website: {url}{Colors.RESET}")
+        
+        map_prompt = f"""
+        The map function generates a list of URLs from a website and it accepts a search parameter. Based on the objective of: {objective}, come up with a 1-2 word search parameter that will help us find the information we need. Only respond with 1-2 words nothing else.
+        """
+
+        print(f"{Colors.YELLOW}Analyzing objective to determine optimal search parameter...{Colors.RESET}")
+        completion = client.chat.completions.create(
+            model="o1-preview",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": map_prompt
+                        }
+                    ]
+                }
+            ]
+        )
+
+        map_search_parameter = completion.choices[0].message.content
+        print(f"{Colors.GREEN}Optimal search parameter identified: {map_search_parameter}{Colors.RESET}")
+
+        print(f"{Colors.YELLOW}Mapping website using the identified search parameter...{Colors.RESET}")
+        map_website = app.map_url(url, params={"search": map_search_parameter})
+        print(f"{Colors.GREEN}Website mapping completed successfully.{Colors.RESET}")
+        print(f"{Colors.GREEN}Located {len(map_website)} relevant links.{Colors.RESET}")
+        return map_website
+    except Exception as e:
+        print(f"{Colors.RED}Error encountered during relevant page identification: {str(e)}{Colors.RESET}")
+        return None
+    
+# Scrape the top 3 pages and see if the objective is met, if so return in json format else return None
+def find_objective_in_top_pages(map_website, objective, app, client):
+    try:
+        # Get top 3 links from the map result
+        top_links = map_website[:3] if isinstance(map_website, list) else []
+        print(f"{Colors.CYAN}Proceeding to analyze top {len(top_links)} links: {top_links}{Colors.RESET}")
+        
+        for link in top_links:
+            print(f"{Colors.YELLOW}Initiating scrape of page: {link}{Colors.RESET}")
+            # Scrape the page
+            scrape_result = app.scrape_url(link, params={'formats': ['markdown']})
+            print(f"{Colors.GREEN}Page scraping completed successfully.{Colors.RESET}")
+     
+            
+            # Check if objective is met
+            check_prompt = f"""
+            Given the following scraped content and objective, determine if the objective is met.
+            If it is, extract the relevant information in a simple and concise JSON format. Use only the necessary fields and avoid nested structures if possible.
+            If the objective is not met with confidence, respond with 'Objective not met'.
+
+            Objective: {objective}
+            Scraped content: {scrape_result['markdown']}
+
+            Remember:
+            1. Only return JSON if you are confident the objective is fully met.
+            2. Keep the JSON structure as simple and flat as possible.
+            3. Do not include any explanations or markdown formatting in your response.
+            """
+        
+            completion = client.chat.completions.create(
+            model="o1-preview",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": check_prompt
+                        }
+                    ]
+                }
+                ]
+            )
+            
+            result = completion.choices[0].message.content
+            
+            if result != "Objective not met":
+                print(f"{Colors.GREEN}Objective potentially fulfilled. Relevant information identified.{Colors.RESET}")
+                try:
+                    return json.loads(result)
+                except json.JSONDecodeError:
+                    print(f"{Colors.RED}Error in parsing response. Proceeding to next page...{Colors.RESET}")
+            else:
+                print(f"{Colors.YELLOW}Objective not met on this page. Proceeding to next link...{Colors.RESET}")
+        
+        print(f"{Colors.RED}All available pages analyzed. Objective not fulfilled in examined content.{Colors.RESET}")
+        return None
+    
+    except Exception as e:
+        print(f"{Colors.RED}Error encountered during page analysis: {str(e)}{Colors.RESET}")
+        return None
+
+# Main function to execute the process
+def main():
+    # Get user input
+    url = input(f"{Colors.BLUE}Enter the website to crawl: {Colors.RESET}")
+    objective = input(f"{Colors.BLUE}Enter your objective: {Colors.RESET}")
+    
+    print(f"{Colors.YELLOW}Initiating web crawling process...{Colors.RESET}")
+    # Find the relevant page
+    map_website = find_relevant_page_via_map(objective, url, app, client)
+    
+    if map_website:
+        print(f"{Colors.GREEN}Relevant pages identified. Proceeding with detailed analysis...{Colors.RESET}")
+        # Find objective in top pages
+        result = find_objective_in_top_pages(map_website, objective, app, client)
+        
+        if result:
+            print(f"{Colors.GREEN}Objective successfully fulfilled. Extracted information:{Colors.RESET}")
+            print(f"{Colors.MAGENTA}{json.dumps(result, indent=2)}{Colors.RESET}")
+        else:
+            print(f"{Colors.RED}Unable to fulfill the objective with the available content.{Colors.RESET}")
+    else:
+        print(f"{Colors.RED}No relevant pages identified. Consider refining the search parameters or trying a different website.{Colors.RESET}")
+
+if __name__ == "__main__":
+    main()

package/skills/reddit-voc-volume/tests/test_links_fallback.py

@@ -25,9 +25,9 @@ class LinksFallbackTests(unittest.TestCase):
             code = voc.main()
         return code, json.loads(stdout.getvalue())
 
-    def test_arctic_miss_uses_rendered_page_without_loading_browser_cookies(self) -> None:
+    def test_arctic_miss_uses_rendered_page_without_direct_network_calls(self) -> None:
         with (
-            mock.patch.object(voc.CookieStore, "get", side_effect=AssertionError("browser cookies loaded")) as cookies,
+            mock.patch.object(voc.urllib.request, "urlopen", side_effect=AssertionError("direct network call")) as urlopen,
             mock.patch.object(voc, "arctic_post_by_permalink", return_value=None) as archive,
             mock.patch.object(voc, "fetch_page_scrape", return_value={"markdown": "Public page"}) as page,
             mock.patch.object(voc, "collect_subreddit_metrics", return_value=[]),
@@ -42,11 +42,11 @@ class LinksFallbackTests(unittest.TestCase):
         self.assertEqual(out["errors"], [])
         archive.assert_called_once()
         self.assertEqual(page.call_args.kwargs["provider"], "social-hub")
-        cookies.assert_not_called()
+        urlopen.assert_not_called()
 
     def test_arctic_miss_without_fallback_reports_archive_not_json(self) -> None:
         with (
-            mock.patch.object(voc.CookieStore, "get", side_effect=AssertionError("browser cookies loaded")) as cookies,
+            mock.patch.object(voc.urllib.request, "urlopen", side_effect=AssertionError("direct network call")) as urlopen,
             mock.patch.object(voc, "arctic_post_by_permalink", side_effect=RuntimeError("archive unavailable")),
             mock.patch.object(voc, "collect_subreddit_metrics", return_value=[]),
             mock.patch.object(voc, "hub_writeback_items", return_value={"status": "not_applicable"}),
@@ -58,7 +58,7 @@ class LinksFallbackTests(unittest.TestCase):
         self.assertFalse(out["ok"])
         self.assertTrue(any("archive unavailable" in error for error in out["errors"]))
         self.assertTrue(any("Arctic lookup failed" in note for note in out["notes"]))
-        cookies.assert_not_called()
+        urlopen.assert_not_called()
 
     def test_auto_fallback_tries_firecrawl_after_social_hub_failure(self) -> None:
         def page_result(_url: str, *, provider: str, **_kwargs: object) -> dict:
@@ -67,7 +67,7 @@ class LinksFallbackTests(unittest.TestCase):
             return {"markdown": "Public page"}
 
         with (
-            mock.patch.object(voc.CookieStore, "get", side_effect=AssertionError("browser cookies loaded")) as cookies,
+            mock.patch.object(voc.urllib.request, "urlopen", side_effect=AssertionError("direct network call")) as urlopen,
             mock.patch.object(voc, "arctic_post_by_permalink", return_value=None),
             mock.patch.object(voc, "fetch_page_scrape", side_effect=page_result) as page,
             mock.patch.object(voc, "social_hub_unavailable_hint", return_value="Hub unavailable"),
@@ -80,7 +80,7 @@ class LinksFallbackTests(unittest.TestCase):
         self.assertEqual(code, 0)
         self.assertEqual(out["items"][0]["page_provider"], "firecrawl")
         self.assertEqual([call.kwargs["provider"] for call in page.call_args_list], ["social-hub", "firecrawl"])
-        cookies.assert_not_called()
+        urlopen.assert_not_called()
 
 
 if __name__ == "__main__":