openai-scraper 1.2 → 1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3640e18019f73e33ea331aac0b8834e0de4cc49311b43b0d578b5d1428d55ce9
-  data.tar.gz: 8a6f66459cd9597285c638ecc11751e4bd20722f28dddd1da864e65e81e9b954
+  metadata.gz: bd10d79aa9bad5ef74d0f5e11869c367a5bf46e5c2031728efa74f0fc74f418d
+  data.tar.gz: 6a428a3c5135ed7effe14ef69fdf25631b39ae923593cf10caa5d3770b8d8db5
 SHA512:
-  metadata.gz: dc8c94459de87a753402bfd8ce76877116d1565b328519c5a8a64701bb0b89bf6f5b6fa90b33f9aae79e4ef165b85451b0c12d9b1b8550f875078e729d7f735e
-  data.tar.gz: 4a20f94caff31aae40299caaa4e06859467719823ea233a8dda77d8226e64526cd25b9a513886519d98c13fcbc5cf4eae1dc6b6d878ec87e3bc71c7bf138fe96
+  metadata.gz: d30640cfd1bb040c6881274ce47ab0727855a285823ba0116349a8f7f367c90fc197412b712a6013fe36abff87ee3dccbf21071e728d2e26128e62df3249f622
+  data.tar.gz: e5cd6fc52e3bf5d0a59cee01e9d99974e97edc95e08be82f966718839ebdf25901263e58c9c10e1b136a2355a785e061c1f16d3c6b748edf5114d412ff2e06da

data/lib/openai-scraper.rb

@@ -22,6 +22,7 @@ end
 module BlackStack
     module OpenAIScraper
         @@openai_apikey = nil
+        @@model = nil
         @@client = nil
         @@browser = nil
         @@history = []
@@ -37,17 +38,19 @@ module BlackStack
 
         def self.set(h)
             @@openai_apikey = h[:openai_apikey] if h[:openai_apikey]
+            @@model = h[:model] if h[:model]
         end
 
         def self.init
             @@client = OpenAI::Client.new(access_token: @@openai_apikey)
-            @@browser = Selenium::WebDriver.for :chrome
+            ## setup a mechanize client
+            @@browser = Mechanize.new
             # load history array from the file ./history.json, only if the file exists
             @@history = JSON.parse(File.read('./history.json')) if File.exist?('./history.json')
         end
 
         def self.finalize
-            @@browser.quit
+            #@@browser.quit
             # overrite the file ./history.json with the current history array
             File.write('./history.json', @@history.to_json)
         end
@@ -92,7 +95,7 @@ List of Commands:\n
 
             response = @@client.chat(
                 parameters: {
-                    model: "gpt-3.5-turbo", # Required.
+                    model: @@model, # Required.
                     #max_tokens: 6000,
                     temperature: 0.5,
                     messages: [
@@ -138,15 +141,48 @@ List of Commands:\n
                     ],
 =end
                 })
+#binding.pry
             raise response.dig("error", "message") if response.dig("error", "message")
             return response.dig("choices", 0, "message", "content")         
         end
 
+        # get all links of the webiste, at a certainly deep level
+        def self.get_links(url, deep = 1, l=nil)
+            l = BlackStack::DummyLogger.new(nil) if l.nil?
+            ret = []
+            return ret if deep<=0
+            # go to the URL
+            l.logs "level #{deep} - get_links: #{url}... "
+                page = @@browser.get(url)
+                # get all links
+                links = page.search('a')
+                # add each link to the array
+                links.each do |link|
+                    txt = link.text.to_s.strip
+                    ret << { 'href' => link['href'], 'text' => txt }
+                end
+                # make ret an array of unique hashes
+                ret.uniq!
+                # remove links that are not belonging the same domain
+                ret.reject! { |h| h['href'] !~ /#{URI.parse(url).host}/ }
+                # remove links that are not belonging the protocols http or https
+                ret.reject! { |h| h['href'] !~ /^https?:\/\// }
+            l.logf 'done'.green
+
+            # get the links inside each link
+            ret.each do |link|
+                ret += get_links(link['href'], deep-1, l)
+            end
+
+            # return
+            ret
+        end
+
         # download the web page, and extract all links.
         #
         def self.wl(url)
             # visit the url
-            @@browser.navigate.to url
+            page = @@browser.get(url)
             
             # wait up to 30 seconds for the page to load
             #wait = Selenium::WebDriver::Wait.new(:timeout => 30)
@@ -157,7 +193,7 @@ List of Commands:\n
             #wait.until { @@browser.execute_script("return jQuery.active") == 0 }
 
             # get all the links
-            links = @@browser.find_elements(:tag_name, 'a')
+            links = page.search('a')
 
             # add the links to a json structure
             h = []
@@ -176,7 +212,7 @@ List of Commands:\n
         #
         def self.wt(url)
             # visit the url
-            @@browser.navigate.to url
+            page = @@browser.get(url)
             
             # wait up to 30 seconds for the page to load
             #wait = Selenium::WebDriver::Wait.new(:timeout => 30)
@@ -186,8 +222,8 @@ List of Commands:\n
             #wait = Selenium::WebDriver::Wait.new(:timeout => 30)
             #wait.until { @@browser.execute_script("return jQuery.active") == 0 }
 
-            # return the text of the webpage
-            @@browser.find_element(:tag_name, 'body').text
+            # return the text of the body
+            page.search('body').text
         end # def wt
 
         # show the promt and wait for the user input

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: openai-scraper
 version: !ruby/object:Gem::Version
-  version: '1.2'
+  version: '1.3'
 platform: ruby
 authors:
 - Leandro Daniel Sardi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-16 00:00:00.000000000 Z
+date: 2023-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -31,65 +31,65 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 1.13.10
 - !ruby/object:Gem::Dependency
-  name: simple_cloud_logging
+  name: mechanize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: 2.8.5
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: 2.8.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: 2.8.5
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.2
+        version: 2.8.5
 - !ruby/object:Gem::Dependency
-  name: colorize
+  name: simple_cloud_logging
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: 1.2.2
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: 1.2.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: 1.2.2
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.8.1
+        version: 1.2.2
 - !ruby/object:Gem::Dependency
-  name: selenium-webdriver
+  name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.10.0
+        version: 0.8.1
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.10.0
+        version: 0.8.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 4.10.0
+        version: 0.8.1
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.10.0
+        version: 0.8.1
 - !ruby/object:Gem::Dependency
   name: ruby-openai
   requirement: !ruby/object:Gem::Requirement