webshaker 0.0.4 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2c5228755a1344e15fa23300e628cddc67815e3b231917cf9721317997d02427
-  data.tar.gz: 159a730316441410231a83291899d3f69d31f6f84a5a0249e5a0302780e0420d
+  metadata.gz: ec1a0256ab5ff877a2b20e7ca35a38dda6d0bfad0381e1af9babdacbe164a6f9
+  data.tar.gz: 580cfbdf80f08e8da800867d10e1811c171aebdc290b9360831cd46a071ffd57
 SHA512:
-  metadata.gz: 342c4bfee8098c756cff385d874e99d7489e2cec1e91f4777a44dfda53febf2ad6fc68f5e2dcbc58ef70519d8fcd9b2180c00ea9aa1f50de72e2fdc6b5bd0e5a
-  data.tar.gz: 5c168c02a39d3d8d1dd798d938423153be85f793b4973a2cc3a26cd2572dcb6f195f41910c3d443cbe1c45b66026d11ff46d25ea299666c16d6e8cb6b28dc925
+  metadata.gz: 93ef3de4bf2e26f92850408083acd86c65e3033aaa51276ac280bf86e1bbf7ce5d5644f38e663647bea76ca10c9ea07a170a309ac1ba3dd1fe5ec2245c2107ab
+  data.tar.gz: a47e4a3105206654cdfcdc9c28f3d10710ed2d14f346daf92e43639eefe9c9523eb9e7a69984af2ca0f79e175881a8303a9e4252ea9963d1149ce1d75c210714

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    webshaker (0.0.4)
+    webshaker (0.0.6)
       nokogiri
       ruby-openai
       selenium-webdriver (~> 4.0)

data/lib/webshaker/ai.rb

@@ -1,27 +1,35 @@
 module Webshaker
   class Ai
-    attr_reader :html_content
+    attr_reader :html_content, :status_update
 
-    def initialize(html_content)
+    def initialize(html_content, status_update: ->(status) {})
       @html_content = html_content
+      @status_update = status_update
     end
 
-    def analyze(with_prompt:, respond_with: :text, temperature: 0.8, full_response: false)
+    def analyze(with_prompt:, model: Webshaker.config.model, respond_with: :text, temperature: 0.8, full_response: false)
+      status_update.call(:ai_start)
+
       response = ai_client.chat(
         parameters: {
-          model: Webshaker.config.model,
+          model:,
           messages: messages(with_prompt).concat((respond_with.to_sym == :json) ? [{role: "user", content: "respond with json"}] : []),
           temperature:
         }.merge(
-          (respond_with == :json) ? {response_format: {type: "json_object"}} : {}
+          (respond_with.to_sym == :json) ? {response_format: {type: "json_object"}} : {}
         )
       )
 
       # Return full response from the ai client if the respond_with is set to :full
-      return response if full_response
+      if full_response
+        status_update.call(:ai_done)
+        return response
+      end
 
       response = response["choices"][0]["message"]["content"]
-      response = JSON.parse(response) if respond_with === :json
+      response = JSON.parse(response) if respond_with.to_sym === :json
+
+      status_update.call(:ai_done)
       response
     end
 

data/lib/webshaker/scraper.rb

@@ -4,11 +4,15 @@ require "nokogiri"
 
 module Webshaker
   class Scraper
-    attr_reader :url, :driver, :options
+    attr_reader :url, :driver, :options, :status_update
 
-    def initialize(url, options = {})
+    def initialize(url, options = {}, status_update: ->(status) {})
       @url = url
       @options = options
+      @status_update = status_update
+
+      status_update.call(:scrape_init)
+
       @driver = Selenium::WebDriver.for(
         :chrome,
         options: Selenium::WebDriver::Chrome::Options.new.tap(&method(:configure))
@@ -16,14 +20,19 @@ module Webshaker
     end
 
     def scrape
+      status_update.call(:scrape_start)
+
       driver.navigate.to url
 
       do_wait
 
       screenshot = driver.screenshot_as :base64
-      html_content = driver.page_source
+      html_content = clean_up(driver.page_source)
       driver.quit
-      ScrapeResult.new(screenshot, clean_up(html_content))
+
+      status_update.call(:scrape_done)
+
+      ScrapeResult.new(screenshot, html_content)
     end
 
     def self.scrape(url, options = {})
@@ -44,6 +53,8 @@ module Webshaker
 
       wait = Selenium::WebDriver::Wait.new(timeout: 10) # Waits a maximum of 10 seconds
 
+      status_update.call(:scrape_waiting)
+
       wait.until do
         if wait_until.is_a?(Proc)
           wait_until.call(driver)
@@ -56,6 +67,7 @@ module Webshaker
 
     def clean_up(html_content)
       # Parse the HTML content
+      status_update.call(:scrape_cleaning)
 
       doc = Nokogiri::HTML5(html_content)
 

data/lib/webshaker/shaker.rb

@@ -2,11 +2,12 @@ require "openai"
 
 module Webshaker
   class Shaker
-    attr_reader :url, :scrape_options
+    attr_reader :url, :scrape_options, :status_update
 
-    def initialize(url, scrape_options = {})
+    def initialize(url, scrape_options = {}, status_update: ->(status) {})
       @url = url
       @scrape_options = scrape_options
+      @status_update = status_update
     end
 
     def shake(with_prompt:, respond_with: :text, temperature: 0.8)
@@ -16,11 +17,11 @@ module Webshaker
     private
 
     def html
-      @html ||= Webshaker::Scraper.new(url, scrape_options).scrape.html
+      @html ||= Webshaker::Scraper.new(url, scrape_options, status_update:).scrape.html
     end
 
     def ai
-      @ai ||= Webshaker::Ai.new(html)
+      @ai ||= Webshaker::Ai.new(html, status_update:)
     end
   end
 end

data/lib/webshaker/version.rb

@@ -1,3 +1,3 @@
 module Webshaker
-  VERSION = "0.0.4"
+  VERSION = "0.0.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webshaker
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.6
 platform: ruby
 authors:
 - Martin Mochetti
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-16 00:00:00.000000000 Z
+date: 2024-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver