webshaker 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e700f078decefd7d40b3469a5ae290c0746481a8d13f214c4020805b10660f20
-  data.tar.gz: 00cbe24e7355ba81e2d54dece6a1e2272e6aae8db52f929eae13053a1d5082b2
+  metadata.gz: ec1a0256ab5ff877a2b20e7ca35a38dda6d0bfad0381e1af9babdacbe164a6f9
+  data.tar.gz: 580cfbdf80f08e8da800867d10e1811c171aebdc290b9360831cd46a071ffd57
 SHA512:
-  metadata.gz: 91082a99565064c4ad876d1b175024bfc5dc2a6291d16f2f22eb9613903eba3e34da16e196d0823e10547eaec9b2b9948c57835894e50d4adeaf18daa7adf2a5
-  data.tar.gz: 9d477bbd2420dd3a3b8c3938e26edd0a2c4e0f911743b41094d6991fae54931e2c768469cb3cb7e38b34ec84ef5fd04e54b02c40f522635c9aa6ffdd0710b51c
+  metadata.gz: 93ef3de4bf2e26f92850408083acd86c65e3033aaa51276ac280bf86e1bbf7ce5d5644f38e663647bea76ca10c9ea07a170a309ac1ba3dd1fe5ec2245c2107ab
+  data.tar.gz: a47e4a3105206654cdfcdc9c28f3d10710ed2d14f346daf92e43639eefe9c9523eb9e7a69984af2ca0f79e175881a8303a9e4252ea9963d1149ce1d75c210714

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    webshaker (0.0.5)
+    webshaker (0.0.6)
       nokogiri
       ruby-openai
       selenium-webdriver (~> 4.0)

data/lib/webshaker/ai.rb

@@ -1,15 +1,18 @@
 module Webshaker
   class Ai
-    attr_reader :html_content
+    attr_reader :html_content, :status_update
 
-    def initialize(html_content)
+    def initialize(html_content, status_update: ->(status) {})
       @html_content = html_content
+      @status_update = status_update
     end
 
-    def analyze(with_prompt:, respond_with: :text, temperature: 0.8, full_response: false)
+    def analyze(with_prompt:, model: Webshaker.config.model, respond_with: :text, temperature: 0.8, full_response: false)
+      status_update.call(:ai_start)
+
       response = ai_client.chat(
         parameters: {
-          model: Webshaker.config.model,
+          model:,
           messages: messages(with_prompt).concat((respond_with.to_sym == :json) ? [{role: "user", content: "respond with json"}] : []),
           temperature:
         }.merge(
@@ -18,10 +21,15 @@ module Webshaker
       )
 
       # Return full response from the ai client if the respond_with is set to :full
-      return response if full_response
+      if full_response
+        status_update.call(:ai_done)
+        return response
+      end
 
       response = response["choices"][0]["message"]["content"]
       response = JSON.parse(response) if respond_with.to_sym === :json
+
+      status_update.call(:ai_done)
       response
     end
 

data/lib/webshaker/scraper.rb

@@ -4,11 +4,15 @@ require "nokogiri"
 
 module Webshaker
   class Scraper
-    attr_reader :url, :driver, :options
+    attr_reader :url, :driver, :options, :status_update
 
-    def initialize(url, options = {})
+    def initialize(url, options = {}, status_update: ->(status) {})
       @url = url
       @options = options
+      @status_update = status_update
+
+      status_update.call(:scrape_init)
+
       @driver = Selenium::WebDriver.for(
         :chrome,
         options: Selenium::WebDriver::Chrome::Options.new.tap(&method(:configure))
@@ -16,14 +20,19 @@ module Webshaker
     end
 
     def scrape
+      status_update.call(:scrape_start)
+
       driver.navigate.to url
 
       do_wait
 
       screenshot = driver.screenshot_as :base64
-      html_content = driver.page_source
+      html_content = clean_up(driver.page_source)
       driver.quit
-      ScrapeResult.new(screenshot, clean_up(html_content))
+
+      status_update.call(:scrape_done)
+
+      ScrapeResult.new(screenshot, html_content)
     end
 
     def self.scrape(url, options = {})
@@ -44,6 +53,8 @@ module Webshaker
 
       wait = Selenium::WebDriver::Wait.new(timeout: 10) # Waits a maximum of 10 seconds
 
+      status_update.call(:scrape_waiting)
+
       wait.until do
         if wait_until.is_a?(Proc)
           wait_until.call(driver)
@@ -56,6 +67,7 @@ module Webshaker
 
     def clean_up(html_content)
       # Parse the HTML content
+      status_update.call(:scrape_cleaning)
 
       doc = Nokogiri::HTML5(html_content)
 

data/lib/webshaker/shaker.rb

@@ -2,11 +2,12 @@ require "openai"
 
 module Webshaker
   class Shaker
-    attr_reader :url, :scrape_options
+    attr_reader :url, :scrape_options, :status_update
 
-    def initialize(url, scrape_options = {})
+    def initialize(url, scrape_options = {}, status_update: ->(status) {})
       @url = url
       @scrape_options = scrape_options
+      @status_update = status_update
     end
 
     def shake(with_prompt:, respond_with: :text, temperature: 0.8)
@@ -16,11 +17,11 @@ module Webshaker
     private
 
     def html
-      @html ||= Webshaker::Scraper.new(url, scrape_options).scrape.html
+      @html ||= Webshaker::Scraper.new(url, scrape_options, status_update:).scrape.html
     end
 
     def ai
-      @ai ||= Webshaker::Ai.new(html)
+      @ai ||= Webshaker::Ai.new(html, status_update:)
     end
   end
 end

data/lib/webshaker/version.rb

@@ -1,3 +1,3 @@
 module Webshaker
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: webshaker
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Martin Mochetti
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-08-16 00:00:00.000000000 Z
+date: 2024-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: selenium-webdriver