getter_cyndi5 0.0.4 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2614abec8711942eeef9deb4e322fa112719f29e936371de5b7ac3f03f767833
-  data.tar.gz: 53d049b3b8115bb0bfc859482bd1e27322b95cc3d93ec64927992b1b608e0151
+  metadata.gz: b4dcc442ee6a82b1bfbe21cd5b644b169000e6793f83dc30be7c24801e1ee0da
+  data.tar.gz: 326ec69f36cbb6a124029f824df09198328025e96b67203e3b758b8705d36801
 SHA512:
-  metadata.gz: 177291453efc5d02b9259869d785b327e397d2c3b71d4ea97edf4d1c8b060b627614b15948bb6598398a687a315734a9a64a02e6b1bf06e4f3b5902087a1c140
-  data.tar.gz: 2c7dd3e3ff2e91fddeadf79033976385c7a15e12334a553bc20a0f0232b529f4d211e37e804119057d4feac01f045894df1a64d4212ea639fcebc35895de1d7e
+  metadata.gz: ba57a3dd888a5c578ec699ec3fbccb613ddcc2a865d533e954aad6f0f19d8b2eb84c349f3a24eca359d0f3f608e2eaaa7c4a0bc01c601ab9871972805d8eb4dd
+  data.tar.gz: c709a5a1011055388456e7b6c2232364b0fe521fea2fe21943888b2de823c0c4eef4c47f5709231722fac662f04f65fc7364ad6db922d210f62ab3ff68f6b538

data/bin/getter_cyndi5

@@ -5,6 +5,17 @@ base_url = ARGV[0] || 'https://thehappyco.com'
 products_page_path = ARGV[1] || '/kelly/products'
 item_row_selector = ARGV[2] || '.item-row'
 item_anchor_selector = ARGV[3] || 'div > div.product-desc.text-center > div.product-title > h3 > a'
-mode = (ARGV[4] || '2').to_i
-filename = ARGV[5] || './tmp/document1.html'
-GetterCyndi5.go(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+item_price_selector = ARGV[4] || 'div > div.product-desc.text-center > div.product-price'
+price_a_text = ARGV[5] || 'One Time Purchase'
+price_b_text = ARGV[6] || 'SmartShip'
+mode = (ARGV[7] || '2').to_i
+filename = ARGV[8] || './tmp/test_document.html'
+GetterCyndi5.go(base_url,
+                products_page_path,
+                item_row_selector,
+                item_anchor_selector,
+                item_price_selector,
+                price_a_text,
+                price_b_text,
+                mode,
+                filename)

data/lib/getter_cyndi5.rb

@@ -3,26 +3,42 @@ class GetterCyndi5
   # Getter
   #
   # Example:
-  #   >> GetterCyndi5.go(base_url = 'https://thehappyco.com', products_page_path = '/kelly/products', item_row_selector = '.item-row', item_anchor_selector = 'div > div.product-desc.text-center > div.product-title > h3 > a', mode = 2, './tmp/document1.html')
+  #   >> GetterCyndi5.go(
+  #         base_url = 'https://thehappyco.com',
+  #         products_page_path = '/kelly/products',
+  #         item_row_selector = '.item-row',
+  #         item_anchor_selector = 'div > div.product-desc.text-center > div.product-title > h3 > a',
+  #         item_price_selector = 'div > div.product-desc.text-center > div.product-price',
+  #         price_a_text: 'One Time Purchase',
+  #         price_b_text: 'SmartShip',
+  #         mode = 2,
+  #         filename = './tmp/test_document.html'
+  #     )
   # Arguments:
   #   base_url: (String)
   #   products_page_path: (String) 
   #   item_row_selector: (String) 
   #   item_anchor_selector: (String)
-  #   mode: (Integer) 0 = retrieve and parse without saving HTML document to file, 1 = retrieve and parse saving HTML document to file, 2 = load and parse HTML document from file
+  #   item_price_selector: (String)
+  #   price_a_text: (String)
+  #   price_b_text: (String)
+  #   mode: (Integer)
+  #     0 = retrieve and parse without saving HTML document to file,
+  #     1 = retrieve and parse saving HTML document to file,
+  #     2 = load and parse HTML document from file
   #   filename: (String)
 
-  def self.go(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
-    all_products = products(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+  def self.go(base_url, products_page_path, item_row_selector, item_anchor_selector, item_price_selector, price_a_text, price_b_text, mode, filename)
+    all_products = products(base_url, products_page_path, item_row_selector, item_anchor_selector, item_price_selector, price_a_text, price_b_text, mode, filename)
     all_products.each do |product|
-      puts "#{product.name} :-: #{product.url}"
+      puts "#{product.item_code}, #{product.name}, #{product.url}, #{product.price_a}, #{product.price_b}"
     end
   end
 
-  def self.products(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+  def self.products(base_url, products_page_path, item_row_selector, item_anchor_selector, item_price_selector, price_a_text, price_b_text, mode, filename)
     retriever = Retriever.new(base_url, products_page_path, item_row_selector, mode, filename)
     retriever.retrieve
-    parser = Parser.new(retriever.document, base_url, item_row_selector, item_anchor_selector)
+    parser = Parser.new(retriever.document, base_url, item_row_selector, item_anchor_selector, item_price_selector, price_a_text, price_b_text)
     products = parser.parse
   end
 end

data/lib/getter_cyndi5/parser.rb

@@ -1,11 +1,14 @@
 require 'getter_cyndi5/product'
 class GetterCyndi5::Parser
   attr_reader :products
-  def initialize(document, base_url, item_row_selector, item_anchor_selector)
+  def initialize(document, base_url, item_row_selector, item_anchor_selector, item_price_selector, price_a_text, price_b_text)
     @document = document
     @base_url = base_url
     @item_row_selector = item_row_selector
     @item_anchor_selector = item_anchor_selector
+    @item_price_selector = item_price_selector
+    @price_a_text = price_a_text
+    @price_b_text = price_b_text
     @products = []
   end
 
@@ -15,7 +18,13 @@ class GetterCyndi5::Parser
       product_element = item_row.css(@item_anchor_selector)[0]
       product_name = product_element.text
       product_url = "#{@base_url}#{product_element.attributes['href']}"
-      product = GetterCyndi5::Product.new(product_name, product_url)
+      product_item_code = item_row.attributes['data-itemcode'].value
+      product_prices = item_row.css(@item_price_selector)
+      price_a_element = product_prices.find { |price| price.children[1].text == @price_a_text }
+      price_a = price_a_element.nil? ? 0.0 : price_a_element.children[0].text.gsub(/[^\d\.]/, '').to_f
+      price_b_element = product_prices.find { |price| price.children[1].text == @price_b_text }
+      price_b = price_b_element.nil? ? 0.0 : price_b_element.children[0].text.gsub(/[^\d\.]/, '').to_f
+      product = GetterCyndi5::Product.new(product_name, product_url, product_item_code, price_a, price_b)
       products.append(product)
     end
     products

data/lib/getter_cyndi5/product.rb

@@ -1,8 +1,9 @@
 class GetterCyndi5::Product
-  attr_accessor :name, :url, :price_a, :price_b, :price_c
-  def initialize(name, url, price_a=0.0, price_b=0.0, price_c=0.0)
+  attr_accessor :name, :url, :item_code, :price_a, :price_b, :price_c
+  def initialize(name, url, item_code, price_a=0.0, price_b=0.0, price_c=0.0)
     @name = name
     @url = url
+    @item_code = item_code
     @price_a = price_a
     @price_b = price_b
     @price_c = price_c

data/lib/getter_cyndi5/retriever.rb

@@ -15,15 +15,17 @@ class GetterCyndi5::Retriever
 
   def retrieve()
     if @mode == 0 || @mode == 1
-      browser = Watir::Browser.new
+      browser = Watir::Browser.new :chrome, args: %w[--headless --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=9222]
       browser.goto(@products_page_url)
       item_row_elements = browser.elements(css: @item_row_selector)
     end
     if @mode == 1
       File.write(@filename, browser.html)
+      browser.close
     end
     if @mode == 0
       @document = Nokogiri::HTML(browser.html)
+      browser.close
     end
     if @mode == 1 || @mode == 2
       @document = File.open(@filename) { |f| Nokogiri::HTML(f) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: getter_cyndi5
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 1.0.0
 platform: ruby
 authors:
 - Cyndi Cavanaugh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-12 00:00:00.000000000 Z
+date: 2021-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty