getter_cyndi5 0.0.6 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cd951e59fcde272f1b72e45d7a7f935a966e193bdf9b91bd219864cdf00db22
-  data.tar.gz: 60023272d347c7d374ec501559d547c5d5fcdfd0998bc3936b23935633a5b86a
+  metadata.gz: 8897522fa8be7a71ee0a93636ad6c8250dc8ceb4b774e4682b6add74440e3561
+  data.tar.gz: 2f457d708b7c100b56a354101efe367299e6bdb97dbaa6c9f9b019b9f68b9318
 SHA512:
-  metadata.gz: 24f20cbe4de2956352b27c1603d726c6797b2022db27b9afd0a531058645d5cd481dd283bb57e823d0faa223d67f02bb58e7e54af2b333752dbaafb18bb0d298
-  data.tar.gz: d175ff2ce3b5bb583dd574455c73dd63d098780c6f5d304ee9d84744be9901ea4b7025a49450973aa3c979671c3f7864ac958c3c062747b1540b5e1a59df2a2c
+  metadata.gz: 5d627257e6e981258d321032a5959c0ddee38193606ae0d1f1538a2bebbc19b31c1570b477131ed29075b0c71cb1de9c1811eebf431c73d118f135689535383d
+  data.tar.gz: 3280ea73a619aea5be935a3fd7f2e622deecf7786e1d3967f3c22cc38e38678f804e21cc909e7e132791fe9ab5a69af36274f6b9010301113dcf1ae8525c0f23

data/bin/getter_cyndi5

@@ -1,10 +1,14 @@
 #!/usr/bin/env ruby
 
 require 'getter_cyndi5'
-base_url = ARGV[0] || 'https://thehappyco.com'
-products_page_path = ARGV[1] || '/kelly/products'
-item_row_selector = ARGV[2] || '.item-row'
-item_anchor_selector = ARGV[3] || 'div > div.product-desc.text-center > div.product-title > h3 > a'
-mode = (ARGV[4] || '2').to_i
-filename = ARGV[5] || './tmp/document1.html'
-GetterCyndi5.go(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+
+options = {
+  base_url: ARGV[0] || 'https://thehappyco.com',
+  products_page_path: ARGV[1] || '/kelly/products',
+  item_row_selector: ARGV[2] || '.item-row',
+  item_anchor_selector: ARGV[3] || 'div > div.product-desc.text-center > div.product-title > h3 > a',
+  item_price_selector: ARGV[4] || 'div > div.product-desc.text-center > div.product-price',
+  mode: (ARGV[7] || '2').to_i,
+  filename: ARGV[8] || './tmp/test_document.html'
+}
+GetterCyndi5.go(**options)

data/lib/getter_cyndi5.rb

@@ -3,26 +3,43 @@ class GetterCyndi5
   # Getter
   #
   # Example:
-  #   >> GetterCyndi5.go(base_url = 'https://thehappyco.com', products_page_path = '/kelly/products', item_row_selector = '.item-row', item_anchor_selector = 'div > div.product-desc.text-center > div.product-title > h3 > a', mode = 2, './tmp/document1.html')
+  #   >> GetterCyndi5.go(
+  #         base_url = 'https://thehappyco.com',
+  #         products_page_path = '/kelly/products',
+  #         item_row_selector = '.item-row',
+  #         item_anchor_selector = 'div > div.product-desc.text-center > div.product-title > h3 > a',
+  #         item_price_selector = 'div > div.product-desc.text-center > div.product-price',
+  #         price_a_text: 'One Time Purchase',
+  #         price_b_text: 'SmartShip',
+  #         mode = 2,
+  #         filename = './tmp/test_document.html'
+  #     )
   # Arguments:
   #   base_url: (String)
   #   products_page_path: (String) 
   #   item_row_selector: (String) 
   #   item_anchor_selector: (String)
-  #   mode: (Integer) 0 = retrieve and parse without saving HTML document to file, 1 = retrieve and parse saving HTML document to file, 2 = load and parse HTML document from file
+  #   item_price_selector: (String)
+  #   price_a_text: (String)
+  #   price_b_text: (String)
+  #   mode: (Integer)
+  #     0 = retrieve and parse without saving HTML document to file,
+  #     1 = retrieve and parse saving HTML document to file,
+  #     2 = load and parse HTML document from file
   #   filename: (String)
 
-  def self.go(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
-    all_products = products(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+  def self.go(options = {})
+    all_products = products(**options)
     all_products.each do |product|
-      puts "#{product.name} :-: #{product.url}"
+      puts "#{product.inspect}"
     end
   end
 
-  def self.products(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
-    retriever = Retriever.new(base_url, products_page_path, item_row_selector, mode, filename)
+  def self.products(options = {})
+    retriever = Retriever.new(**options)
     retriever.retrieve
-    parser = Parser.new(retriever.document, base_url, item_row_selector, item_anchor_selector)
+
+    parser = Parser.new(document: retriever.document, **options)
     products = parser.parse
   end
 end

data/lib/getter_cyndi5/parser.rb

@@ -1,21 +1,34 @@
 require 'getter_cyndi5/product'
 class GetterCyndi5::Parser
+  DATA_PREFIX = 'data-'.freeze
   attr_reader :products
-  def initialize(document, base_url, item_row_selector, item_anchor_selector)
-    @document = document
-    @base_url = base_url
-    @item_row_selector = item_row_selector
-    @item_anchor_selector = item_anchor_selector
+  def initialize(options = {})
+    @options = options
     @products = []
   end
 
-  def parse     
-    item_rows = @document.css(@item_row_selector)
+  def parse
+    document = @options.fetch(:document)
+    base_url = @options.fetch(:base_url)
+    item_row_selector = @options.fetch(:item_row_selector)
+    item_anchor_selector = @options.fetch(:item_anchor_selector)
+    item_price_selector = @options.fetch(:item_price_selector)
+    item_rows = document.css(item_row_selector)
     item_rows.each do |item_row|
-      product_element = item_row.css(@item_anchor_selector)[0]
-      product_name = product_element.text
-      product_url = "#{@base_url}#{product_element.attributes['href']}"
-      product = GetterCyndi5::Product.new(product_name, product_url)
+      item_data = item_row.attributes.select { |k, v| k.start_with? DATA_PREFIX }.map { |k, v| [k.delete_prefix(DATA_PREFIX), v.value]}.to_h
+      product_element = item_row.css(item_anchor_selector)[0]
+      price_elements = item_row.css(item_price_selector)
+      prices = {}
+      price_elements.each do |price_element|
+        prices[price_element.children[1].text] = price_element.children[0].text.gsub(/[^\d\.]/, '').to_f
+      end
+      attributes = {
+        name: product_element.text,
+        url: "#{base_url}#{product_element.attributes['href']}",
+        item_data: item_data,
+        prices: prices
+      }
+      product = GetterCyndi5::Product.new(attributes)
       products.append(product)
     end
     products

data/lib/getter_cyndi5/product.rb

@@ -1,10 +1,6 @@
 class GetterCyndi5::Product
-  attr_accessor :name, :url, :price_a, :price_b, :price_c
-  def initialize(name, url, price_a=0.0, price_b=0.0, price_c=0.0)
-    @name = name
-    @url = url
-    @price_a = price_a
-    @price_b = price_b
-    @price_c = price_c
+  attr_accessor :attributes
+  def initialize(attributes = {})
+    @attributes = attributes
   end
 end

data/lib/getter_cyndi5/retriever.rb

@@ -4,34 +4,30 @@ require 'watir'
 require 'webdrivers'
 
 class GetterCyndi5::Retriever
-  def initialize(base_url, products_page_path, item_row_selector, mode, filename)
-    @base_url = base_url
-    @products_page_path = products_page_path
-    @products_page_url = "#{@base_url}#{@products_page_path}"
-    @item_row_selector = item_row_selector
-    @mode = mode
-    @filename = filename
+  def initialize(options = {})
+    @options = options
   end
 
+  attr_reader :document
+  attr_reader :item_row_elements
   def retrieve()
-    if @mode == 0 || @mode == 1
-      browser = Watir::Browser.new
-      browser.goto(@products_page_url)
-      item_row_elements = browser.elements(css: @item_row_selector)
-      browser.close
+    products_page_url = "#{@options.fetch(:base_url)}#{@options.fetch(:products_page_path)}"
+    mode = @options.fetch(:mode)
+    if mode == 0 || mode == 1
+      browser = Watir::Browser.new :chrome, args: %w[--headless --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=9222]
+      browser.goto(products_page_url)
+      @item_row_elements = browser.elements(css: @options.fetch(:item_row_selector))
     end
-    if @mode == 1
-      File.write(@filename, browser.html)
+    if mode == 1
+      File.write(@options.fetch(:filename), browser.html)
+      browser.close
     end
-    if @mode == 0
+    if mode == 0
       @document = Nokogiri::HTML(browser.html)
+      browser.close
     end
-    if @mode == 1 || @mode == 2
-      @document = File.open(@filename) { |f| Nokogiri::HTML(f) }
+    if mode == 1 || mode == 2
+      @document = File.open(@options.fetch(:filename)) { |f| Nokogiri::HTML(f) }
     end
   end
-
-  def document
-    @document
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: getter_cyndi5
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 2.0.1
 platform: ruby
 authors:
 - Cyndi Cavanaugh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-12 00:00:00.000000000 Z
+date: 2021-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty