getter_cyndi5 0.0.8 → 2.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75f153b3644ea1beac5c5a3967734d3d03d1a2b8ac5e08740dd2236beffcaa53
-  data.tar.gz: 979d808e9a7eaf40843eef450c0b776058bd306c14a71102d57783fc2f34acf4
+  metadata.gz: c9bfded80458e5ce9dce37a23bdcea8bf1fb2594eb43dfdd82ab5273d5ed8fd3
+  data.tar.gz: 69e337e0f111cd8c603a0f5d5367afb827c5fb9b582a773b84c59112412abba6
 SHA512:
-  metadata.gz: a321509cbdf2cf0bd51f09d0490597e63e84dd5614ac656255e6ce36b84d97fbe462309a5cac3a46f59c9b7fdcd151075c1a28d2ec8ca75af0d2da8cca37b363
-  data.tar.gz: fbb193d9559ee820fbec6cd96e5e5a297daea5b88ab9ae7306e2ca03931edc63687c21d6e4795a3e0717c1c7488f001fb1381841a6cd7e76e0c121fbf2da7ec2
+  metadata.gz: d5880f7c7e73ee8bac8fcec1273bc2957fb941ec3d08c11ef4ba9156a94142e35296f8413856a42c202a61ac6b96e030feee13091d3ae0977f669e5110c3e77d
+  data.tar.gz: afeb7eeab86d2501921596849aab93eb87a3dbbe0cf4fa4a5ef4c015adcfc994ac08b5ab50efa1e8c0efdc89068870474822b7e1b238b67d75baa71c6cea8547

data/bin/getter_cyndi5

@@ -1,10 +1,14 @@
 #!/usr/bin/env ruby
 
 require 'getter_cyndi5'
-base_url = ARGV[0] || 'https://thehappyco.com'
-products_page_path = ARGV[1] || '/kelly/products'
-item_row_selector = ARGV[2] || '.item-row'
-item_anchor_selector = ARGV[3] || 'div > div.product-desc.text-center > div.product-title > h3 > a'
-mode = (ARGV[4] || '2').to_i
-filename = ARGV[5] || './tmp/document1.html'
-GetterCyndi5.go(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+
+options = {
+  base_url: ARGV[0] || 'https://thehappyco.com',
+  products_page_path: ARGV[1] || '/kelly/products',
+  item_row_selector: ARGV[2] || '.item-row',
+  item_anchor_selector: ARGV[3] || 'div > div.product-desc.text-center > div.product-title > h3 > a',
+  item_price_selector: ARGV[4] || 'div > div.product-desc.text-center > div.product-price',
+  mode: (ARGV[7] || '2').to_i,
+  filename: ARGV[8] || './tmp/test_document.html'
+}
+GetterCyndi5.go(**options)

data/lib/getter_cyndi5.rb

@@ -3,29 +3,47 @@ class GetterCyndi5
   # Getter
   #
   # Example:
-  #   >> GetterCyndi5.go(base_url = 'https://thehappyco.com', products_page_path = '/kelly/products', item_row_selector = '.item-row', item_anchor_selector = 'div > div.product-desc.text-center > div.product-title > h3 > a', mode = 2, './tmp/document1.html')
+  #   >> GetterCyndi5.go(
+  #         base_url = 'https://thehappyco.com',
+  #         products_page_path = '/kelly/products',
+  #         item_row_selector = '.item-row',
+  #         item_anchor_selector = 'div > div.product-desc.text-center > div.product-title > h3 > a',
+  #         item_price_selector = 'div > div.product-desc.text-center > div.product-price',
+  #         price_a_text: 'One Time Purchase',
+  #         price_b_text: 'SmartShip',
+  #         mode = 2,
+  #         filename = './tmp/test_document.html'
+  #     )
   # Arguments:
   #   base_url: (String)
   #   products_page_path: (String) 
   #   item_row_selector: (String) 
   #   item_anchor_selector: (String)
-  #   mode: (Integer) 0 = retrieve and parse without saving HTML document to file, 1 = retrieve and parse saving HTML document to file, 2 = load and parse HTML document from file
+  #   item_price_selector: (String)
+  #   price_a_text: (String)
+  #   price_b_text: (String)
+  #   mode: (Integer)
+  #     0 = retrieve and parse without saving HTML document to file,
+  #     1 = retrieve and parse saving HTML document to file,
+  #     2 = load and parse HTML document from file
   #   filename: (String)
 
-  def self.go(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
-    all_products = products(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+  def self.go(options = {})
+    all_products = products(**options)
     all_products.each do |product|
-      puts "#{product.name} :-: #{product.url}"
+      puts "#{product.inspect}"
     end
   end
 
-  def self.products(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
-    retriever = Retriever.new(base_url, products_page_path, item_row_selector, mode, filename)
+  def self.products(options = {})
+    retriever = Retriever.new(**options)
     retriever.retrieve
-    parser = Parser.new(retriever.document, base_url, item_row_selector, item_anchor_selector)
+
+    parser = Parser.new(document: retriever.document, **options)
     products = parser.parse
   end
 end
 
 require 'getter_cyndi5/retriever'
-require 'getter_cyndi5/parser'
+require 'getter_cyndi5/parser'
+require 'getter_cyndi5/importer'

data/lib/getter_cyndi5/importer.rb

@@ -0,0 +1,13 @@
+class Importer
+  def initialize(importer_options = {})
+    @importer_options = importer_options
+  end
+
+  def import(product)
+      { :item_code => product.attributes[:item_data].fetch(@importer_options.fetch(:itemcode_key), ''),
+        :name => product.attributes[:name],
+        :url => product.attributes[:url],
+        :price_a => product.attributes[:prices].fetch(@importer_options.fetch(:price_a_key), 0.0),
+        :price_b => product.attributes[:prices].fetch(@importer_options.fetch(:price_b_key), 0.0) }
+  end
+end

data/lib/getter_cyndi5/parser.rb

@@ -1,23 +1,39 @@
 require 'getter_cyndi5/product'
 class GetterCyndi5::Parser
+  DATA_PREFIX = 'data-'.freeze
   attr_reader :products
-  def initialize(document, base_url, item_row_selector, item_anchor_selector)
-    @document = document
-    @base_url = base_url
-    @item_row_selector = item_row_selector
-    @item_anchor_selector = item_anchor_selector
+  def initialize(options = {})
+    @options = options
     @products = []
   end
 
-  def parse     
-    item_rows = @document.css(@item_row_selector)
+  def parse
+    base_url = @options.fetch(:base_url)
+    item_anchor_selector = @options.fetch(:item_anchor_selector)
+    item_price_selector = @options.fetch(:item_price_selector)
     item_rows.each do |item_row|
-      product_element = item_row.css(@item_anchor_selector)[0]
-      product_name = product_element.text
-      product_url = "#{@base_url}#{product_element.attributes['href']}"
-      product = GetterCyndi5::Product.new(product_name, product_url)
+      item_data = item_row.attributes.select { |k, v| k.start_with? DATA_PREFIX }.map { |k, v| [k.delete_prefix(DATA_PREFIX), v.value]}.to_h
+      product_element = item_row.css(item_anchor_selector)[0]
+      price_elements = item_row.css(item_price_selector)
+      prices = {}
+      price_elements.each do |price_element|
+        prices[price_element.children[1].text] = price_element.children[0].text.gsub(/[^\d\.]/, '').to_f
+      end
+      attributes = {
+        name: product_element.text,
+        url: "#{base_url}#{product_element.attributes['href']}",
+        item_data: item_data,
+        prices: prices
+      }
+      product = GetterCyndi5::Product.new(attributes)
       products.append(product)
     end
     products
   end
+
+  def item_rows
+    document = @options.fetch(:document)
+    item_row_selector = @options.fetch(:item_row_selector)
+    document.css(item_row_selector)
+  end
 end

data/lib/getter_cyndi5/product.rb

@@ -1,10 +1,6 @@
 class GetterCyndi5::Product
-  attr_accessor :name, :url, :price_a, :price_b, :price_c
-  def initialize(name, url, price_a=0.0, price_b=0.0, price_c=0.0)
-    @name = name
-    @url = url
-    @price_a = price_a
-    @price_b = price_b
-    @price_c = price_c
+  attr_accessor :attributes
+  def initialize(attributes = {})
+    @attributes = attributes
   end
 end

data/lib/getter_cyndi5/retriever.rb

@@ -4,35 +4,30 @@ require 'watir'
 require 'webdrivers'
 
 class GetterCyndi5::Retriever
-  def initialize(base_url, products_page_path, item_row_selector, mode, filename)
-    @base_url = base_url
-    @products_page_path = products_page_path
-    @products_page_url = "#{@base_url}#{@products_page_path}"
-    @item_row_selector = item_row_selector
-    @mode = mode
-    @filename = filename
+  def initialize(options = {})
+    @options = options
   end
 
+  attr_reader :document
+  attr_reader :item_row_elements
   def retrieve()
-    if @mode == 0 || @mode == 1
+    products_page_url = "#{@options.fetch(:base_url)}#{@options.fetch(:products_page_path)}"
+    mode = @options.fetch(:mode)
+    if mode == 0 || mode == 1
       browser = Watir::Browser.new :chrome, args: %w[--headless --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=9222]
-      browser.goto(@products_page_url)
-      item_row_elements = browser.elements(css: @item_row_selector)
+      browser.goto(products_page_url)
+      @item_row_elements = browser.elements(css: @options.fetch(:item_row_selector))
     end
-    if @mode == 1
-      File.write(@filename, browser.html)
+    if mode == 1
+      File.write(@options.fetch(:filename), browser.html)
       browser.close
     end
-    if @mode == 0
+    if mode == 0
       @document = Nokogiri::HTML(browser.html)
       browser.close
     end
-    if @mode == 1 || @mode == 2
-      @document = File.open(@filename) { |f| Nokogiri::HTML(f) }
+    if mode == 1 || mode == 2
+      @document = File.open(@options.fetch(:filename)) { |f| Nokogiri::HTML(f) }
     end
   end
-
-  def document
-    @document
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: getter_cyndi5
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 2.0.3
 platform: ruby
 authors:
 - Cyndi Cavanaugh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-13 00:00:00.000000000 Z
+date: 2021-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -66,6 +66,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 4.6.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.13.0
 description: A simple getter cyndi5 gem
 email: cynthiacavanaugh1@gmail.com
 executables:
@@ -75,6 +89,7 @@ extra_rdoc_files: []
 files:
 - bin/getter_cyndi5
 - lib/getter_cyndi5.rb
+- lib/getter_cyndi5/importer.rb
 - lib/getter_cyndi5/parser.rb
 - lib/getter_cyndi5/product.rb
 - lib/getter_cyndi5/retriever.rb