getter_cyndi5 0.0.7 → 2.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f375f528599295fb8680b981577d5f4696a4f0a82dc6cd98c0e8e71c38adb900
-  data.tar.gz: 1e68edbf8781aa1d1ee17318d4a5cfb074cd4dc0b4b7fef78d5b8802ad15e37a
+  metadata.gz: 41b26c485b1a30f7c06914529d8c5a191cf48598524612f320848c479e6e1410
+  data.tar.gz: e81a4c7846c2b8a222828482a237083b1721546f6ee7e87c4ec79c71302930cd
 SHA512:
-  metadata.gz: 3bed2a5ed995bafaaa11a157dd9c19888ae5ed6989ddbbe0bf10200e2b3cfb520717f176238c2299ae3ca8ff0fe4455c258f5b671c3b23bcd11ff1ade050e7e8
-  data.tar.gz: cf5821fea29ee00a65031413a62affa6f3fea672580b5a8c81c905f4fc9fc1f10ab2e54ea33cc95fa6f8eafc8b04d13bb42f7c09de38ced5c08fa2dbb899c21d
+  metadata.gz: ea53f1c726b4d4d5d97fcfd6e7e52bc003f2369b11c0047e2b6ebde387c6d13df53014f14fe8a6704a40b770e7a6ab504ecadf209e1bbcb99337b060160f6078
+  data.tar.gz: cb5e36cea8f71d160f7e1232c812c4b9613662d8570c0bdaae59502f440153268995d66632dca8d36d23ab0f61e30324d4636bf41fbba973de0ae1c8a62cfe88

data/bin/getter_cyndi5

@@ -1,10 +1,14 @@
 #!/usr/bin/env ruby
 
 require 'getter_cyndi5'
-base_url = ARGV[0] || 'https://thehappyco.com'
-products_page_path = ARGV[1] || '/kelly/products'
-item_row_selector = ARGV[2] || '.item-row'
-item_anchor_selector = ARGV[3] || 'div > div.product-desc.text-center > div.product-title > h3 > a'
-mode = (ARGV[4] || '2').to_i
-filename = ARGV[5] || './tmp/document1.html'
-GetterCyndi5.go(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+
+options = {
+  base_url: ARGV[0] || 'https://thehappyco.com',
+  products_page_path: ARGV[1] || '/kelly/products',
+  item_row_selector: ARGV[2] || '.item-row',
+  item_anchor_selector: ARGV[3] || 'div > div.product-desc.text-center > div.product-title > h3 > a',
+  item_price_selector: ARGV[4] || 'div > div.product-desc.text-center > div.product-price',
+  mode: (ARGV[7] || '2').to_i,
+  filename: ARGV[8] || './tmp/test_document.html'
+}
+GetterCyndi5.go(**options)

data/lib/getter_cyndi5.rb

@@ -3,29 +3,47 @@ class GetterCyndi5
   # Getter
   #
   # Example:
-  #   >> GetterCyndi5.go(base_url = 'https://thehappyco.com', products_page_path = '/kelly/products', item_row_selector = '.item-row', item_anchor_selector = 'div > div.product-desc.text-center > div.product-title > h3 > a', mode = 2, './tmp/document1.html')
+  #   >> GetterCyndi5.go(
+  #         base_url = 'https://thehappyco.com',
+  #         products_page_path = '/kelly/products',
+  #         item_row_selector = '.item-row',
+  #         item_anchor_selector = 'div > div.product-desc.text-center > div.product-title > h3 > a',
+  #         item_price_selector = 'div > div.product-desc.text-center > div.product-price',
+  #         price_a_text: 'One Time Purchase',
+  #         price_b_text: 'SmartShip',
+  #         mode = 2,
+  #         filename = './tmp/test_document.html'
+  #     )
   # Arguments:
   #   base_url: (String)
   #   products_page_path: (String) 
   #   item_row_selector: (String) 
   #   item_anchor_selector: (String)
-  #   mode: (Integer) 0 = retrieve and parse without saving HTML document to file, 1 = retrieve and parse saving HTML document to file, 2 = load and parse HTML document from file
+  #   item_price_selector: (String)
+  #   price_a_text: (String)
+  #   price_b_text: (String)
+  #   mode: (Integer)
+  #     0 = retrieve and parse without saving HTML document to file,
+  #     1 = retrieve and parse saving HTML document to file,
+  #     2 = load and parse HTML document from file
   #   filename: (String)
 
-  def self.go(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
-    all_products = products(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
+  def self.go(options = {})
+    all_products = products(**options)
     all_products.each do |product|
-      puts "#{product.name} :-: #{product.url}"
+      puts "#{product.inspect}"
     end
   end
 
-  def self.products(base_url, products_page_path, item_row_selector, item_anchor_selector, mode, filename)
-    retriever = Retriever.new(base_url, products_page_path, item_row_selector, mode, filename)
+  def self.products(options = {})
+    retriever = Retriever.new(**options)
     retriever.retrieve
-    parser = Parser.new(retriever.document, base_url, item_row_selector, item_anchor_selector)
+
+    parser = Parser.new(document: retriever.document, **options)
     products = parser.parse
   end
 end
 
 require 'getter_cyndi5/retriever'
-require 'getter_cyndi5/parser'
+require 'getter_cyndi5/parser'
+require 'getter_cyndi5/importer'

data/lib/getter_cyndi5/importer.rb

@@ -0,0 +1,13 @@
+class Importer
+  def initialize(importer_options = {})
+    @importer_options = importer_options
+  end
+
+  def import(product)
+      { :item_code => product.attributes[:item_data].fetch(@importer_options.fetch(:itemcode_key), ''),
+        :name => product.attributes[:name],
+        :url => product.attributes[:url],
+        :price_a => product.attributes[:prices].fetch(@importer_options.fetch(:price_a_key), 0.0),
+        :price_b => product.attributes[:prices].fetch(@importer_options.fetch(:price_b_key), 0.0) }
+  end
+end

data/lib/getter_cyndi5/parser.rb

@@ -1,21 +1,34 @@
 require 'getter_cyndi5/product'
 class GetterCyndi5::Parser
+  DATA_PREFIX = 'data-'.freeze
   attr_reader :products
-  def initialize(document, base_url, item_row_selector, item_anchor_selector)
-    @document = document
-    @base_url = base_url
-    @item_row_selector = item_row_selector
-    @item_anchor_selector = item_anchor_selector
+  def initialize(options = {})
+    @options = options
     @products = []
   end
 
-  def parse     
-    item_rows = @document.css(@item_row_selector)
+  def parse
+    document = @options.fetch(:document)
+    base_url = @options.fetch(:base_url)
+    item_row_selector = @options.fetch(:item_row_selector)
+    item_anchor_selector = @options.fetch(:item_anchor_selector)
+    item_price_selector = @options.fetch(:item_price_selector)
+    item_rows = document.css(item_row_selector)
     item_rows.each do |item_row|
-      product_element = item_row.css(@item_anchor_selector)[0]
-      product_name = product_element.text
-      product_url = "#{@base_url}#{product_element.attributes['href']}"
-      product = GetterCyndi5::Product.new(product_name, product_url)
+      item_data = item_row.attributes.select { |k, v| k.start_with? DATA_PREFIX }.map { |k, v| [k.delete_prefix(DATA_PREFIX), v.value]}.to_h
+      product_element = item_row.css(item_anchor_selector)[0]
+      price_elements = item_row.css(item_price_selector)
+      prices = {}
+      price_elements.each do |price_element|
+        prices[price_element.children[1].text] = price_element.children[0].text.gsub(/[^\d\.]/, '').to_f
+      end
+      attributes = {
+        name: product_element.text,
+        url: "#{base_url}#{product_element.attributes['href']}",
+        item_data: item_data,
+        prices: prices
+      }
+      product = GetterCyndi5::Product.new(attributes)
       products.append(product)
     end
     products

data/lib/getter_cyndi5/product.rb

@@ -1,10 +1,6 @@
 class GetterCyndi5::Product
-  attr_accessor :name, :url, :price_a, :price_b, :price_c
-  def initialize(name, url, price_a=0.0, price_b=0.0, price_c=0.0)
-    @name = name
-    @url = url
-    @price_a = price_a
-    @price_b = price_b
-    @price_c = price_c
+  attr_accessor :attributes
+  def initialize(attributes = {})
+    @attributes = attributes
   end
 end

data/lib/getter_cyndi5/retriever.rb

@@ -4,35 +4,30 @@ require 'watir'
 require 'webdrivers'
 
 class GetterCyndi5::Retriever
-  def initialize(base_url, products_page_path, item_row_selector, mode, filename)
-    @base_url = base_url
-    @products_page_path = products_page_path
-    @products_page_url = "#{@base_url}#{@products_page_path}"
-    @item_row_selector = item_row_selector
-    @mode = mode
-    @filename = filename
+  def initialize(options = {})
+    @options = options
   end
 
+  attr_reader :document
+  attr_reader :item_row_elements
   def retrieve()
-    if @mode == 0 || @mode == 1
-      browser = Watir::Browser.new
-      browser.goto(@products_page_url)
-      item_row_elements = browser.elements(css: @item_row_selector)
+    products_page_url = "#{@options.fetch(:base_url)}#{@options.fetch(:products_page_path)}"
+    mode = @options.fetch(:mode)
+    if mode == 0 || mode == 1
+      browser = Watir::Browser.new :chrome, args: %w[--headless --no-sandbox --disable-dev-shm-usage --disable-gpu --remote-debugging-port=9222]
+      browser.goto(products_page_url)
+      @item_row_elements = browser.elements(css: @options.fetch(:item_row_selector))
     end
-    if @mode == 1
-      File.write(@filename, browser.html)
+    if mode == 1
+      File.write(@options.fetch(:filename), browser.html)
       browser.close
     end
-    if @mode == 0
+    if mode == 0
       @document = Nokogiri::HTML(browser.html)
       browser.close
     end
-    if @mode == 1 || @mode == 2
-      @document = File.open(@filename) { |f| Nokogiri::HTML(f) }
+    if mode == 1 || mode == 2
+      @document = File.open(@options.fetch(:filename)) { |f| Nokogiri::HTML(f) }
     end
   end
-
-  def document
-    @document
-  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: getter_cyndi5
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 2.0.2
 platform: ruby
 authors:
 - Cyndi Cavanaugh
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-12 00:00:00.000000000 Z
+date: 2021-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: httparty
@@ -66,6 +66,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 4.6.0
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.13.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.13.0
 description: A simple getter cyndi5 gem
 email: cynthiacavanaugh1@gmail.com
 executables:
@@ -75,6 +89,7 @@ extra_rdoc_files: []
 files:
 - bin/getter_cyndi5
 - lib/getter_cyndi5.rb
+- lib/getter_cyndi5/importer.rb
 - lib/getter_cyndi5/parser.rb
 - lib/getter_cyndi5/product.rb
 - lib/getter_cyndi5/retriever.rb