amazon_order 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: e3a6bf0fa3f7452a0d5902ad1f18573a91533e3b
-  data.tar.gz: 20b21eb7ee25fb98487583bb8369f371acef22bc
+SHA256:
+  metadata.gz: ed9bb5a826d4c1d9248638828f0fab21219b2dccba0306e5e3c512801a2ce60e
+  data.tar.gz: dd2bcce492a9218a499bc63a3778f3c9d5a1f3baa3f4a018e4fbb416f84c69d4
 SHA512:
-  metadata.gz: 56ee5c0283f97849f9cc461b780ceab6cbc4e468fd039066cef4f1ba3968b60b1cc5e34192c100eb8f8ec786435aa5e9d3b0620d2e0009d516fba0a53b00be81
-  data.tar.gz: 3679036cfa689b836654f8c57c4bcd093fecdc3ca4a17a900f49457ba424d71bb99eb36aa53bd3d2fc0b14c3fb01c2a8b6525ca92e71568ccc4262a6cf5c9b9f
+  metadata.gz: 9e0f7903e9248d6e83a75e2bcf7810f2376d607453b8d05fe57c10deaeff21246fffb4d7c6c07e213ddf0aa67480be748c7253f6b0abdd181401d3467780853d
+  data.tar.gz: b556a0c0596b71e14d552da3e60aa6e8faf76c35e3a6e1f07469ffcfbe3798e74ad6cecbd76d67b1f9018993ebdc140174291f7b4f582e97f1206f552ee2c5a5

data/.travis.yml

@@ -1,5 +1,7 @@
 sudo: false
 language: ruby
 rvm:
-  - 2.2.4
-before_install: gem install bundler -v 1.14.6
+  - 2.5.8
+  - 2.6.6
+  - 2.7.1
+before_install: gem install bundler -v 1.17.3

data/README.md

@@ -37,11 +37,14 @@ Or install it yourself as:
 
 [chromedriver](https://sites.google.com/a/chromium.org/chromedriver/downloads) is required. Please [download chromedriver](http://chromedriver.storage.googleapis.com/index.html) and update chromedriver regularly.  
 
-Create _.env_ following the instructions of https://github.com/kyamaguchi/amazon_auth
+Create credentials following the instructions of https://github.com/kyamaguchi/amazon_auth  
+Use `envchain` or _.env_
 
 ```
 amazon_auth
 
+envchain amazon ...
+# OR
 vi .env
 ```
 
@@ -53,7 +56,7 @@ In console
 
 ```ruby
 require 'amazon_order'
-client = AmazonOrder::Client.new(verbose: true, limit: 10)
+client = AmazonOrder::Client.new(keep_cookie: true, verbose: true, limit: 10)
 client.fetch_amazon_orders
 # Fetch orders of specified year
 client.fetch_orders_for_year(year: 2016)
@@ -65,8 +68,8 @@ client.go_to_amazon_order_page
 client.fetch_orders_for_year(year: 2015)
 ```
 
-Downloaded pages will be stored into `order` directory.
-You can reset by moving that directory.
+Downloaded pages will be stored into `tmp/orders` directory.  
+`tmp` comes from `Capybara.save_path`.  
 
 Once `fetch_amazon_orders` succeeds, you can load orders information of downloaded pages anytime.
 (You don't need to fetch pages with launching browser every time.)
@@ -86,7 +89,7 @@ products.size
 orders.group_by{|o| o.order_placed.strftime('%Y') }.sort_by{|year,_| year }.map{|year,records| puts [year, records.map(&:order_total).sum].inspect };nil
 ```
 
-Example of data
+#### Example of data
 
 ```ruby
 console> pp orders.first.to_hash
@@ -111,16 +114,24 @@ console> pp products.first.to_hash
  "https://images-fe.ssl-images-amazon.com/images/I/51TODrMIEnL.jpg"}
 ```
 
+#### Export csv
+
+```ruby
+client.generate_csv
+```
+
 #### Options
 
 Limit fetching with number of pages: `client = AmazonOrder::Client.new(limit: 5)`
-`limit: nil` for no limit. default is 5
+(`limit: nil` for no limit. default is 5)
 
 Set year range: `client = AmazonOrder::Client.new(year_from: 2012, year_to: 2013)`
-default is Time.current.year
+(default is Time.current.year)
 
 ##### Options of amazon_auth gem
 
+Keep cookies(keep signin): `keep_cookie: true`
+
 Firefox: `driver: :firefox`
 
 Output debug log: `debug: true`

data/amazon_order.gemspec

@@ -21,10 +21,10 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "amazon_auth", "~> 0.3.3"
-  spec.add_development_dependency "bundler", "~> 1.14"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_runtime_dependency "amazon_auth", "~> 0.4.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
   spec.add_development_dependency "byebug"
   spec.add_development_dependency "pry-rescue"
   spec.add_development_dependency "pry-stack_explorer"

data/lib/amazon_order.rb

@@ -3,6 +3,7 @@ require "amazon_order/version"
 require "amazon_order/client"
 require "amazon_order/parsers/base"
 require "amazon_order/parsers/order"
+require "amazon_order/parsers/shipment"
 require "amazon_order/parsers/product"
 require "amazon_order/parser"
 require "amazon_order/writer"

data/lib/amazon_order/client.rb

@@ -6,11 +6,14 @@ module AmazonOrder
 
     def initialize(options = {})
       @options = options
-      @base_dir = @options.fetch(:base_dir, 'orders')
       @client = AmazonAuth::Client.new(@options)
       extend(AmazonAuth::SessionExtension)
     end
 
+    def base_dir
+      options.fetch(:base_dir, 'orders')
+    end
+
     def year_from
       options.fetch(:year_from, Time.current.year)
     end
@@ -37,7 +40,7 @@ module AmazonOrder
 
     def load_amazon_orders
       orders = []
-      Dir.glob("#{@base_dir}/*html").each do |filepath|
+      Dir.glob(file_glob_pattern).each do |filepath|
         log "Loading #{filepath}"
         parser = AmazonOrder::Parser.new(filepath)
         orders += parser.orders
@@ -45,13 +48,34 @@ module AmazonOrder
       orders.sort_by{|o| -o.fetched_at.to_i }.uniq(&:order_number)
     end
 
+    def file_glob_pattern
+      File.join(Capybara.save_path, base_dir, '*html')
+    end
+
+    def generate_csv
+      writer.generate_csv
+    end
+
+    def writer
+      @_writer ||= AmazonOrder::Writer.new(file_glob_pattern)
+    end
+
     def sign_in
       @client.sign_in
     end
 
     def go_to_amazon_order_page
+      if doc.css('.cvf-account-switcher').present?
+        log "Account switcher page was displayed"
+        session.first('.cvf-account-switcher-profile-details').click
+        wait_for_selector('#nav-main') # Wait for page loading
+      end
       link = links_for('a').find{|link| link =~ %r{/order-history} }
-      session.visit link
+      if link.present?
+        session.visit link
+      else
+        log "Link for order history wasn't found in #{session.current_url}"
+      end
     end
 
     def fetch_orders_for_year(options = {})
@@ -83,7 +107,7 @@ module AmazonOrder
     def save_page_for(year, page)
       log "Saving year:#{year} page:#{page}"
       path = ['order', year.to_s, "p#{page}", Time.current.strftime('%Y%m%d%H%M%S')].join('-') + '.html'
-      session.save_page(File.join(@base_dir, path))
+      session.save_page(File.join(base_dir, path))
     end
 
     def selected_year

data/lib/amazon_order/parsers/base.rb

@@ -6,6 +6,7 @@ module AmazonOrder
       def initialize(node, options = {})
         @node = node
         @fetched_at = options[:fetched_at]
+        @containing_object = options[:containing_object]
       end
 
       def inspect

data/lib/amazon_order/parsers/order.rb

@@ -3,7 +3,6 @@ module AmazonOrder
     class Order < Base
       ATTRIBUTES = %w[
                      order_placed order_number order_total
-                     shipment_status shipment_note
                      order_details_path
                      all_products_displayed
                    ]
@@ -17,36 +16,55 @@ module AmazonOrder
       end
 
       def order_total
-        @_order_total ||= @node.css('.order-info .a-col-left .a-column')[1].css('.value').text.strip.gsub(/[^\d\.]/,'').to_f
+        @_order_total ||= @node.css('.order-info .a-col-left .a-column')[1].css('.value').text.strip.gsub(/[^\d\.]/, '').to_f
       end
 
-      def shipment_status
-        # class names like "shipment-is-delivered" in '.shipment' node may be useful
-        @_shipment_status ||= @node.css('.shipment .shipment-top-row').present? ? @node.css('.shipment .shipment-top-row .a-row')[0].text.strip : nil
+      def order_details_path
+        @_order_details_path ||= @node.css('.order-info .a-col-right .a-row')[1].css('a.a-link-normal')[0].attr('href')
       end
 
-      def shipment_note
-        @_shipment_note ||= @node.css('.shipment .shipment-top-row').present? ? @node.css('.shipment .shipment-top-row .a-row')[1].text.strip : nil
+      def order_type
+        if @node.css('[id^=Leave-Service-Feedback]').present?
+          return :service_order
+        elsif @node.css('.shipment').present?
+          :shipment_order
+        else
+          :digital_order
+        end
       end
 
-      def order_details_path
-        @_order_details_path ||= @node.css('.order-info .a-col-right .a-row')[1].css('a.a-link-normal')[0].attr('href')
+      def shipments
+        @_shipments ||= @node.css('.shipment')
+          .map do |shipment|
+          AmazonOrder::Parsers::Shipment.new(shipment,
+                                             containing_object: self,
+                                             fetched_at: fetched_at)
+        end
       end
 
-      def all_products_displayed
-        @_all_products_displayed ||= @node.css('.a-box.order-info ~ .a-box .a-col-left .a-row').last.css('.a-link-emphasis').present?
+      def products
+        @products ||= shipment_products + digital_products
       end
 
-      def products
-        @_products ||= @node.css('.a-box.order-info ~ .a-box .a-col-left .a-row')[0].css('.a-fixed-left-grid').map{|e| AmazonOrder::Parsers::Product.new(e, fetched_at: fetched_at) }
+      def shipment_products
+        @shipment_products ||= shipments.flat_map(&:products)
+      end
+
+      def digital_products
+        @_products ||= @node.css('.a-box:not(.shipment) .a-fixed-left-grid').map { |e| AmazonOrder::Parsers::Product.new(e, fetched_at: fetched_at) }
       end
 
+      # might be broken now that orders have multiple shipments
+      def all_products_displayed
+        @_all_products_displayed ||= @node.css('.a-box.order-info ~ .a-box .a-col-left .a-row').last.css('.a-link-emphasis').present?
+      end
 
       def to_hash
         super do |hash|
           hash.merge!(products: products.map(&:to_hash))
         end
       end
+
     end
   end
 end

data/lib/amazon_order/parsers/product.rb

@@ -21,7 +21,10 @@ module AmazonOrder
       end
 
       def image_url
-        @_image_url ||= get_original_image_url(@node.css('.a-col-left img')[0].attr('src'))
+        @_image_url ||= begin
+          img = @node.css('.a-col-left img')[0]
+          get_original_image_url(img.attr('data-a-hires').presence || img.attr('src'))
+        end
       end
     end
   end

data/lib/amazon_order/parsers/shipment.rb

@@ -0,0 +1,38 @@
+module AmazonOrder
+  module Parsers
+    class Shipment < Base
+      ATTRIBUTES = %w[
+                     shipment_status
+                     shipment_note
+                   ]
+
+      # TODO shipment_date
+
+      def order
+        @containing_object
+      end
+
+      def shipment_status
+        # class names like "shipment-is-delivered" in '.shipment' node may be useful
+        @_shipment_status ||= @node.css('.shipment-top-row').present? ? @node.css('.shipment .shipment-top-row .a-row')[0].text.strip : nil
+      end
+
+      def shipment_note
+        @_shipment_note ||= case order.order_type
+        when :shipment_order
+          @node.css('.shipment-top-row').present? ? @node.css('.shipment .shipment-top-row .a-row')[1].text.strip : nil
+        when :service_order
+          nil
+        when :digital_order
+          nil
+        end
+      end
+
+
+      def products
+        @_products ||= @node.css('.a-fixed-left-grid').map { |e| AmazonOrder::Parsers::Product.new(e, fetched_at: fetched_at) }
+      end
+
+    end
+  end
+end

data/lib/amazon_order/version.rb

@@ -1,3 +1,3 @@
 module AmazonOrder
-  VERSION = "0.1.0"
+  VERSION = "0.1.5"
 end

data/lib/amazon_order/writer.rb

@@ -1,7 +1,7 @@
 module AmazonOrder
   class Writer
-    def initialize(base_dir, options = {})
-      @base_dir = base_dir
+    def initialize(file_glob_pattern, options = {})
+      @file_glob_pattern = file_glob_pattern
       @output_dir = options.fetch(:output_dir, 'tmp')
     end
 
@@ -9,7 +9,7 @@ module AmazonOrder
       data['orders']
     end
 
-    def print_produts
+    def print_products
       data['products']
     end
 
@@ -21,7 +21,8 @@ module AmazonOrder
         csv_file = "#{@output_dir}/#{resource}#{Time.current.strftime('%Y%m%d%H%M%S')}.csv"
         puts "    Writing #{csv_file}"
         CSV.open(csv_file, 'wb') do |csv|
-          data[resource].each{|r| csv << r }
+          csv << attributes_for(resource)
+          data[resource].each { |r| csv << r }
         end
         csv_file
       end
@@ -32,7 +33,7 @@ module AmazonOrder
     def data
       @_data ||= begin
         data = {'orders' => [], 'products' => []}
-        Dir.glob("#{@base_dir}/*html").each do |filepath|
+        Dir.glob(@file_glob_pattern).each do |filepath|
           puts "    Parsing #{filepath}"
           parser = AmazonOrder::Parser.new(filepath)
           data['orders'] += parser.orders.map(&:values)
@@ -41,5 +42,14 @@ module AmazonOrder
         data
       end
     end
+
+    def attributes_for(resource)
+      case resource
+      when 'orders'
+        AmazonOrder::Parsers::Order::ATTRIBUTES
+      when 'products'
+        AmazonOrder::Parsers::Product::ATTRIBUTES
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: amazon_order
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Kazuho Yamaguchi
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-07-05 00:00:00.000000000 Z
+date: 2020-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amazon_auth
@@ -16,56 +16,56 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.3
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.14'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -131,6 +131,7 @@ files:
 - lib/amazon_order/parsers/base.rb
 - lib/amazon_order/parsers/order.rb
 - lib/amazon_order/parsers/product.rb
+- lib/amazon_order/parsers/shipment.rb
 - lib/amazon_order/version.rb
 - lib/amazon_order/writer.rb
 homepage: https://github.com/kyamaguchi/amazon_order
@@ -152,8 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Scrape information of amazon orders