shopee 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3aff6cf07785ae43aa42c1f7d802ed94a74a1f53
-  data.tar.gz: c9ac1da5895b9ed8d9e823b4f9a77bb6630aa83e
+  metadata.gz: 0dea88ac985e2a00925725ba22d00524f20f5686
+  data.tar.gz: cc396b07bf301732655b22c836f007f3e076cb52
 SHA512:
-  metadata.gz: 902449372a269b53f95fadb46d434ada76547d44bf67242e2feb2b62482126bc9e176894fef5aa4314ad5054fc60c9aecac0189b368d9b07076dc8019d7aa63a
-  data.tar.gz: da698c2812a2dccb9f88f1c39df063c1a1db922e00077a23108bf6c11b0d610f6930034aab6d5787da20c529e96d8b704fcd87d6807d894ced913b97f4c5e8b6
+  metadata.gz: 5960417b9ea1160df982eba61c4f10389d2afed77f278b038a820af1bf45570f9221ac7d8fe0e959c5b3843e52bc487602b40b115eeceef57db0a953fcfc356a
+  data.tar.gz: 6024ea6d9e1aac0d2d72c32ececa5592e0f607e3509646da1a4e9edfe4dbd4511b8fe0835d6b8d1c9d93a02fa426543b46d94dff8612aa909822f33dfdaf9fb4

data/Gemfile

@@ -6,6 +6,7 @@ gem 'minitest'
 gem 'vcr'
 gem 'webmock'
 gem 'fuzzy-string-match'
+gem 'commander', '~> 4.3', '>= 4.3.5'
 
 group :test do
   gem 'rake'

data/README.md

@@ -20,11 +20,11 @@ $ gem install shopee
 ## 1. List all categories of market of Mobile01
 $ shopee allcate
 
-## 2. List top 30 goods by a category
+## 2. List goods by a category
 $ shopee list [category_name]
 
-## 3. Find top 3 goods in a category by the keyword
-$ shopee search [category_name] [keyword]
+## 3. Find top number goods in a category by the keyword
+$ shopee search [category_name] [keyword] [num]
 
 ```
 
@@ -186,7 +186,7 @@ GPS
 ```
 
 
-**2. List top 30 goods by a category**
+**2. List goods by a category**
 ```ruby
 require 'shopee'
 ##Example ：category is '電腦資訊'
@@ -231,13 +231,13 @@ puts shopeecate.goods
 {"name"=>"全新未拆 羅技MX Master  fervidity (5)　商品所在地:台中市　(可交換商品)", "price"=>"9,500元  10,000元", "num"=>"0", "update_time"=>"2015-11-15"}
 {"name"=>"賣 99.99 新Samsung Galaxy Tab S 8.4 白3G/16G LTE ( 可通話平...  星空下的雨 (31)　商品所在地:新北市　(可交換商品)", "price"=>nil, "num"=>nil, "update_time"=>nil}
 ```
-**3. Find top 3 goods in a category by the keyword**
+**3. Find top num goods in a category by the keyword**
 ```ruby
 require 'shopee'
 ##Example ：category is '電腦資訊'; keyword is 'samsung'
-cshopeecate = ShopeeScrape::ShopeeListGoodsByCate.new(category)
+shopeecate = ShopeeScrape::ShopeeListGoodsByCate.new(category)
 goods = shopeecate.goods
-puts shopeecate.search_keyword(goods, keyword)
+puts shopeecate.search_keyword(goods, keyword, num)
 ```
 - Expected output
 ```

data/lib/data/mobile_category.rb

@@ -140,3 +140,4 @@ ALL_LINK = { "電腦資訊"=>"http://www.mobile01.com/mpcatlist.php?c=1",
   "南部地區"=>"http://www.mobile01.com/mpcatlist.php?c=90",
   "東部與外島地區"=>"http://www.mobile01.com/mpcatlist.php?c=91"
 }
+CATEGORY_LIST = ["電腦資訊", "手持通訊", "攝影器材", "數位家電", "休閒旅遊", "生活用品", "汽車", "機車", "自行車", "男性時尚", "女性流行", "代購與虛擬物品", "房屋地產"]

data/lib/shopee.rb

@@ -7,6 +7,7 @@ module ShopeeScrape
   class ShopeeListGoodsByCate
 
     require_relative './data/mobile_category'
+    GOOD_INFO = "//div[contains(@class, 'img')]"
     GOOD_NAME = "//div[contains(@class, 'subject')]"
     GOOD_PRICE = "//div[contains(@class, 'price')]"
     GOOD_NUM = "//div[contains(@class, 'num')]"
@@ -27,46 +28,85 @@ module ShopeeScrape
     private
 
     def parse_html(id)
+      @document = []
       url = ALL_LINK[id]
-      @document = Oga.parse_html(open(url))
-    end
+      @document << Oga.parse_html(open(url))
 
-    def extract_goods
-      name = []
-      price = []
-      num = []
-      update_time = []
-      @document.xpath(GOOD_NAME).map do |good|
-        name << good.text
+      page_num = 2
+      if CATEGORY_LIST.include?(id) == true
+        page_num = 5
       end
 
-      @document.xpath(GOOD_PRICE).map do |good|
-        price << good.text
+      check = 1
+      i = 1
+
+      while check == 1 && i < page_num
+        i += 1
+        begin
+          url_t = url +'&p='+i.to_s
+          open url_t, :proxy=>true
+          @document << Oga.parse_html(open(url_t))
+        rescue
+          check = 0
+        end
       end
 
-      @document.xpath(GOOD_NUM).map do |good|
-        num << good.text
-      end
+    end
 
-      @document.xpath(GOOD_UPTIME).map do |good|
-        update_time << good.text
-      end
+    def extract_goods
 
-      number = name.length
       results = []
-      # puts number
-      if number > 32
-        number = 32
-      end
-      for i in 2..number-1
-        element = {}
-        element['name'] = name[i]
-        element['price'] = price[i]
-        element['num'] = num[i]
-        element['update_time'] = update_time[i]
-        results << element
-      end
 
+      @document.each do |doc|
+
+        name = []
+        price = []
+        num = []
+        update_time = []
+        pic = []
+        link = []
+
+        doc.xpath(GOOD_NAME).map do |good|
+          name << good.text
+        end
+
+        doc.xpath(GOOD_PRICE).map do |good|
+          price << good.text
+        end
+
+        doc.xpath(GOOD_NUM).map do |good|
+          num << good.text
+        end
+
+        doc.xpath(GOOD_UPTIME).map do |good|
+          update_time << good.text
+        end
+
+        doc.xpath(GOOD_INFO).map do |good|
+          link << good.css('a').attribute('href')[0].to_s
+          # puts good.css('img').attribute('src')[0].to_s
+          pic << good.css('img').attribute('src')[0].to_s
+        end
+
+        number = name.length
+
+        # puts number
+        if number > 32
+          number = 32
+        end
+        for i in 2..number-1
+          element = {}
+          element['name'] = name[i]
+          element['price'] = price[i]
+          element['num'] = num[i]
+          element['update_time'] = update_time[i]
+          element['link'] = link[i]
+          element['pic'] = pic[i]
+          results << element
+        end
+
+      end
+      puts results.length
       results
     end
 

data/lib/shopee/version.rb

@@ -1,4 +1,4 @@
 module ShopeeScrape
-  VERSION = '0.1.2'
-  DATE = '2015-12-06'
+  VERSION = '0.2.0'
+  DATE = '2016-01-09'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: shopee
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Sheng Jung Wu
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-06 00:00:00.000000000 Z
+date: 2016-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -125,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Scrape categories of mobile01!