mmonitor 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: aa406deb1d049e0bb84bdb8c4cd55c534af658fa
+  data.tar.gz: 058830a5dbe7e114996a3a4c46148769e318d409
+SHA512:
+  metadata.gz: 1da90674d7fcf473f2aac1a0f29882b1315ae80e2d6642cdf1954ef4263d8bed22b22ad3d499ae88176c9b51d694ff1d966da969f9438d4f815241b01e9a0824
+  data.tar.gz: bd655bbb204e52493ccd90491f006eda03803104268fd6dc945f6080d5e2f98d7f7c37e96cb17e76a7203f855c4d471ea1a7d3dfb3b6a257e9082a5de9ca2266

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in crawlers.gemspec
+gemspec

data/README.md

@@ -0,0 +1,77 @@
+# MMonitor
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mmonitor'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mmonitor
+
+## Usage
+
+TODO: Write usage instructions here
+
+``` ruby
+# -*- encoding: utf-8 -*-
+require 'mmonitor'
+items = {}
+url = 'http://loreal.tmall.com/category.htm?search=y&scene=taobao_shop'
+a = MMonitor::Crawler.new(url)
+items.merge!(a.items)
+
+url = 'http://www.amazon.cn/s/ref=sr_nr_p_89_0?rh=i%3Aaps%2Ck%3A%E5%B7%B4%E9%BB%8E%E6%AC%A7%E8%8E%B1%E9%9B%85%2Cp_89%3AL%27Oreal+Paris+%E5%B7%B4%E9%BB%8E%E6%AC%A7%E8%8E%B1%E9%9B%85&keywords=%E5%B7%B4%E9%BB%8E%E6%AC%A7%E8%8E%B1%E9%9B%85&ie=UTF8&qid=1397212889&rnid=125596071'
+b = MMonitor::Crawler.new(url)
+items.merge!(b.items)
+
+url = 'http://search.jd.com/search?keyword=%E7%BE%8E%E5%AE%9D%E8%8E%B2&enc=utf-8&qr=&qrst=UNEXPAND&et=&rt=1&stop=1&area=1&wtype=1&ev=exbrand_%E7%BE%8E%E5%AE%9D%E8%8E%B2%EF%BC%88MAYBELLINE%EF%BC%89%40&uc=0#select'
+c = MMonitor::Crawler.new(url)
+items.merge!(c.items)
+
+url = 'http://search.jumei.com/?filter=0-11-1&search=%E6%AC%A7%E8%8E%B1%E9%9B%85&from=search_topbar_%E6%AC%A7%E8%8E%B1%E9%9B%85_word_pos1&cat=&bid=1'
+d = MMonitor::Crawler.new(url)
+items.merge!(d.items)
+
+url = 'http://search.lefeng.com/search/search?key=%E6%AC%A7%E8%8E%B1%E9%9B%85&wt.s_pg=Isearch&wt.s_pf=public'
+e = MMonitor::Crawler.new(url)
+items.merge!(e.items)
+
+url = 'http://search.suning.com/%E5%B7%B4%E9%BB%8E%E6%AC%A7%E8%8E%B1%E9%9B%85/cityId=9264&iy=-1&ct=1&si=5&st=0'
+f = MMonitor::Crawler.new(url)
+items.merge!(f.items)
+
+url = 'http://www.yhd.com/ctg/s2/c0-0/b/a-s1-v0-p1-price-d0-f06-m1-rt0-pid-mid0-k%E6%AC%A7%E8%8E%B1%E9%9B%85/'
+j = MMonitor::Crawler.new(url)
+items.merge!(j.items)
+
+url = 'http://searchex.yixun.com/html?key=%E6%AC%A7%E8%8E%B1%E9%9B%85&area=1&sort=0&show=0&size=40&pf=1&as=1&charset=utf-8&YTAG=1.100000401#list'
+h = MMonitor::Crawler.new(url)
+items.merge!(h.items)
+
+require 'csv'
+header_row = ['平台ID',  'SKU ID','商品名称',  '平台',  '原价',  '销售价',  '商品图片', '价格图片', '抓取日期']
+CSV.open('测试数据.csv', "wb:GB18030", col_sep: ',') do |csv|
+  csv << header_row
+  items.each do |item_id, item|
+    csv << [
+      item_id,
+      item[:sku_id],
+      item[:title],
+      item[:provider],
+      item[:tag_price],
+      item[:price],
+      item[:photo_url],
+      item[:price_url],
+      Time.now.to_date.to_s
+    ]
+  end
+end
+```

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/mmonitor.rb

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+require 'active_support/core_ext'
+
+module MMonitor
+  VERSION = '0.0.2'
+
+  module Strategies # 目前支持的解析规则
+    autoload :Amazon, 'mmonitor/strategies/amazon'
+    autoload :Jd,     'mmonitor/strategies/jd'
+    autoload :Jumei,  'mmonitor/strategies/jumei'
+    autoload :Lefeng, 'mmonitor/strategies/lefeng'
+    autoload :Suning, 'mmonitor/strategies/suning'
+    autoload :Tmall,  'mmonitor/strategies/tmall'
+    autoload :Yhd,    'mmonitor/strategies/yhd'
+    autoload :Yixun,  'mmonitor/strategies/yixun'
+  end
+  
+  autoload :Crawler, 'mmonitor/crawler'
+  autoload :Parser,  'mmonitor/parser'
+  autoload :Spider,  'mmonitor/spider'
+end

data/lib/mmonitor/crawler.rb

@@ -0,0 +1,83 @@
+# -*- encoding: utf-8 -*-
+require 'addressable/uri'
+
+module MMonitor
+  class Crawler # 爬虫，负责数据抓取流程
+
+    attr_accessor :adapter, :url, :params, :pages, :page, :page_key, :total
+
+    def initialize(url)
+          self.page = 1
+      self.page_key = 'page'
+      process(url)
+    end
+
+    def items
+      self.adapter.items
+    end
+
+    private
+
+    # URL格式化
+    def process(uri)
+       uri = Addressable::URI.parse(uri)
+      host = uri.host
+      self.url = "#{uri.scheme}://#{host}#{uri.path}"
+      self.params = uri.query_values || {}
+
+      self.adapter = case
+      when host.include?('amazon.cn')
+        Strategies::Amazon.new(html)
+      when host.include?('jd.com')
+        Strategies::Jd.new(html)
+      when host.include?('jumei.com')
+        Strategies::Jumei.new(html)
+      when host.include?('lefeng.com')
+        Strategies::Lefeng.new(html)
+      when host.include?('suning.com')
+        Strategies::Suning.new(html)
+      when host.include?('tmall.com')
+        Strategies::Tmall.new(html)
+      when host.include?('yhd.com')
+        Strategies::Yhd.new(html, self.url)
+      when host.include?('yixun.com')
+        Strategies::Yixun.new(html)
+      else
+        puts host
+        puts '_'*88
+        nil
+      end 
+      # 当前页数
+      self.pages = self.adapter.pages
+      self.total = self.adapter.total
+      self.page_key = self.adapter.page_key
+      next_page
+    end
+
+    def next_page
+      puts "分页提示：#{self.page}/#{self.pages}"
+      puts '_'*88
+      if self.pages > self.page
+        self.page += 1
+        self.params[self.page_key] = self.adapter.page(self.page)
+        self.adapter.body = html
+        self.adapter.process
+      else
+        self.adapter.extra
+        self.adapter.body = nil
+        self.adapter.item = nil
+        puts "产品差异：#{self.items.count}/#{self.total}" unless self.total.nil?
+        return nil
+      end
+      next_page
+    end
+
+    def html
+      if self.adapter.nil?
+        Spider.get_html(self.url, self.params)
+      else
+        self.adapter.get_html(self.url, self.params) 
+      end
+    end
+  end
+end

data/lib/mmonitor/parser.rb

@@ -0,0 +1,133 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  
+  class Parser # 解析器，负责网页模板解析。
+
+    attr_accessor :body, :items, :item
+
+    def initialize(dom)
+      return nil if dom.nil?
+       self.body = dom.at(css_path[:body])
+      self.items ||= {}
+      process unless self.body.nil?
+    end
+
+    def process
+      self.list.each do |item|
+        self.item = item
+        id = spu_id
+        unless id.nil?
+          self.items[id] = {
+             provider: self.provider,
+                title: title,
+               sku_id: sku_id,
+            photo_url: photo_url,
+            tag_price: tag_price,
+                price: price,
+            price_url: price_url
+          }
+        end
+      end
+    end
+
+    def list # 产品列表
+      products = []
+      if css_path[:list].is_a?(Array)
+        css_path[:list].each do |li|
+               dom = self.body.at(li)
+          products += dom.css(css_path[:item]) unless dom.nil?
+        end
+      else
+        products += self.body.at(css_path[:list]).css(css_path[:item])
+      end
+      products
+    end
+
+    def extra # 拓展操作
+
+    end
+
+    def pages
+      self.body.at(css_path[:pages]).text.to_i
+    end
+
+    def total
+      if css_path.has_key?(:total)
+        squish( self.body.at(css_path[:total]).try(:text) )
+      else
+        nil
+      end
+    end
+
+    def page_key
+      'page'
+    end
+
+    def page(num)
+      num
+    end
+
+    def get_html(url, params)
+      Spider.get_html(url, params)
+    end
+
+    def spu_id # 产品ID
+      squish( self.item[css_path[:spu_id]] )
+    end
+
+    def sku_id # 单品ID
+      if css_path.has_key?(:sku_id)
+        squish( self.item[css_path[:sku_id]] )
+      else
+        nil
+      end
+    end
+
+    def title # 产品标题
+      squish( self.item.at(css_path[:title]).try(:text) )
+    end
+
+    def photo_url # 产品图片
+      self.item.at(css_path[:photo_url])['src']
+    end
+
+    def price # 产品售价
+      if css_path.has_key?(:price)
+        squish( self.item.at(css_path[:price]).try(:text) ).to_f
+      else
+        nil
+      end
+    end
+
+    def tag_price # 产品原价（可选）
+      if css_path.has_key?(:tag_price)
+        squish( self.item.at(css_path[:tag_price]).try(:text) ).to_f
+      else
+        nil
+      end
+    end
+
+    def price_url # 产品售价图片（可选）
+      if css_path.has_key?(:price_url)
+        self.item.at(css_path[:price_url])['src']
+      else
+        nil
+      end
+    end
+
+    private
+
+    # 字符处理
+    def squish(string)
+      string.gsub(/￥|¥/, '').strip.gsub(/\s+/, '') rescue nil
+    end
+
+    # 无货产品，通过图片获取价格
+    def pirce_from_ocr(price_url)
+      str = MMonitor::Spider.get_ocr(price_url)
+      str.match( /\d+\.\d+/ ).to_f
+    end
+  end
+
+end

data/lib/mmonitor/spider.rb

@@ -0,0 +1,80 @@
+# -*- encoding: utf-8 -*-
+require 'oj'
+require 'faraday'
+require 'nokogiri'
+require 'rtesseract'
+require 'mini_magick'
+
+module MMonitor
+  module Spider # 蜘蛛，负责http请求处理
+    class << self
+      # 抓取HTML
+      def get_html(url, params={})
+        body = get(url, params)
+        $body = body
+        ::Nokogiri::HTML(body)
+      end
+      # 抓取JSON
+      def get_json(url, params={})
+        body = get(url, params)
+        ::Oj.load(body) rescue {}
+      end
+      # 抓取图片上的文字
+      def get_ocr(photo_url)
+        image = MiniMagick::Image.open(photo_url)
+        image.combine_options do |c|
+          c.background '#FFFFFF'
+          c.colorspace 'GRAY'
+          c.alpha 'remove'
+        end
+        image.format 'jpg'
+        ocr = RTesseract.new(image.path, processor: 'mini_magick')
+        str = ocr.to_s
+        image.destroy!
+        return str
+      end
+      # 分页
+      def number_page(total, limit)
+        count = total / limit
+        count += 1 if total % limit > 0
+        count
+      end
+      
+      private
+
+      def get(url, params={})
+        resp = conn.get url, params
+        # 根据状态返回/跳转/终止
+        case resp.status
+        when 200 # 正常
+          resp.body
+        when 302 # 跳转
+          get( resp.headers['location'] )
+        else
+          nil
+        end
+      end
+
+      # 连接
+      def conn
+        @conn ||= Faraday.new(ssl: false)
+        @conn.headers[:user_agent] = switcher
+        @conn.headers[:accept]     = 'text/html,application/json;q=0.9'
+        @conn
+      end
+      # 混淆
+      def switcher
+        [
+          'Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1; 360se)',
+          'Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; TencentTraveler 4.0; .NET CLR 2.0.50727)',
+          'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',
+          'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C)',
+          'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C)',
+          'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; SE 2.X MetaSr 1.0)',
+          'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.802.30 Safari/535.1 SE 2.X MetaSr 1.0',
+          'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.3 (KHTML, like Gecko) Maxthon/3.3.2.1000 Chrome/16.0.883.0 Safari/535.3',
+        ].sample
+      end
+    end
+  end
+end

data/lib/mmonitor/strategies/amazon.rb

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  module Strategies
+    # http://www.amazon.cn/s/ref=sr_nr_p_89_0?rh=i%3Aaps%2Ck%3A%E5%B7%B4%E9%BB%8E%E6%AC%A7%E8%8E%B1%E9%9B%85%2Cp_89%3AL%27Oreal+Paris+%E5%B7%B4%E9%BB%8E%E6%AC%A7%E8%8E%B1%E9%9B%85&keywords=%E5%B7%B4%E9%BB%8E%E6%AC%A7%E8%8E%B1%E9%9B%85&ie=UTF8&qid=1397212889&rnid=125596071
+    class Amazon < MMonitor::Parser
+      # 输出的产品
+      def provider
+        :amazon
+      end
+
+      def css_path
+        {
+          body:   'div#rightResultsATF',
+          list:   ['div#atfResults', 'div#btfResults'],
+          item:   'div.prod.celwidget',
+          total:  '#resultCount > span',
+          pages:  '#pagn > span.pagnDisabled',
+          spu_id: 'name',
+          title:  'h3.newaps>a',
+          photo_url: 'div.image.imageContainer > a > div > img',
+          tag_price: 'ul.rsltGridList > li.newp > div > a > del',
+          price:     'ul.rsltGridList > li.newp > div > a > span'
+        }
+      end
+
+    end
+  end
+end

data/lib/mmonitor/strategies/jd.rb

@@ -0,0 +1,72 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  module Strategies
+    # http://search.jd.com/search?keyword=%B0%CD%C0%E8%C5%B7%C0%B3%D1%C5&qr=&qrst=UNEXPAND&et=&rt=1&stop=1&ev=exbrand_%C5%B7%C0%B3%D1%C5%A3%A8LOREAL%A3%A9%40&area=1&wtype=1
+    class Jd < MMonitor::Parser
+      # 输出的产品
+      def provider
+        :jd
+      end
+
+      def css_path
+        {
+          body:      'body>div.w.main>div.right-extra',
+          list:      'div#plist>ul.list-h',
+          item:      'li',
+          total:     'div#filter>div.fore1>div.total>span>strong',
+          pages:     'div#filter>div.fore1>div#top_pagi>span',
+          spu_id:    'sku',
+          title:     'div>div.p-name>a',
+          photo_url: 'div>div.p-img >a>img'
+        }
+      end
+
+      def spu_id # 产品ID
+        id = squish( self.item[css_path[:spu_id]] )
+        id == 'scroll_loading' ? nil : id
+      end
+
+      def photo_url # 产品图片
+        self.item.at(css_path[:photo_url])['data-lazyload']
+      end
+
+      def pages
+        self.body.at(css_path[:pages]).text.split('/')[1].to_i
+      end
+
+      def extra
+        get_prices(self.items.keys)
+      end
+
+      private
+      # 设置价格
+      def set_prices(prices)
+        prices.each do |item|
+          id = item['id'].gsub('J_', '')
+          if self.items.has_key?(id) && item['m'].to_i > 0
+            self.items[id][:tag_price] = item['m'].to_f
+            self.items[id][:price] = item['p'].to_f
+          else
+            puts item['id']
+            puts '_'*88
+          end
+        end
+      end
+      # 获取价格
+      def get_prices(sku_ids)
+        url = "http://p.3.cn/prices/mgets"
+        sku_ids = sku_ids.map { |sku_id| "J_#{sku_id}" }
+        sku_ids.each_slice(100) do |skuids|
+          params = {
+            skuids: skuids.join(',')
+          }
+          prices = MMonitor::Spider.get_json(url, params)
+          unless prices.empty?
+            set_prices(prices)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/mmonitor/strategies/jumei.rb

@@ -0,0 +1,45 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  module Strategies
+    # http://search.jumei.com/?filter=0-11-1&search=%E6%AC%A7%E8%8E%B1%E9%9B%85&from=search_topbar_%E6%AC%A7%E8%8E%B1%E9%9B%85_word_pos1&cat=&bid=1
+    class Jumei < MMonitor::Parser
+      # 输出的产品
+      def provider
+        :jumei
+      end
+
+      def css_path
+        {
+          body:      'body>div#container>div#body>div#search_result_wrap>div#search_list_wrap',
+          list:      'div.products_wrap>ul',
+          item:      'li',
+          total:     'div.search_list_head_fiex>div>div.head_pagecount>span',
+          pages:     'div.search_list_head_fiex>div>div.head_pageInfo',
+          spu_id:    'pid',
+          title:     'div>div.s_l_name>a',
+          price:     'div > div.search_list_price > span',
+          tag_price: 'div > div.search_list_price > del',
+          photo_url: 'div>div.s_l_pic>a>img'
+        }
+      end
+
+      def page_key
+        'filter'
+      end
+
+      def pages
+        self.body.at(css_path[:pages]).text.split('/')[1].to_i
+      end
+
+      def page(num)
+        "0-11-#{num}"
+      end
+
+      def photo_url # 产品图片
+        self.item.at(css_path[:photo_url])['original']
+      end
+
+    end
+  end
+end

data/lib/mmonitor/strategies/lefeng.rb

@@ -0,0 +1,82 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  module Strategies
+    # http://search.lefeng.com/search/search?key=%E6%AC%A7%E8%8E%B1%E9%9B%85&wt.s_pg=Isearch&wt.s_pf=public
+    class Lefeng < MMonitor::Parser
+      # 输出的产品
+      def provider
+        :lefeng
+      end
+
+      def css_path
+        {
+          body:      'body>div.mainBg>div.main>div.cont>div#clothList',
+          list:      'div.smPruArea>div.makeup',
+          item:      'div.pruwrap',
+          total:     'div#sm-nav > span.tpages > span.tpageNum > i',
+          pages:     'div#sm-nav > span.tpages > span.tpageNum > em',
+          spu_id:    'id',
+          sku_id:    'skuids',
+          title:     'dl>dd.nam>a',
+          photo_url: 'dl>dt>a>img',
+          tag_price: 'dl>dd.pri>del',
+          price_url: 'dl>dd.pri>img',
+        }
+      end
+
+      def pages
+        self.body.at(css_path[:pages]).text.split('/')[1].to_i
+      end
+
+      def page_key
+        'pageNo'
+      end
+
+      def price
+        price_from_cart(spu_id, sku_id) || pirce_from_ocr(price_url)
+      end
+
+      def sku_id
+         self.item.at('dl')[css_path[:sku_id]]
+      end
+
+      def photo_url # 产品图片
+        self.item.at(css_path[:photo_url])['pagespeed_lazy_src']
+      end
+
+      def price_url # 产品售价图片（可选）
+        self.item.at(css_path[:price_url])['pagespeed_lazy_src'].gsub('_73_', '_75_')
+      end
+
+      private
+      # 通过购物车，获取价格
+      def price_from_cart(spu_id, sku_id)
+        url = "http://shopping.lefeng.com/cart/ajaxUpsellcart.jsp"
+        params = {
+          'productId' => spu_id,
+          'skuId' => sku_id,
+          'quantity' => 1,
+          'itemType' => 0,
+          'process' => 'add',
+          'skuExtraPriceId' => 0,
+          'fromSite' => 'webAllProvince',
+          '_' => 1398446675733
+        }
+        json = MMonitor::Spider.get_json(url, params)
+        if json["errorInfo"].blank?
+           return json['totalPrice'] if json['totalPrice'].is_a?(Float)
+        end
+        puts '将通过图片识别价格，很耗资源呢~'
+        puts '_'*88
+        nil
+      end
+      # 无货产品，通过图片获取价格
+      def pirce_from_ocr(price_url)
+        price_url.gsub!('_73_', '_75_')
+        str = MMonitor::Spider.get_ocr(price_url)
+        str.match( /\d+\.\d+/ )[0].to_f
+      end
+    end
+  end
+end

data/lib/mmonitor/strategies/suning.rb

@@ -0,0 +1,69 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  module Strategies
+    # http://search.suning.com/%E5%B7%B4%E9%BB%8E%E6%AC%A7%E8%8E%B1%E9%9B%85/cityId=9264&iy=-1&ct=1&si=5&st=0
+    class Suning < MMonitor::Parser
+      # 输出的产品
+      def provider
+        :suning
+      end
+
+      def css_path
+        {
+          body:      'body>div.wrap-search>div.w1190>div.proList.mb10',
+          list:      'div#productTab>div#proShow>ul.container',
+          item:      'li.item',
+          total:     'div#filterContent>div.searchKeyT>i',
+          pages:     '#refresh>div.page>span>i#pageTotal',
+          spu_id:    'name',
+          sku_id:    'class',
+          title:     'div > span > a > p',
+          photo_url: 'a > img',
+          price_url: 'div > p > img'
+        }
+      end
+
+      def spu_id
+        self.item[css_path[:spu_id]].to_i
+      end
+
+      def sku_id
+        self.item[css_path[:sku_id]].split(' ')[0]
+      end
+
+      def price_url # 产品售价图片（可选）
+        dom = self.item.at(css_path[:price_url])
+        dom['src2'] unless dom.nil?
+      end
+
+      def price
+        price_from_ajax(sku_id) || pirce_from_ocr(price_url)
+      end
+
+      def page_key
+        'cp'
+      end
+
+      def page(num)
+        num-1
+      end
+
+      def get_html(url, params)
+        uri = Addressable::URI.new
+        uri.query_values = params
+        Spider.get_html("#{url}&#{uri.query}")
+      end
+
+      private
+      # 浏览记录，获取价格
+      def price_from_ajax(sku_id)
+        url = "http://product.suning.com/0000000000/browseHistory_10052_10051_#{sku_id}_9264_.html"
+        json = MMonitor::Spider.get_json(url)
+        return json['price'].to_f if json.has_key?('price')
+        nil
+      end
+      
+    end
+  end
+end

data/lib/mmonitor/strategies/tmall.rb

@@ -0,0 +1,39 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  module Strategies
+    # http://loreal.tmall.com/category.htm?search=y&scene=taobao_shop
+    class Tmall < MMonitor::Parser
+      # 输出的产品
+      def provider
+        :tmall
+      end
+
+      def css_path
+        {
+          body:      '#TmshopSrchNav>div',
+          list:      'div.J_TItems',
+          item:      'dl.item',
+          pages:     'div.filter.clearfix.J_TFilter > p > b.ui-page-s-len', # .text.split('/')[1]
+          spu_id:    'data-id',
+          title:     'dd.detail > a',
+          photo_url: 'dt > a:nth-child(1) > img', # 'data-ks-lazyload'
+          price:     'dd.detail > div > div.cprice-area > span.c-price'
+        }
+      end
+
+      def photo_url # 产品图片
+        self.item.at(css_path[:photo_url])['data-ks-lazyload']
+      end
+
+      def pages
+        self.body.at(css_path[:pages]).text.split('/')[1].to_i
+      end
+
+      def page_key
+        'pageNo'
+      end
+
+    end
+  end
+end

data/lib/mmonitor/strategies/yhd.rb

@@ -0,0 +1,94 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  module Strategies
+    # http://www.yhd.com/ctg/s2/c0-0/b/a-s1-v0-p1-price-d0-f06-m1-rt0-pid-mid0-k%E6%AC%A7%E8%8E%B1%E9%9B%85/
+    class Yhd < MMonitor::Parser
+
+      attr_accessor :body, :items, :item, :url, :pages_count, :page
+
+      def initialize(dom, url)
+        self.body = dom.at(css_path[:body])
+        self.items ||= {}
+        self.url = url.sub('s2', 'searchPage')
+        self.pages_count = self.body.at(css_path[:pages]).text.split('/')[1].to_i
+        self.page = 1
+        process
+        get_ajax(true)
+        next_page
+      end
+
+      # 输出的产品
+      def provider
+        :yhd
+      end
+
+      def css_path
+        {
+          body:      '#plist',
+          list:      '#itemSearchList',
+          item:      'li.search_item > div.search_item_box',
+          pages:     '#rankOpDiv > div > div.select_page_num', # .text.split('/')[1]
+          spu_id:    'pmid',
+          title:     'p.title > a',
+          photo_url: 'a.search_prod_img > img', # 'data-ks-lazyload'
+          price:     'div.pricebox.clearfix  > span:nth-child(1)'
+        }
+      end
+
+      def spu_id # 产品图片
+        self.item.at(css_path[:title])[css_path[:spu_id]]
+      end
+
+      def photo_url # 产品图片
+        img = self.item.at(css_path[:photo_url])
+        img['original'] || img['src']
+      end
+
+      # 分页伪装
+      def pages
+        1
+      end
+
+      private
+
+      def next_page
+        puts "分页提示：#{self.page}/#{self.pages_count}"
+        puts '_'*88
+        if self.pages_count > self.page
+          self.page += 1
+          self.url.gsub!("-p#{self.page-1}-", "-p#{self.page}-")
+          get_ajax # 执行
+          get_ajax(true)
+        else
+          self.body = nil
+          self.item = nil
+          return nil
+        end
+        next_page
+      end
+
+      def more_params
+        {
+          'isGetMoreProducts' => 1,
+          'moreProductsDefaultTemplate' => 0
+        }
+      end
+
+      # 加载更多商品
+      def get_ajax(more=false)
+           params = more ? more_params : {}
+             json = MMonitor::Spider.get_json(self.url, params)
+        if json.has_key?('value')
+          html = json['value']
+          unless html.nil?
+            self.body = ::Nokogiri::HTML("<div id='itemSearchList'>#{html}</div>")
+            # 执行
+            self.process
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/mmonitor/strategies/yixun.rb

@@ -0,0 +1,36 @@
+# -*- encoding: utf-8 -*-
+
+module MMonitor
+  module Strategies
+    # http://searchex.yixun.com/html?key=%E6%AC%A7%E8%8E%B1%E9%9B%85&area=1&sort=0&show=0&size=40&pf=1&as=1&charset=utf-8&YTAG=1.100000401#list
+    class Yixun < MMonitor::Parser
+      # 输出的产品
+      def provider
+        :yixun
+      end
+
+      def css_path
+        {
+          body:      'div#container > div > div.grid_m.smain > div.grid_m_inner',
+          list:      'div.goods > ul#itemList',
+          item:      'li.goods_li',
+          total:     '#list > div.sort_page > div.sort_page_txt > b',
+          pages:     '#list > div.sort_page > div.sort_page_num > span', # .text.split('/')[1]
+          spu_id:    'commid',
+          title:     'div > div.mod_goods_info > p.mod_goods_tit > a',
+          photo_url: 'div > div.mod_goods_img > a > img', # 'data-ks-lazyload'
+          price:     'div > div.mod_goods_info > p.mod_goods_price > span.mod_price > span'
+        }
+      end
+
+      def photo_url # 产品图片
+        self.item.at(css_path[:photo_url])['init_src']
+      end
+
+      def pages
+        self.body.at(css_path[:pages]).text.split('/')[1].to_i
+      end
+
+    end
+  end
+end

data/mmonitor.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+lib = "mmonitor"
+lib_file = File.expand_path("../lib/#{lib}.rb", __FILE__)
+File.read(lib_file) =~ /\bVERSION\s*=\s*["'](.+?)["']/
+version = $1
+
+Gem::Specification.new do |spec|
+  spec.name          = lib
+  spec.version       = version
+  spec.authors       = ["Xu Fei", "Howl王"]
+  spec.email         = ["xfstart07@gmail.com", "howl.wong@gmail.com"]
+  spec.summary       = %q{Write a short summary. Required.}
+  spec.description   = %q{Write a longer description. Optional.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files = %w(Gemfile README.md Rakefile)
+  spec.files << "#{lib}.gemspec"
+  spec.files += Dir.glob("lib/**/*.rb")
+
+  spec.add_runtime_dependency 'oj', '~> 2.8', '>= 2.8.1'
+  spec.add_runtime_dependency 'faraday', '~> 0.9', '>= 0.9.0'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.6', '>= 1.6.1'
+  spec.add_runtime_dependency 'rtesseract', '~> 1.2', '>= 1.2.1'
+  spec.add_runtime_dependency 'mini_magick', '~> 3.7', '>= 3.7.0'
+  spec.add_runtime_dependency 'addressable', '~> 2.3', '>= 2.3.6'
+  spec.add_runtime_dependency 'activesupport', '~> 4.1', '>= 4.1.0'
+
+  spec.add_development_dependency 'pry', '~> 0.9', '>= 0.9.12.6'
+  spec.add_development_dependency 'awesome_print', '~> 1.2', '>= 1.2.0'
+end

metadata

@@ -0,0 +1,242 @@
+--- !ruby/object:Gem::Specification
+name: mmonitor
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Xu Fei
+- Howl王
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: oj
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.1
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+- !ruby/object:Gem::Dependency
+  name: rtesseract
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+- !ruby/object:Gem::Dependency
+  name: mini_magick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.7.0
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.3.6
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.1.0
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.12.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.12.6
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+description: Write a longer description. Optional.
+email:
+- xfstart07@gmail.com
+- howl.wong@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- lib/mmonitor.rb
+- lib/mmonitor/crawler.rb
+- lib/mmonitor/parser.rb
+- lib/mmonitor/spider.rb
+- lib/mmonitor/strategies/amazon.rb
+- lib/mmonitor/strategies/jd.rb
+- lib/mmonitor/strategies/jumei.rb
+- lib/mmonitor/strategies/lefeng.rb
+- lib/mmonitor/strategies/suning.rb
+- lib/mmonitor/strategies/tmall.rb
+- lib/mmonitor/strategies/yhd.rb
+- lib/mmonitor/strategies/yixun.rb
+- mmonitor.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.3.0
+signing_key: 
+specification_version: 4
+summary: Write a short summary. Required.
+test_files: []