digger 0.1.5 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c4a94163a25d4b53ad5b477040f69b8fccca026adc313f8f61759317c1bf198
-  data.tar.gz: 307b443277c16708103c172e5fb4ef4d833f7d2631a7f85779570a3cbeac8925
+  metadata.gz: ed96af1c5ae92569e1de4885958ac5852864c045f6c6337d5f17d91747d8ed80
+  data.tar.gz: 82003ae80f54cd3f9b805757e5dcb4c7894bba91c4c376cf08ebd43e6de6e80b
 SHA512:
-  metadata.gz: 9da40123fd09615d0c69ca5104d1141b82981813ef8d175bc567a0f35e8f7dd868ce235bc0f265308e2133710f63c3ce8325d6b528ae060eccb77904c12e3139
-  data.tar.gz: c05be67df6db25345acfdc3615690c0467029dd40ffde262863923b6f0786696a16fd547277ec4d3764ed39b399174dd28e10ca7eb31ffb55f773e08b04f2986
+  metadata.gz: b7aad69fd46c7d1688026ece2e1efe14d7dea29b42f94656d794655e12a92677bd3e1034f0c776bf197bcd75c96bd49377df399433bd2e1c13507520af1addc5
+  data.tar.gz: 60055a69ec3ad77e80fc4f1b50bb3a6c298e2274827ae21b64d1d82dae53a1d5338ccc99bcdb09c7d2e946abd007a488ff3034a753bc54586e9b651aeb3c5ce7

data/digger.gemspec

@@ -1,26 +1,27 @@
 # coding: utf-8
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'digger/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "digger"
+  spec.name          = 'digger'
   spec.version       = Digger::VERSION
-  spec.authors       = ["binz"]
-  spec.email         = ["xinkiang@gmail.com"]
+  spec.authors       = ['binz']
+  spec.email         = ['xinkiang@gmail.com']
   spec.summary       = %q{Dig need stractual infomation from web page.}
   spec.description   = %q{}
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.homepage      = ''
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'bundler', '~> 2.0'
 
-  spec.add_runtime_dependency 'nokogiri', '~> 1.6'
   spec.add_runtime_dependency 'http-cookie', '~> 1.0'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.6'
 end

data/lib/digger/index.rb

@@ -8,33 +8,63 @@ module Digger
 
     def urls
       @urls ||= begin
-        args = self.args.map{|a| (a.respond_to? :each) ? a.to_a : [a]}
-        args.shift.product(*args).map{|arg| pattern_applied_url(arg)}
+        args = self.args.map { |a| a.respond_to?(:each) ? a.to_a : [a] }
+        args.shift.product(*args).map { |arg| pattern_applied_url(arg) }
       end
     end
 
     def pattern_applied_url(arg)
-      pattern.gsub('*').each_with_index{|_, i| arg[i]}
+      pattern.gsub('*').each_with_index { |_, i| arg[i] }
+    end
+
+    def self.slow_down(entities, conf = {}, &block)
+      raise NoBlockError, 'No block given' unless block
+
+      config = {
+        sleep_range_seconds: 4...10, # 随机等待时间范围
+        fail_max_cnt: 10, # 最多失败次数
+        fail_unit_seconds: 10 * 60, # 失败等待时间
+        when_fail: ->(ent, e, failed_cnt) {}
+      }.merge(conf)
+      failed_cnt = 0
+      cursor = 0
+      result = []
+      while cursor < entities.length
+        begin
+          result << block.call(entities[cursor])
+        rescue StandardError => e
+          failed_cnt += 1
+          config[:when_fail].call(entities[cursor], e, failed_cnt)
+          break if failed_cnt >= config[:fail_max_cnt]
+
+          sleep(failed_cnt * config[:fail_unit_seconds])
+        else
+          cursor += 1
+          sleep(rand(config[:sleep_range_seconds]))
+        end
+      end
+      result
     end
 
     def self.batch(entities, cocurrence = 1, &block)
-      raise NoBlockError, "No block given" unless block
+      raise NoBlockError, 'No block given' unless block
 
       if cocurrence > 1
-        results = {}
-        entities.each_slice(cocurrence) do |group|
+        results = Array.new(entities.size)
+        entities.each_slice(cocurrence).with_index do |group, idx1|
           threads = []
-          group.each do |entity|
+          group.each_with_index do |entity, idx2|
+            index = idx1 * cocurrence + idx2
             threads << Thread.new(entity) do |ent|
-              results[ent] = block.call(ent)
+              results[index] = block.call(ent)
             end
           end
-          threads.each{|thread| thread.join}
+          threads.each(&:join)
         end
-        entities.map{|ent| results[ent]}
+        results
       else
-        entities.map{|ent| block.call(ent) }
+        entities.map { |ent| block.call(ent) }
       end
     end
   end
-end
+end

data/lib/digger/model.rb

@@ -1,16 +1,19 @@
 
 module Digger
   class Model
-    @@digger_config = {'pattern'=>{}, 'index'=>{}}
+    @@digger_config = {
+      'pattern' => {},
+      'index' => {}
+    }
 
     class << self
       # patterns
       def pattern_config
-        @@digger_config['pattern'][self.name] ||= {}
+        @@digger_config['pattern'][name] ||= {}
       end
 
       Pattern::TYPES.each do |method|
-        define_method method, ->(pairs, &block){
+        define_method method, -> (pairs, &block) {
           pairs.each_pair do |key, value|
             pattern_config[key] = Pattern.new(type: method, value: value, block: block)
           end
@@ -18,21 +21,22 @@ module Digger
       end
 
       def validate_presence(*keys)
-        keys_all = pattern_config.keys
-        raise "Pattern keys #{(keys - keys_all).join(', ')} should be present" unless keys.all?{|k| keys_all.include?(k) }
+        is_all = pattern_config.keys.all? { |k| keys.include?(k) }
+        raise "Pattern keys #{(keys - keys_all).join(', ')} should be present" unless is_all
       end
 
       def validate_includeness(*keys)
-        raise "Pattern keys #{(pattern_config.keys - keys).join(', ')} should not be included" unless pattern_config.keys.all?{|k| keys.include?(k)}
+        is_all = pattern_config.keys.all? { |k| keys.include?(k) }
+        raise "Pattern keys #{(pattern_config.keys - keys).join(', ')} should not be included" if is_all
       end
 
       # index page
       def index_config
-        @@digger_config['index'][self.name]
+        @@digger_config['index'][name]
       end
 
       def index_page(pattern, *args)
-         @@digger_config['index'][self.name] = Index.new(pattern, args)
+        @@digger_config['index'][name] = Index.new(pattern, args)
       end
 
       def index_page?
@@ -55,13 +59,15 @@ module Digger
     end
 
     def dig_urls(urls, cocurrence = 1, opts = {})
-      Index.batch(urls, cocurrence){|url| dig_url(url, opts) }
+      Index.batch(urls, cocurrence) { |url| dig_url(url, opts) }
     end
 
     def dig(cocurrence = 1)
       if self.class.index_page?
-        self.class.index_config.process(cocurrence){|url| dig_url(url) }
+        self.class.index_config.process(cocurrence) do |url|
+          dig_url(url)
+        end
       end
     end
   end
-end
+end

data/lib/digger/page.rb

@@ -3,6 +3,8 @@ require 'json'
 require 'ostruct'
 require 'set'
 require 'kconv'
+require 'uri'
+require 'http/cookie'
 
 # https://github.com/taganaka/polipus/blob/master/lib/polipus/page.rb
 module Digger
@@ -27,16 +29,12 @@ module Digger
     # OpenStruct it holds users defined data
     attr_accessor :user_data
 
-    attr_accessor :aliases
-
-    attr_accessor :domain_aliases
+    attr_accessor :aliases, :domain_aliases, :fetched_at
 
     # Whether the current page should be stored
     # Default: true
     attr_accessor :storable
 
-    attr_accessor :fetched_at
-
     #
     # Create a new page
     #
@@ -60,7 +58,7 @@ module Digger
     end
 
     def title
-      doc.title if doc
+      doc&.title
     end
 
     #
@@ -74,6 +72,7 @@ module Digger
         doc.search('//a[@href]').each do |a|
           u = a['href']
           next if u.nil? || u.empty?
+
           abs = to_absolute(u) rescue next
           @links << abs if abs && in_domain?(abs)
         end
@@ -100,7 +99,11 @@ module Digger
     end
 
     def jsonp
-      @jsonp ||= JSON.parse body.match(/^[^\(]+?\((.+)\)[^\)]*$/)[1]
+      @jsonp ||= JSON.parse body.match(/^[^(]+?\((.+)\)[^)]*$/)[1]
+    end
+
+    def cookies
+      @cookies ||= (headers['set-cookie'] || []).flat_map { |c| ::HTTP::Cookie.parse(c, url) }
     end
 
     #
@@ -162,7 +165,7 @@ module Digger
     # returns +false+ otherwise.
     #
     def not_found?
-      404 == @code
+      @code == 404
     end
 
     #
@@ -176,6 +179,7 @@ module Digger
               end unless @base
 
       return nil if @base && @base.to_s.empty?
+
       @base
     end
 
@@ -186,16 +190,7 @@ module Digger
     def to_absolute(link)
       return nil if link.nil?
 
-      # link = link.to_s.encode('utf-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '')
-
-      # remove anchor
-      link =
-        begin
-          URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/, '')))
-        rescue URI::Error
-          return nil
-        end
-
+      link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#[\w]*$/, '')
       relative = begin
                    URI(link)
                  rescue URI::Error
@@ -253,6 +248,7 @@ module Digger
 
     def expired?(ttl)
       return false if fetched_at.nil?
+
       (Time.now.to_i - ttl) > fetched_at
     end
 
@@ -282,4 +278,4 @@ module Digger
       from_hash hash
     end
   end
-end
+end

data/lib/digger/pattern.rb

@@ -1,111 +1,120 @@
 require 'nokogiri'
 
 module Digger
+  # Extractor patterns definition
   class Pattern
     attr_accessor :type, :value, :block
 
     def initialize(hash = {})
-      hash.each_pair{|key, value| send("#{key}=", value) if %w{type value block}.include?(key.to_s)}
+      hash.each_pair { |key, value| send("#{key}=", value) if %w[type value block].include?(key.to_s)}
     end
 
-    def safe_block
-      block && begin
-        if block.respond_to?(:call)
-          block
-        elsif block.strip == '' # 
-          nil
-        else
-          proc{ $SAFE = 2; eval block }.call
-        end
-      rescue StandardError
-        nil
+    def safe_block(&default_block)
+      if block.nil? || (block.is_a?(String) && block.strip.empty?)
+        default_block || ->(v) { v }
+      elsif block.respond_to?(:call)
+        block
+      else
+        proc {
+          $SAFE = 2
+          eval block
+        }.call
       end
     end
 
     def self.wrap(hash)
-      Hash[hash.map{|key, value| [key, value.is_a?(Pattern) ? value : Pattern.new(value)]}]
+      hash.transform_values { |value| value.is_a?(Pattern) ? value : Pattern.new(value) }
     end
 
     MATCH_MAX = 3
 
-    TYPES_REGEXP = 0.upto(MATCH_MAX).map{|i| "match_#{i}"} + %w{match_many}
-    TYPES_CSS = %w{css_one css_many}
-    TYPES_JSON = %w{json jsonp}
-    
-    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON
+    TYPES_REGEXP = 0.upto(MATCH_MAX).map { |i| "match_#{i}" } + %w[match_many]
+    TYPES_CSS = %w[css_one css_many].freeze
+    TYPES_JSON = %w[json jsonp].freeze
+
+    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON + ['cookie']
+
+    def match_page(page)
+      return unless page.success?
 
-    def match_page(page, &callback)
-      blk = callback || safe_block
       if TYPES_REGEXP.include?(type) # regular expression
-        blk ||= ->(text){ text.strip }
-        # content is String
-        if type == 'match_many'
-          match = page.body.gsub(value).to_a
-        else
-          index = TYPES_REGEXP.index(type)
-          matches = page.body.match(value)
-          match = matches.nil? ? nil : matches[index]
-        end
+        regexp_match(page.body)
       elsif TYPES_CSS.include?(type) # css expression
-        blk ||= ->(node){ node.content.strip }
-        # content is Nokogiri::HTML::Document
-        if type == 'css_one'
-          match = page.doc.css(value).first
-        else
-          match = page.doc.css(value)
-        end
+        css_match(page.doc)
       elsif TYPES_JSON.include?(type)
-        json = page.send(type)
-        match = json_fetch(json, value)
+        json_match(page)
+      else
+        cookie_get(page.cookies)
+      end
+    end
+
+    def cookie_get(cookies)
+      cookie = cookies.find { |c| c.name == value }&.value
+      safe_block.call(cookie)
+    end
+
+    def json_match(page)
+      json = page.send(type)
+      keys = json_index_keys(value)
+      match = json_fetch(json, keys)
+      safe_block.call(match)
+    end
+
+    def css_match(doc)
+      block = safe_block { |node| node.content.strip }
+      # content is Nokogiri::HTML::Document
+      contents = doc.css(value)
+      if type == 'css_many'
+        contents.map { |node| block.call(node) }.uniq
+      else
+        block.call(contents.first)
       end
-      if match.nil?
-        nil
-      elsif %w{css_many match_many}.include? type
-        match.map{|node| blk.call(node) }.uniq
+    end
+
+    def regexp_match(body)
+      block = safe_block(&:strip)
+      # content is String
+      if type == 'match_many'
+        body.gsub(value).to_a.map { |node| block.call(node) }.uniq
       else
-        blk.call(match)
+        index = TYPES_REGEXP.index(type)
+        matches = body.match(value)
+        block.call(matches[index]) unless matches.nil?
       end
-    rescue
-      nil
     end
 
     def json_fetch(json, keys)
-      if keys.is_a? String
-        # parse json keys like '$.k1.k2[0]'
-        parts = keys.match(/^\$[\S]*$/)[0].scan(/(\.([\w]+)|\[([\d]+)\])/).map do |p|
-          p[1].nil? ? { index: p[2].to_i  } : { key: p[1] }
-        end
-        json_fetch(json, parts)
-      elsif keys.is_a? Array
-        if keys.length == 0
-          json
-        else
-          pt = keys.shift
-          json_fetch(json[pt[:index] || pt[:key]], keys)
-        end
+      if keys.empty?
+        json
+      else
+        pt = keys.shift
+        json_fetch(json[pt[:index] || pt[:key]], keys)
       end
     end
 
+    def json_index_keys(keys)
+      keys.to_s.match(/^\$\S*$/)[0].scan(/(\.(\w+)|\[(\d+)\])/).map do |p|
+        p[1].nil? ? { index: p[2].to_i } : { key: p[1] }
+      end
+    end
+
+    private :json_index_keys, :json_fetch
+
+    # Nokogiri node methods
     class Nokogiri::XML::Node
-      %w{one many}.each do |name|
-        define_method "inner_#{name}" do |css, &block| 
-          callback = ->(node) do
-            if node
-              (block || ->(n){n.text.strip}).call(node)
-            else
-              nil
-            end
-          end
+      %w[one many].each do |name|
+        define_method "inner_#{name}" do |css, &block|
+          callback = ->(node) { (block || ->(n) { n.text.strip }).call(node) if node }
           if name == 'one' # inner_one
             callback.call(self.css(css).first)
           else # inner_many
-            self.css(css).map{|node| callback.call(node)}
+            self.css(css).map { |node| callback.call(node) }
           end
         end
       end
       def source
         to_xml
       end
-    end # nokogiri
+    end
   end
-end
+end

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = "0.1.5"
+  VERSION = '0.1.9'.freeze
 end

data/spec/digger_spec.rb

@@ -1,15 +1,14 @@
 require 'digger'
 
 http = Digger::HTTP.new
-page = http.fetch_page('http://nan.so/')
+page = http.fetch_page('http://www.baidu.com/')
 
-pattern = Digger::Pattern.new({type: 'css_many', value: '.sites>a>span' })
+pattern = Digger::Pattern.new({ type: 'css_many', value: '#s-top-left>a' })
 
 class Item < Digger::Model
-  css_many sites: '.sites>a>span'
-  css_one logo: '.logo'
+  css_many sites: '#s-top-left>a'
   validate_presence :sites
-  validate_includeness :sites, :logo
+  validate_includeness :sites
 end
 
 describe Digger do
@@ -19,12 +18,12 @@ describe Digger do
 
   it "pattern should match content" do
     sites = pattern.match_page(page)
-    expect(sites.include?('百度网盘')).to eq(true)
+    expect(sites.include?('新闻')).to eq(true)
   end
 
   it "model should dig content" do
     item = Item.new.match_page(page)
-    expect(item[:sites].include?('读远')).to be(true)
+    expect(item[:sites].include?('新闻')).to be(true)
   end
 
   it "validation support" do

data/spec/index_spec.rb

@@ -0,0 +1,28 @@
+require 'digger'
+
+describe Digger::Index do
+  it 'batch digger' do
+    list = [1, 2, 3, 4, 5, 6, 7, 8]
+    pt = Digger::Index.batch(list, 3) do |num|
+      sleep(rand(1..3))
+      "##{num}"
+    end
+    expect(pt.join).to eq(list.map { |num| "##{num}" }.join)
+  end
+
+  it 'slow down' do
+    list = [1, 2, 3, 4]
+    conf = {
+      sleep_range_seconds: 1...2,
+      fail_unit_seconds: 1,
+      fail_max_cnt: 2,
+      when_fail: ->(_, e, nth) { puts "#{nth}: #{e.message}" }
+    }
+    pt = Digger::Index.slow_down(list, conf) do |num|
+      raise 'error' if num == 3
+      num
+    end
+    p pt
+    expect(pt.size).to eq(2)
+  end
+end

data/spec/page_spec.rb

@@ -1,5 +1,7 @@
 require 'digger'
 require 'json'
+require 'uri'
+require 'cgi'
 
 describe Digger::Page do
   it 'page json' do
@@ -11,4 +13,15 @@ describe Digger::Page do
     expect(j1.json['b'][0]).to eq(1)
     expect(j2.jsonp['b'][1]).to eq(2)
   end
-end
+
+  it 'fetch baidu' do
+    http = Digger::HTTP.new
+    page = http.fetch_page('http://baidu.com/')
+    expect(page.code).to eq(200)
+  end
+
+  # it 'page uri' do
+  #   link = 'https://www.baidu.com/s?wd=%E5%93%88%E5%93%88#hello'
+  #   link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#\w*$/, '')
+  # end
+end

data/spec/pattern_spec.rb

@@ -2,14 +2,19 @@ require 'digger'
 require 'json'
 
 describe Digger::Pattern do
-  it 'json fetch' do
-    json = JSON.parse('{"a":1,"b":[1,2,3]}')
-    pt = Digger::Pattern.new
-    expect(pt.json_fetch(json, '$')['a']).to eq(1)
-    expect(pt.json_fetch(json, '$.a')).to eq(1)
-    expect(pt.json_fetch(json, '$.b').length).to eq(3)
-    expect(pt.json_fetch(json, '$.b[2]')).to eq(3)
-  end
-
+  # it 'json fetch' do
+  #   json = JSON.parse('[{"a":1,"b":[1,2,3]}]')
+  #   pt = Digger::Pattern.new
+  #   expect(pt.json_fetch(json, '$[0]')['a']).to eq(1)
+  #   expect(pt.json_fetch(json, '$[0].a')).to eq(1)
+  #   expect(pt.json_fetch(json, '$[0].b').length).to eq(3)
+  #   expect(pt.json_fetch(json, '$[0].b[2]')).to eq(3)
+  # end
 
-end
+  it 'parse cookoe' do
+    page = Digger::HTTP.new.fetch_page('https://xueqiu.com/')
+    pt = Digger::Pattern.new({ type: 'cookie', value: 'xq_a_token', block: ->(v) { "!!#{v}" } })
+    result = pt.match_page(page)
+    expect(result.length).to eq(42)
+  end
+end

metadata

@@ -1,71 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.9
 platform: ruby
 authors:
 - binz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-25 00:00:00.000000000 Z
+date: 2021-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: http-cookie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: http-cookie
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
 description: ''
 email:
 - xinkiang@gmail.com
@@ -87,6 +87,7 @@ files:
 - lib/digger/pattern.rb
 - lib/digger/version.rb
 - spec/digger_spec.rb
+- spec/index_spec.rb
 - spec/page_spec.rb
 - spec/pattern_spec.rb
 - spec/validate_spec.rb
@@ -115,6 +116,7 @@ specification_version: 4
 summary: Dig need stractual infomation from web page.
 test_files:
 - spec/digger_spec.rb
+- spec/index_spec.rb
 - spec/page_spec.rb
 - spec/pattern_spec.rb
 - spec/validate_spec.rb