digger 0.1.7 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f506fd615df8b9d732d6b67bedc72644df26dc3f5725cfe5dba10c1098bae0b
-  data.tar.gz: 0e9afcb19ba0be5ce4a90787d54c3b43d58100bac3ec45bf5ef93781a60b6eb1
+  metadata.gz: b036f2d202aa8360cf3b07822168fb8e3e3e084d4b2f5ccb5eca026dd4c47981
+  data.tar.gz: 85d13e567add73e38c25738852dc3b30e3ad1900579dca815d5aef4cb16dbca8
 SHA512:
-  metadata.gz: 8608a2ee8e06ddd846772d40dc3e417560229729b7b736eda8a7f50977a7d2c6fc523f86fe64480b5172c5295137eae7c85f945f7ca310502c54c8b90dd75e8d
-  data.tar.gz: 59fef1a13adc8f983c16428ee4d08d6ccdecea09b7583452b6ca07689727c3d6a386f5a5b767a20d117c8c4bc9775a2a6b32fd4caff1c1a5e85ee2af82e39d1b
+  metadata.gz: decb9abb96f56dc7f75ba95151b02c7415018a789583da1324ab4786317d7e0bfc272a83b47399b51672bdb30b3d90c502f25358d7438b9ee26a8d39764a58dd
+  data.tar.gz: 73f7dfd179175bcda4a24a120cad95a61093f17166e389e44b37a158ff8ee1ea927844d9b85773914f14d99473776f11752c7aa32fadfec1b8b703e584591c3f

data/digger.gemspec

@@ -1,26 +1,27 @@
 # coding: utf-8
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'digger/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "digger"
+  spec.name          = 'digger'
   spec.version       = Digger::VERSION
-  spec.authors       = ["binz"]
-  spec.email         = ["xinkiang@gmail.com"]
+  spec.authors       = ['binz']
+  spec.email         = ['xinkiang@gmail.com']
   spec.summary       = %q{Dig need stractual infomation from web page.}
   spec.description   = %q{}
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.homepage      = ''
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", ">= 12.3.3"
+  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'bundler', '~> 2.0'
 
-  spec.add_runtime_dependency 'nokogiri', '~> 1.6'
   spec.add_runtime_dependency 'http-cookie', '~> 1.0'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.6'
 end

data/lib/digger/index.rb

@@ -17,6 +17,35 @@ module Digger
       pattern.gsub('*').each_with_index { |_, i| arg[i] }
     end
 
+    def self.slow_down(entities, conf = {}, &block)
+      raise NoBlockError, 'No block given' unless block
+
+      config = {
+        sleep_range_seconds: 4...10, # 随机等待时间范围
+        fail_max_cnt: 10, # 最多失败次数
+        fail_unit_seconds: 10 * 60, # 失败等待时间
+        when_fail: ->(ent, e, failed_cnt) {}
+      }.merge(conf)
+      failed_cnt = 0
+      cursor = 0
+      result = []
+      while cursor < entities.length
+        begin
+          result << block.call(entities[cursor])
+        rescue StandardError => e
+          failed_cnt += 1
+          config[:when_fail].call(entities[cursor], e, failed_cnt)
+          break if failed_cnt >= config[:fail_max_cnt]
+
+          sleep(failed_cnt * config[:fail_unit_seconds])
+        else
+          cursor += 1
+          sleep(rand(config[:sleep_range_seconds]))
+        end
+      end
+      result
+    end
+
     def self.batch(entities, cocurrence = 1, &block)
       raise NoBlockError, 'No block given' unless block
 

data/lib/digger/page.rb

@@ -4,6 +4,7 @@ require 'ostruct'
 require 'set'
 require 'kconv'
 require 'uri'
+require 'http/cookie'
 
 # https://github.com/taganaka/polipus/blob/master/lib/polipus/page.rb
 module Digger
@@ -101,6 +102,10 @@ module Digger
       @jsonp ||= JSON.parse body.match(/^[^(]+?\((.+)\)[^)]*$/)[1]
     end
 
+    def cookies
+      @cookies ||= (headers['set-cookie'] || []).flat_map { |c| ::HTTP::Cookie.parse(c, url) }
+    end
+
     #
     # Discard links, a next call of page.links will return an empty array
     #
@@ -273,4 +278,4 @@ module Digger
       from_hash hash
     end
   end
-end
+end

data/lib/digger/pattern.rb

@@ -6,12 +6,14 @@ module Digger
     attr_accessor :type, :value, :block
 
     def initialize(hash = {})
-      hash.each_pair { |key, value| send("#{key}=", value) if %w[type value block].include?(key.to_s)}
+      hash.each_pair do |key, value|
+        send("#{key}=", value) if %w[type value block].include?(key.to_s)
+      end
     end
 
     def safe_block(&default_block)
       if block.nil? || (block.is_a?(String) && block.strip.empty?)
-        default_block
+        default_block || ->(v) { v }
       elsif block.respond_to?(:call)
         block
       else
@@ -31,30 +33,56 @@ module Digger
     TYPES_REGEXP = 0.upto(MATCH_MAX).map { |i| "match_#{i}" } + %w[match_many]
     TYPES_CSS = %w[css_one css_many].freeze
     TYPES_JSON = %w[json jsonp].freeze
+    TYPES_OTHER = %w[cookie plain lines header body].freeze
 
-    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON
+    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON + TYPES_OTHER
 
     def match_page(page)
       return unless page.success?
+
       if TYPES_REGEXP.include?(type) # regular expression
         regexp_match(page.body)
       elsif TYPES_CSS.include?(type) # css expression
         css_match(page.doc)
       elsif TYPES_JSON.include?(type)
         json_match(page)
+      elsif TYPES_OTHER.include?(type)
+        send("get_#{type}", page)
       end
     end
 
+    def get_header(page)
+      header = (page.headers[value.to_s.downcase] || []).first
+      safe_block.call(header)
+    end
+
+    def get_body(page)
+      safe_block.call(page.body)
+    end
+
+    def get_plain(page)
+      safe_block.call(page.doc.text)
+    end
+
+    def get_lines(page)
+      block = safe_block
+      page.body.split("\n").map(&:strip).filter { |line| !line.empty? }.map { |line| block.call(line) }
+    end
+
+    def get_cookie(page)
+      cookie = page.cookies.find { |c| c.name == value }&.value
+      safe_block.call(cookie)
+    end
+
     def json_match(page)
-      block = safe_block { |j| j }
       json = page.send(type)
       keys = json_index_keys(value)
       match = json_fetch(json, keys)
-      block.call(match)
+      safe_block.call(match)
     end
 
     def css_match(doc)
-      block = safe_block { |node| node.content.strip }
+      block = safe_block { |node| node&.content&.strip }
       # content is Nokogiri::HTML::Document
       contents = doc.css(value)
       if type == 'css_many'
@@ -68,7 +96,8 @@ module Digger
       block = safe_block(&:strip)
       # content is String
       if type == 'match_many'
-        body.gsub(value).to_a.map { |node| block.call(node) }.uniq
+        regexp = value.is_a?(Regexp) ? value : Regexp.new(value.to_s)
+        body.gsub(regexp).to_a.map { |node| block.call(node) }.uniq
       else
         index = TYPES_REGEXP.index(type)
         matches = body.match(value)
@@ -93,21 +122,24 @@ module Digger
 
     private :json_index_keys, :json_fetch
 
-    # Nokogiri node methods
-    class Nokogiri::XML::Node
-      %w[one many].each do |name|
-        define_method "inner_#{name}" do |css, &block|
-          callback = ->(node) { (block || ->(n) { n.text.strip }).call(node) if node }
-          if name == 'one' # inner_one
-            callback.call(self.css(css).first)
-          else # inner_many
-            self.css(css).map { |node| callback.call(node) }
-          end
-        end
+    class ::Nokogiri::XML::Node
+      def inner_one(expr, &block)
+        fn = block || ->(node) { node&.content&.strip }
+        fn.call(css(expr)&.first)
+      end
+
+      def inner_many(expr, &block)
+        fn = block || ->(node) { node&.content&.strip }
+        css(expr)&.map { |node| fn.call(node) }
       end
+
       def source
         to_xml
       end
+
+      def inner_number
+        content&.match(/\d+/).to_s.to_i
+      end
     end
   end
 end

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = '0.1.7'.freeze
+  VERSION = '0.2.1'.freeze
 end

data/spec/index_spec.rb

@@ -9,4 +9,20 @@ describe Digger::Index do
     end
     expect(pt.join).to eq(list.map { |num| "##{num}" }.join)
   end
+
+  it 'slow down' do
+    list = [1, 2, 3, 4]
+    conf = {
+      sleep_range_seconds: 1...2,
+      fail_unit_seconds: 1,
+      fail_max_cnt: 2,
+      when_fail: ->(_, e, nth) { puts "#{nth}: #{e.message}" }
+    }
+    pt = Digger::Index.slow_down(list, conf) do |num|
+      raise 'error' if num == 3
+      num
+    end
+    p pt
+    expect(pt.size).to eq(2)
+  end
 end

data/spec/page_spec.rb

@@ -1,6 +1,7 @@
 require 'digger'
 require 'json'
 require 'uri'
+require 'cgi'
 
 describe Digger::Page do
   it 'page json' do
@@ -15,13 +16,12 @@ describe Digger::Page do
 
   it 'fetch baidu' do
     http = Digger::HTTP.new
-    page = http.fetch_page('http://www.baidu.com/')
+    page = http.fetch_page('http://baidu.com/')
     expect(page.code).to eq(200)
   end
 
-  it 'page uri' do
-    link ='https://www.baidu.com/s?wd=%E5%93%88%E5%93%88#hello'
-    link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#[\w]*$/, '')
-    p link
-  end
-end
+  # it 'page uri' do
+  #   link = 'https://www.baidu.com/s?wd=%E5%93%88%E5%93%88#hello'
+  #   link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#\w*$/, '')
+  # end
+end

data/spec/pattern_spec.rb

@@ -2,14 +2,26 @@ require 'digger'
 require 'json'
 
 describe Digger::Pattern do
-  it 'json fetch' do
-    json = JSON.parse('[{"a":1,"b":[1,2,3]}]')
-    pt = Digger::Pattern.new
-    expect(pt.json_fetch(json, '$[0]')['a']).to eq(1)
-    expect(pt.json_fetch(json, '$[0].a')).to eq(1)
-    expect(pt.json_fetch(json, '$[0].b').length).to eq(3)
-    expect(pt.json_fetch(json, '$[0].b[2]')).to eq(3)
-  end
-
+  # it 'json fetch' do
+  #   json = JSON.parse('[{"a":1,"b":[1,2,3]}]')
+  #   pt = Digger::Pattern.new
+  #   expect(pt.json_fetch(json, '$[0]')['a']).to eq(1)
+  #   expect(pt.json_fetch(json, '$[0].a')).to eq(1)
+  #   expect(pt.json_fetch(json, '$[0].b').length).to eq(3)
+  #   expect(pt.json_fetch(json, '$[0].b[2]')).to eq(3)
+  # end
 
-end
+  it 'parse cookie & others' do
+    page = Digger::HTTP.new.fetch_page('https://xueqiu.com/')
+    p1 = Digger::Pattern.new({ type: 'cookie', value: 'xq_a_token', block: ->(v) { "!!#{v}" } })
+    # cookie
+    result = p1.match_page(page)
+    expect(result.length).to eq(42)
+    # header
+    p2 = Digger::Pattern.new({ type: 'header', value: 'transfer-encoding' })
+    expect(p2.match_page(page)).to eq('chunked')
+    # get_plain
+    p3 = Digger::Pattern.new({ type: 'plain' })
+    expect(p3.match_page(page).length).to be > 100
+  end
+end

metadata

@@ -1,71 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.1
 platform: ruby
 authors:
 - binz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-26 00:00:00.000000000 Z
+date: 2022-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: http-cookie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: http-cookie
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
 description: ''
 email:
 - xinkiang@gmail.com