digger 0.1.2 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: ed6cbdd1efd570809d15e3fec518c6f4b2a7e2a0
-  data.tar.gz: 5c59c6ab0243aca0c6f130a3073fc0433dfe8211
+SHA256:
+  metadata.gz: f3e89f179fa868ecd2879180d1fbfbf03ba0ebee3731b9c8b4741d22663ff4aa
+  data.tar.gz: 1b27e4a1446e9835203bf5497aeebc3bc4ab58998a0fc443eeaaf7e7ec86c2c7
 SHA512:
-  metadata.gz: d7c57067e3a91f488791968f1752ebf6f89aaab10200508d89d02cf7f19a4d6c6de8ebf29f826db6ae70cd345a604f65dd4c9264b1f1575ff2e81c64b572712b
-  data.tar.gz: 870cf5d2a98d005a7140fe400c79afdef9c37eb29778fe9efb406061e47f85624645d180449001c3e4827b98080b41bc978d6cd2039f55e81a8565e8c0f36153
+  metadata.gz: 5671e5d2484ca744e5c75f97beeb473e1970291fc094c0274714af27cc847ef47bcf3b1b81534e6f299f35648fdf3aabec9d90777e4b2fbe49dc2629c048f610
+  data.tar.gz: 496534bb394d17792dc7173759b83c0dd62b8569a241b2a6a57e60a14ea99ddafd7e030fe16480c5782ec319f6dc1d3563919f7202026fb008522dd26cae6c01

data/digger.gemspec

@@ -18,8 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "bundler", "~> 2.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
 
   spec.add_runtime_dependency 'nokogiri', '~> 1.6'
   spec.add_runtime_dependency 'http-cookie', '~> 1.0'

data/lib/digger/http.rb

@@ -49,7 +49,7 @@ module Digger
       url = URI(url)
       pages = []
       get(url, referer) do |response, code, location, redirect_to, response_time|
-        handle_compression response
+        handle_compression response if handle_compression?
         pages << Page.new(location, body: response.body,
                                     code: code,
                                     headers: response.to_hash,
@@ -70,6 +70,13 @@ module Digger
       [Page.new(url, error: e, referer: referer, depth: depth)]
     end
 
+    #
+    # Accept response compression, may bring encoding error if true
+    # 
+    def handle_compression?
+      @opts[:handle_compression]
+    end
+
     #
     # The maximum number of redirects to follow
     #
@@ -185,7 +192,7 @@ module Digger
       opts['User-Agent'] = user_agent if user_agent
       opts['Referer'] = referer.to_s if referer
       opts['Cookie']  = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
-      opts['Accept-Encoding'] = 'gzip,deflate'
+      opts['Accept-Encoding'] = 'gzip,deflate' if handle_compression?
 
       retries = 0
       begin

data/lib/digger/model.rb

@@ -17,6 +17,15 @@ module Digger
         }
       end
 
+      def validate_presence(*keys)
+        keys_all = pattern_config.keys
+        raise "Pattern keys #{(keys - keys_all).join(', ')} should be present" unless keys.all?{|k| keys_all.include?(k) }
+      end
+
+      def validate_includeness(*keys)
+        raise "Pattern keys #{(pattern_config.keys - keys).join(', ')} should not be included" unless pattern_config.keys.all?{|k| keys.include?(k)}
+      end
+
       # index page
       def index_config
         @@digger_config['index'][self.name]

data/lib/digger/page.rb

@@ -3,6 +3,7 @@ require 'json'
 require 'ostruct'
 require 'set'
 require 'kconv'
+require 'uri'
 
 # https://github.com/taganaka/polipus/blob/master/lib/polipus/page.rb
 module Digger
@@ -95,7 +96,13 @@ module Digger
       end
     end
 
+    def json
+      @json ||= JSON.parse body
+    end
 
+    def jsonp
+      @jsonp ||= JSON.parse body.match(/^[^\(]+?\((.+)\)[^\)]*$/)[1]
+    end
 
     #
     # Discard links, a next call of page.links will return an empty array
@@ -180,16 +187,7 @@ module Digger
     def to_absolute(link)
       return nil if link.nil?
 
-      # link = link.to_s.encode('utf-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '')
-
-      # remove anchor
-      link =
-        begin
-          URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/, '')))
-        rescue URI::Error
-          return nil
-        end
-
+      link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#[\w]*$/, '')
       relative = begin
                    URI(link)
                  rescue URI::Error

data/lib/digger/pattern.rb

@@ -27,33 +27,36 @@ module Digger
     end
 
     MATCH_MAX = 3
-    
-    TYPES = 0.upto(MATCH_MAX).map{|i| "match_#{i}"} + %w{match_many css_one css_many}
 
-    def regexp?
-      TYPES.index(type) <= MATCH_MAX + 1 # match_many in addition
-    end
+    TYPES_REGEXP = 0.upto(MATCH_MAX).map{|i| "match_#{i}"} + %w{match_many}
+    TYPES_CSS = %w{css_one css_many}
+    TYPES_JSON = %w{json jsonp}
+    
+    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON
 
     def match_page(page, &callback)
       blk = callback || safe_block
-      if regexp? # regular expression
-        index = TYPES.index(type)
-        blk ||= ->(text){text.strip}
+      if TYPES_REGEXP.include?(type) # regular expression
+        blk ||= ->(text){ text.strip }
         # content is String
         if type == 'match_many'
           match = page.body.gsub(value).to_a
         else
+          index = TYPES_REGEXP.index(type)
           matches = page.body.match(value)
           match = matches.nil? ? nil : matches[index]
         end
-      else # css expression
-        blk ||= ->(node){node.content.strip}
+      elsif TYPES_CSS.include?(type) # css expression
+        blk ||= ->(node){ node.content.strip }
         # content is Nokogiri::HTML::Document
         if type == 'css_one'
           match = page.doc.css(value).first
-        elsif type == 'css_many' # css_many
+        else
           match = page.doc.css(value)
         end
+      elsif TYPES_JSON.include?(type)
+        json = page.send(type)
+        match = json_fetch(json, value)
       end
       if match.nil?
         nil
@@ -66,6 +69,23 @@ module Digger
       nil
     end
 
+    def json_fetch(json, keys)
+      if keys.is_a? String
+        # parse json keys like '$.k1.k2[0]'
+        parts = keys.match(/^\$[\S]*$/)[0].scan(/(\.([\w]+)|\[([\d]+)\])/).map do |p|
+          p[1].nil? ? { index: p[2].to_i  } : { key: p[1] }
+        end
+        json_fetch(json, parts)
+      elsif keys.is_a? Array
+        if keys.length == 0
+          json
+        else
+          pt = keys.shift
+          json_fetch(json[pt[:index] || pt[:key]], keys)
+        end
+      end
+    end
+
     class Nokogiri::XML::Node
       %w{one many}.each do |name|
         define_method "inner_#{name}" do |css, &block| 

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = "0.1.2"
+  VERSION = "0.1.6"
 end

data/spec/digger_spec.rb

@@ -1,12 +1,14 @@
 require 'digger'
 
 http = Digger::HTTP.new
-page = http.fetch_page('http://nan.so/')
+page = http.fetch_page('http://www.baidu.com/')
 
-pattern = Digger::Pattern.new({type: 'css_many', value: '.sites>a>span' })
+pattern = Digger::Pattern.new({ type: 'css_many', value: '#s-top-left>a' })
 
 class Item < Digger::Model
-  css_many sites: '.sites>a>span'
+  css_many sites: '#s-top-left>a'
+  validate_presence :sites
+  validate_includeness :sites
 end
 
 describe Digger do
@@ -16,12 +18,15 @@ describe Digger do
 
   it "pattern should match content" do
     sites = pattern.match_page(page)
-    expect(sites.include?('百度网盘')).to eq(true)
+    expect(sites.include?('新闻')).to eq(true)
   end
 
   it "model should dig content" do
     item = Item.new.match_page(page)
-    expect(item[:sites].include?('读远')).to be(true)
+    expect(item[:sites].include?('新闻')).to be(true)
+  end
+
+  it "validation support" do
   end
 
   it "index multiple threading" do

data/spec/page_spec.rb

@@ -0,0 +1,27 @@
+require 'digger'
+require 'json'
+require 'uri'
+
+describe Digger::Page do
+  it 'page json' do
+    json_str = '{"a":1,"b":[1,2,3]}'
+    j1 = Digger::Page.new('', body: json_str)
+    j2 = Digger::Page.new('', body: "hello(#{json_str});")
+    expect(j1.json['a']).to eq(1)
+    expect(j2.jsonp['a']).to eq(1)
+    expect(j1.json['b'][0]).to eq(1)
+    expect(j2.jsonp['b'][1]).to eq(2)
+  end
+
+  it 'fetch baidu' do
+    http = Digger::HTTP.new
+    page = http.fetch_page('http://www.baidu.com/')
+    expect(page.code).to eq(200)
+  end
+
+  it 'page uri' do
+    link ='https://www.baidu.com/s?wd=%E5%93%88%E5%93%88#hello'
+    link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#[\w]*$/, '')
+    p link
+  end
+end

data/spec/pattern_spec.rb

@@ -0,0 +1,15 @@
+require 'digger'
+require 'json'
+
+describe Digger::Pattern do
+  it 'json fetch' do
+    json = JSON.parse('{"a":1,"b":[1,2,3]}')
+    pt = Digger::Pattern.new
+    expect(pt.json_fetch(json, '$')['a']).to eq(1)
+    expect(pt.json_fetch(json, '$.a')).to eq(1)
+    expect(pt.json_fetch(json, '$.b').length).to eq(3)
+    expect(pt.json_fetch(json, '$.b[2]')).to eq(3)
+  end
+
+
+end

data/spec/validate_spec.rb

@@ -0,0 +1,10 @@
+require 'digger'
+
+class Item < Digger::Model
+  # css_many sites: '.sites>a>span'
+  css_one logo: '.logo'
+  css_one title: '.title'
+
+  validate_presence :sites
+  validate_includeness :sites, :logo, :title
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.6
 platform: ruby
 authors:
 - binz
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-17 00:00:00.000000000 Z
+date: 2021-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
@@ -87,11 +87,14 @@ files:
 - lib/digger/pattern.rb
 - lib/digger/version.rb
 - spec/digger_spec.rb
+- spec/page_spec.rb
+- spec/pattern_spec.rb
+- spec/validate_spec.rb
 homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -106,10 +109,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.2.32
+signing_key:
 specification_version: 4
 summary: Dig need stractual infomation from web page.
 test_files:
 - spec/digger_spec.rb
+- spec/page_spec.rb
+- spec/pattern_spec.rb
+- spec/validate_spec.rb