digger 0.1.1 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 70e98c345c7da377cc9f84f68cb6b7f702f7b42b
-  data.tar.gz: 5f3416916d1b726810075e5ea744a39ac3dd4487
+SHA256:
+  metadata.gz: 5c4a94163a25d4b53ad5b477040f69b8fccca026adc313f8f61759317c1bf198
+  data.tar.gz: 307b443277c16708103c172e5fb4ef4d833f7d2631a7f85779570a3cbeac8925
 SHA512:
-  metadata.gz: 5750dde57359a775d378e5dc84c75062bb3494af58f97708e84692cd5e6c9840719c40d6f0602244b542cc7e14d275dedd116b57119174366b05b04249674b99
-  data.tar.gz: 2272ee6dd594a818b69cd0e4cb5eaba1f71954683d421f3f8654b9de1bc18a03e1f70cdf99a457b9d29f9add2736858bb2ffe22d31fa97073693bed4d8f64d44
+  metadata.gz: 9da40123fd09615d0c69ca5104d1141b82981813ef8d175bc567a0f35e8f7dd868ce235bc0f265308e2133710f63c3ce8325d6b528ae060eccb77904c12e3139
+  data.tar.gz: c05be67df6db25345acfdc3615690c0467029dd40ffde262863923b6f0786696a16fd547277ec4d3764ed39b399174dd28e10ca7eb31ffb55f773e08b04f2986

data/digger.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 10.0"
 
   spec.add_runtime_dependency 'nokogiri', '~> 1.6'

data/lib/digger/http.rb

@@ -49,7 +49,7 @@ module Digger
       url = URI(url)
       pages = []
       get(url, referer) do |response, code, location, redirect_to, response_time|
-        handle_compression response
+        handle_compression response if handle_compression?
         pages << Page.new(location, body: response.body,
                                     code: code,
                                     headers: response.to_hash,
@@ -70,6 +70,13 @@ module Digger
       [Page.new(url, error: e, referer: referer, depth: depth)]
     end
 
+    #
+    # Accept response compression, may bring encoding error if true
+    # 
+    def handle_compression?
+      @opts[:handle_compression]
+    end
+
     #
     # The maximum number of redirects to follow
     #
@@ -185,7 +192,7 @@ module Digger
       opts['User-Agent'] = user_agent if user_agent
       opts['Referer'] = referer.to_s if referer
       opts['Cookie']  = ::HTTP::Cookie.cookie_value(cookie_jar.cookies(url)) if accept_cookies?
-      opts['Accept-Encoding'] = 'gzip,deflate'
+      opts['Accept-Encoding'] = 'gzip,deflate' if handle_compression?
 
       retries = 0
       begin

data/lib/digger/model.rb

@@ -17,6 +17,15 @@ module Digger
         }
       end
 
+      def validate_presence(*keys)
+        keys_all = pattern_config.keys
+        raise "Pattern keys #{(keys - keys_all).join(', ')} should be present" unless keys.all?{|k| keys_all.include?(k) }
+      end
+
+      def validate_includeness(*keys)
+        raise "Pattern keys #{(pattern_config.keys - keys).join(', ')} should not be included" unless pattern_config.keys.all?{|k| keys.include?(k)}
+      end
+
       # index page
       def index_config
         @@digger_config['index'][self.name]
@@ -39,14 +48,14 @@ module Digger
       result
     end
 
-    def dig_url(url)
-      client = Digger::HTTP.new
+    def dig_url(url, opts = {})
+      client = Digger::HTTP.new(opts)
       page = client.fetch_page(url)
       match_page(page)
     end
 
-    def dig_urls(urls, cocurrence = 1)
-      Index.batch(urls, cocurrence){|url| dig_url(url) }
+    def dig_urls(urls, cocurrence = 1, opts = {})
+      Index.batch(urls, cocurrence){|url| dig_url(url, opts) }
     end
 
     def dig(cocurrence = 1)

data/lib/digger/page.rb

@@ -95,7 +95,13 @@ module Digger
       end
     end
 
+    def json
+      @json ||= JSON.parse body
+    end
 
+    def jsonp
+      @jsonp ||= JSON.parse body.match(/^[^\(]+?\((.+)\)[^\)]*$/)[1]
+    end
 
     #
     # Discard links, a next call of page.links will return an empty array

data/lib/digger/pattern.rb

@@ -27,33 +27,36 @@ module Digger
     end
 
     MATCH_MAX = 3
-    
-    TYPES = 0.upto(MATCH_MAX).map{|i| "match_#{i}"} + %w{match_many css_one css_many}
 
-    def regexp?
-      TYPES.index(type) <= MATCH_MAX + 1 # match_many in addition
-    end
+    TYPES_REGEXP = 0.upto(MATCH_MAX).map{|i| "match_#{i}"} + %w{match_many}
+    TYPES_CSS = %w{css_one css_many}
+    TYPES_JSON = %w{json jsonp}
+    
+    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON
 
     def match_page(page, &callback)
       blk = callback || safe_block
-      if regexp? # regular expression
-        index = TYPES.index(type)
-        blk ||= ->(text){text.strip}
+      if TYPES_REGEXP.include?(type) # regular expression
+        blk ||= ->(text){ text.strip }
         # content is String
         if type == 'match_many'
           match = page.body.gsub(value).to_a
         else
+          index = TYPES_REGEXP.index(type)
           matches = page.body.match(value)
           match = matches.nil? ? nil : matches[index]
         end
-      else # css expression
-        blk ||= ->(node){node.content.strip}
+      elsif TYPES_CSS.include?(type) # css expression
+        blk ||= ->(node){ node.content.strip }
         # content is Nokogiri::HTML::Document
         if type == 'css_one'
           match = page.doc.css(value).first
-        elsif type == 'css_many' # css_many
+        else
           match = page.doc.css(value)
         end
+      elsif TYPES_JSON.include?(type)
+        json = page.send(type)
+        match = json_fetch(json, value)
       end
       if match.nil?
         nil
@@ -66,6 +69,23 @@ module Digger
       nil
     end
 
+    def json_fetch(json, keys)
+      if keys.is_a? String
+        # parse json keys like '$.k1.k2[0]'
+        parts = keys.match(/^\$[\S]*$/)[0].scan(/(\.([\w]+)|\[([\d]+)\])/).map do |p|
+          p[1].nil? ? { index: p[2].to_i  } : { key: p[1] }
+        end
+        json_fetch(json, parts)
+      elsif keys.is_a? Array
+        if keys.length == 0
+          json
+        else
+          pt = keys.shift
+          json_fetch(json[pt[:index] || pt[:key]], keys)
+        end
+      end
+    end
+
     class Nokogiri::XML::Node
       %w{one many}.each do |name|
         define_method "inner_#{name}" do |css, &block| 

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = "0.1.1"
+  VERSION = "0.1.5"
 end

data/spec/digger_spec.rb

@@ -7,6 +7,9 @@ pattern = Digger::Pattern.new({type: 'css_many', value: '.sites>a>span' })
 
 class Item < Digger::Model
   css_many sites: '.sites>a>span'
+  css_one logo: '.logo'
+  validate_presence :sites
+  validate_includeness :sites, :logo
 end
 
 describe Digger do
@@ -24,6 +27,9 @@ describe Digger do
     expect(item[:sites].include?('读远')).to be(true)
   end
 
+  it "validation support" do
+  end
+
   it "index multiple threading" do
     
   end

data/spec/page_spec.rb

@@ -0,0 +1,14 @@
+require 'digger'
+require 'json'
+
+describe Digger::Page do
+  it 'page json' do
+    json_str = '{"a":1,"b":[1,2,3]}'
+    j1 = Digger::Page.new('', body: json_str)
+    j2 = Digger::Page.new('', body: "hello(#{json_str});")
+    expect(j1.json['a']).to eq(1)
+    expect(j2.jsonp['a']).to eq(1)
+    expect(j1.json['b'][0]).to eq(1)
+    expect(j2.jsonp['b'][1]).to eq(2)
+  end
+end

data/spec/pattern_spec.rb

@@ -0,0 +1,15 @@
+require 'digger'
+require 'json'
+
+describe Digger::Pattern do
+  it 'json fetch' do
+    json = JSON.parse('{"a":1,"b":[1,2,3]}')
+    pt = Digger::Pattern.new
+    expect(pt.json_fetch(json, '$')['a']).to eq(1)
+    expect(pt.json_fetch(json, '$.a')).to eq(1)
+    expect(pt.json_fetch(json, '$.b').length).to eq(3)
+    expect(pt.json_fetch(json, '$.b[2]')).to eq(3)
+  end
+
+
+end

data/spec/validate_spec.rb

@@ -0,0 +1,10 @@
+require 'digger'
+
+class Item < Digger::Model
+  # css_many sites: '.sites>a>span'
+  css_one logo: '.logo'
+  css_one title: '.title'
+
+  validate_presence :sites
+  validate_includeness :sites, :logo, :title
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.5
 platform: ruby
 authors:
 - binz
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-02 00:00:00.000000000 Z
+date: 2021-12-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -87,11 +87,14 @@ files:
 - lib/digger/pattern.rb
 - lib/digger/version.rb
 - spec/digger_spec.rb
+- spec/page_spec.rb
+- spec/pattern_spec.rb
+- spec/validate_spec.rb
 homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -106,10 +109,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.2.32
+signing_key:
 specification_version: 4
 summary: Dig need stractual infomation from web page.
 test_files:
 - spec/digger_spec.rb
+- spec/page_spec.rb
+- spec/pattern_spec.rb
+- spec/validate_spec.rb