digger 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f506fd615df8b9d732d6b67bedc72644df26dc3f5725cfe5dba10c1098bae0b
-  data.tar.gz: 0e9afcb19ba0be5ce4a90787d54c3b43d58100bac3ec45bf5ef93781a60b6eb1
+  metadata.gz: 19e59bc2161a078d80d00adf538a7c33891a53f9beeb453748eec7e0810c5b65
+  data.tar.gz: da1d93c663b42a6e0b7be2f136bea3b0f3a86c3c36c57a72347c70e7e0538508
 SHA512:
-  metadata.gz: 8608a2ee8e06ddd846772d40dc3e417560229729b7b736eda8a7f50977a7d2c6fc523f86fe64480b5172c5295137eae7c85f945f7ca310502c54c8b90dd75e8d
-  data.tar.gz: 59fef1a13adc8f983c16428ee4d08d6ccdecea09b7583452b6ca07689727c3d6a386f5a5b767a20d117c8c4bc9775a2a6b32fd4caff1c1a5e85ee2af82e39d1b
+  metadata.gz: d13b06c8491f9cda42f8a8fd4fa9547aa6a95d075efbf52c789760adaa66d0c2bde24f810e23e4713df4362ae82cb82b0d61996386caaad2c685eb22f1a375db
+  data.tar.gz: a63a5cfe70b154b446dcbd0d2937d94a33fb033277c0d17aa1500a9ccd86dc8791e3cc40e8487ecc1c609eec304b91fb4a29b75b89a74b8efe24e29c49957d4e

data/digger.gemspec

@@ -1,26 +1,27 @@
 # coding: utf-8
+
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'digger/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "digger"
+  spec.name          = 'digger'
   spec.version       = Digger::VERSION
-  spec.authors       = ["binz"]
-  spec.email         = ["xinkiang@gmail.com"]
+  spec.authors       = ['binz']
+  spec.email         = ['xinkiang@gmail.com']
   spec.summary       = %q{Dig need stractual infomation from web page.}
   spec.description   = %q{}
-  spec.homepage      = ""
-  spec.license       = "MIT"
+  spec.homepage      = ''
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
-  spec.require_paths = ["lib"]
+  spec.require_paths = ['lib']
 
-  spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", ">= 12.3.3"
+  spec.add_development_dependency 'rake', '>= 12.3.3'
+  spec.add_development_dependency 'bundler', '~> 2.0'
 
-  spec.add_runtime_dependency 'nokogiri', '~> 1.6'
   spec.add_runtime_dependency 'http-cookie', '~> 1.0'
+  spec.add_runtime_dependency 'nokogiri', '~> 1.6'
 end

data/lib/digger/page.rb

@@ -4,6 +4,7 @@ require 'ostruct'
 require 'set'
 require 'kconv'
 require 'uri'
+require 'http/cookie'
 
 # https://github.com/taganaka/polipus/blob/master/lib/polipus/page.rb
 module Digger
@@ -101,6 +102,10 @@ module Digger
       @jsonp ||= JSON.parse body.match(/^[^(]+?\((.+)\)[^)]*$/)[1]
     end
 
+    def cookies
+      @cookies ||= (headers['set-cookie'] || []).flat_map { |c| ::HTTP::Cookie.parse(c, url) }
+    end
+
     #
     # Discard links, a next call of page.links will return an empty array
     #
@@ -273,4 +278,4 @@ module Digger
       from_hash hash
     end
   end
-end
+end

data/lib/digger/pattern.rb

@@ -11,7 +11,7 @@ module Digger
 
     def safe_block(&default_block)
       if block.nil? || (block.is_a?(String) && block.strip.empty?)
-        default_block
+        default_block || ->(v) { v }
       elsif block.respond_to?(:call)
         block
       else
@@ -32,25 +32,32 @@ module Digger
     TYPES_CSS = %w[css_one css_many].freeze
     TYPES_JSON = %w[json jsonp].freeze
 
-    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON
+    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON + ['cookie']
 
     def match_page(page)
       return unless page.success?
+
       if TYPES_REGEXP.include?(type) # regular expression
         regexp_match(page.body)
       elsif TYPES_CSS.include?(type) # css expression
         css_match(page.doc)
       elsif TYPES_JSON.include?(type)
         json_match(page)
+      else
+        cookie_get(page.cookies)
       end
     end
 
+    def cookie_get(cookies)
+      cookie = cookies.find { |c| c.name == value }&.value
+      safe_block.call(cookie)
+    end
+
     def json_match(page)
-      block = safe_block { |j| j }
       json = page.send(type)
       keys = json_index_keys(value)
       match = json_fetch(json, keys)
-      block.call(match)
+      safe_block.call(match)
     end
 
     def css_match(doc)

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = '0.1.7'.freeze
+  VERSION = '0.1.8'.freeze
 end

data/spec/page_spec.rb

@@ -1,6 +1,7 @@
 require 'digger'
 require 'json'
 require 'uri'
+require 'cgi'
 
 describe Digger::Page do
   it 'page json' do
@@ -15,13 +16,12 @@ describe Digger::Page do
 
   it 'fetch baidu' do
     http = Digger::HTTP.new
-    page = http.fetch_page('http://www.baidu.com/')
+    page = http.fetch_page('http://baidu.com/')
     expect(page.code).to eq(200)
   end
 
-  it 'page uri' do
-    link ='https://www.baidu.com/s?wd=%E5%93%88%E5%93%88#hello'
-    link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#[\w]*$/, '')
-    p link
-  end
-end
+  # it 'page uri' do
+  #   link = 'https://www.baidu.com/s?wd=%E5%93%88%E5%93%88#hello'
+  #   link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#\w*$/, '')
+  # end
+end

data/spec/pattern_spec.rb

@@ -2,14 +2,19 @@ require 'digger'
 require 'json'
 
 describe Digger::Pattern do
-  it 'json fetch' do
-    json = JSON.parse('[{"a":1,"b":[1,2,3]}]')
-    pt = Digger::Pattern.new
-    expect(pt.json_fetch(json, '$[0]')['a']).to eq(1)
-    expect(pt.json_fetch(json, '$[0].a')).to eq(1)
-    expect(pt.json_fetch(json, '$[0].b').length).to eq(3)
-    expect(pt.json_fetch(json, '$[0].b[2]')).to eq(3)
-  end
-
+  # it 'json fetch' do
+  #   json = JSON.parse('[{"a":1,"b":[1,2,3]}]')
+  #   pt = Digger::Pattern.new
+  #   expect(pt.json_fetch(json, '$[0]')['a']).to eq(1)
+  #   expect(pt.json_fetch(json, '$[0].a')).to eq(1)
+  #   expect(pt.json_fetch(json, '$[0].b').length).to eq(3)
+  #   expect(pt.json_fetch(json, '$[0].b[2]')).to eq(3)
+  # end
 
-end
+  it 'parse cookoe' do
+    page = Digger::HTTP.new.fetch_page('https://xueqiu.com/')
+    pt = Digger::Pattern.new({ type: 'cookie', value: 'xq_a_token', block: ->(v) { "!!#{v}" } })
+    result = pt.match_page(page)
+    expect(result.length).to eq(42)
+  end
+end

metadata

@@ -1,71 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - binz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-26 00:00:00.000000000 Z
+date: 2021-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 12.3.3
+        version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: nokogiri
+  name: http-cookie
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: http-cookie
+  name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
 description: ''
 email:
 - xinkiang@gmail.com