digger 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c4a94163a25d4b53ad5b477040f69b8fccca026adc313f8f61759317c1bf198
-  data.tar.gz: 307b443277c16708103c172e5fb4ef4d833f7d2631a7f85779570a3cbeac8925
+  metadata.gz: f3e89f179fa868ecd2879180d1fbfbf03ba0ebee3731b9c8b4741d22663ff4aa
+  data.tar.gz: 1b27e4a1446e9835203bf5497aeebc3bc4ab58998a0fc443eeaaf7e7ec86c2c7
 SHA512:
-  metadata.gz: 9da40123fd09615d0c69ca5104d1141b82981813ef8d175bc567a0f35e8f7dd868ce235bc0f265308e2133710f63c3ce8325d6b528ae060eccb77904c12e3139
-  data.tar.gz: c05be67df6db25345acfdc3615690c0467029dd40ffde262863923b6f0786696a16fd547277ec4d3764ed39b399174dd28e10ca7eb31ffb55f773e08b04f2986
+  metadata.gz: 5671e5d2484ca744e5c75f97beeb473e1970291fc094c0274714af27cc847ef47bcf3b1b81534e6f299f35648fdf3aabec9d90777e4b2fbe49dc2629c048f610
+  data.tar.gz: 496534bb394d17792dc7173759b83c0dd62b8569a241b2a6a57e60a14ea99ddafd7e030fe16480c5782ec319f6dc1d3563919f7202026fb008522dd26cae6c01

data/digger.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 2.0"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", ">= 12.3.3"
 
   spec.add_runtime_dependency 'nokogiri', '~> 1.6'
   spec.add_runtime_dependency 'http-cookie', '~> 1.0'

data/lib/digger/page.rb

@@ -3,6 +3,7 @@ require 'json'
 require 'ostruct'
 require 'set'
 require 'kconv'
+require 'uri'
 
 # https://github.com/taganaka/polipus/blob/master/lib/polipus/page.rb
 module Digger
@@ -186,16 +187,7 @@ module Digger
     def to_absolute(link)
       return nil if link.nil?
 
-      # link = link.to_s.encode('utf-8', 'binary', :invalid => :replace, :undef => :replace, :replace => '')
-
-      # remove anchor
-      link =
-        begin
-          URI.encode(URI.decode(link.to_s.gsub(/#[a-zA-Z0-9_-]*$/, '')))
-        rescue URI::Error
-          return nil
-        end
-
+      link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#[\w]*$/, '')
       relative = begin
                    URI(link)
                  rescue URI::Error

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

data/spec/digger_spec.rb

@@ -1,15 +1,14 @@
 require 'digger'
 
 http = Digger::HTTP.new
-page = http.fetch_page('http://nan.so/')
+page = http.fetch_page('http://www.baidu.com/')
 
-pattern = Digger::Pattern.new({type: 'css_many', value: '.sites>a>span' })
+pattern = Digger::Pattern.new({ type: 'css_many', value: '#s-top-left>a' })
 
 class Item < Digger::Model
-  css_many sites: '.sites>a>span'
-  css_one logo: '.logo'
+  css_many sites: '#s-top-left>a'
   validate_presence :sites
-  validate_includeness :sites, :logo
+  validate_includeness :sites
 end
 
 describe Digger do
@@ -19,12 +18,12 @@ describe Digger do
 
   it "pattern should match content" do
     sites = pattern.match_page(page)
-    expect(sites.include?('百度网盘')).to eq(true)
+    expect(sites.include?('新闻')).to eq(true)
   end
 
   it "model should dig content" do
     item = Item.new.match_page(page)
-    expect(item[:sites].include?('读远')).to be(true)
+    expect(item[:sites].include?('新闻')).to be(true)
   end
 
   it "validation support" do

data/spec/page_spec.rb

@@ -1,5 +1,6 @@
 require 'digger'
 require 'json'
+require 'uri'
 
 describe Digger::Page do
   it 'page json' do
@@ -11,4 +12,16 @@ describe Digger::Page do
     expect(j1.json['b'][0]).to eq(1)
     expect(j2.jsonp['b'][1]).to eq(2)
   end
+
+  it 'fetch baidu' do
+    http = Digger::HTTP.new
+    page = http.fetch_page('http://www.baidu.com/')
+    expect(page.code).to eq(200)
+  end
+
+  it 'page uri' do
+    link ='https://www.baidu.com/s?wd=%E5%93%88%E5%93%88#hello'
+    link = link.to_s.encode('utf-8', 'binary', invalid: :replace, undef: :replace, replace: '').gsub(/#[\w]*$/, '')
+    p link
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - binz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-25 00:00:00.000000000 Z
+date: 2021-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -28,16 +28,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement