digger 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 586e17ec8338bf102a91c8ee241632619a55b6b7
-  data.tar.gz: d93040c059eaf4c4e34cc329c4938a324feea142
+SHA256:
+  metadata.gz: 5c4a94163a25d4b53ad5b477040f69b8fccca026adc313f8f61759317c1bf198
+  data.tar.gz: 307b443277c16708103c172e5fb4ef4d833f7d2631a7f85779570a3cbeac8925
 SHA512:
-  metadata.gz: 089a7c6b6b1f8904202cfa8f362cc30d39b1dbfaaff5be721ded38f9448fe03aed23c4986ff587ab60499bc661a7abe104d3096f9eb9bbd0fa91dae18abcd933
-  data.tar.gz: 7ed6a2c324d55e634d86d1a450c12af045cba6049905e3a97d4eca3fe025fb11f92812c0e88d9c41cf5fa00eb44f454efb7efcfc598ad0b05257a6b0c6328a19
+  metadata.gz: 9da40123fd09615d0c69ca5104d1141b82981813ef8d175bc567a0f35e8f7dd868ce235bc0f265308e2133710f63c3ce8325d6b528ae060eccb77904c12e3139
+  data.tar.gz: c05be67df6db25345acfdc3615690c0467029dd40ffde262863923b6f0786696a16fd547277ec4d3764ed39b399174dd28e10ca7eb31ffb55f773e08b04f2986

data/digger.gemspec

@@ -18,7 +18,7 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 10.0"
 
   spec.add_runtime_dependency 'nokogiri', '~> 1.6'

data/lib/digger/page.rb

@@ -95,7 +95,13 @@ module Digger
       end
     end
 
+    def json
+      @json ||= JSON.parse body
+    end
 
+    def jsonp
+      @jsonp ||= JSON.parse body.match(/^[^\(]+?\((.+)\)[^\)]*$/)[1]
+    end
 
     #
     # Discard links, a next call of page.links will return an empty array

data/lib/digger/pattern.rb

@@ -27,33 +27,36 @@ module Digger
     end
 
     MATCH_MAX = 3
-    
-    TYPES = 0.upto(MATCH_MAX).map{|i| "match_#{i}"} + %w{match_many css_one css_many}
 
-    def regexp?
-      TYPES.index(type) <= MATCH_MAX + 1 # match_many in addition
-    end
+    TYPES_REGEXP = 0.upto(MATCH_MAX).map{|i| "match_#{i}"} + %w{match_many}
+    TYPES_CSS = %w{css_one css_many}
+    TYPES_JSON = %w{json jsonp}
+    
+    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON
 
     def match_page(page, &callback)
       blk = callback || safe_block
-      if regexp? # regular expression
-        index = TYPES.index(type)
-        blk ||= ->(text){text.strip}
+      if TYPES_REGEXP.include?(type) # regular expression
+        blk ||= ->(text){ text.strip }
         # content is String
         if type == 'match_many'
           match = page.body.gsub(value).to_a
         else
+          index = TYPES_REGEXP.index(type)
           matches = page.body.match(value)
           match = matches.nil? ? nil : matches[index]
         end
-      else # css expression
-        blk ||= ->(node){node.content.strip}
+      elsif TYPES_CSS.include?(type) # css expression
+        blk ||= ->(node){ node.content.strip }
         # content is Nokogiri::HTML::Document
         if type == 'css_one'
           match = page.doc.css(value).first
-        elsif type == 'css_many' # css_many
+        else
           match = page.doc.css(value)
         end
+      elsif TYPES_JSON.include?(type)
+        json = page.send(type)
+        match = json_fetch(json, value)
       end
       if match.nil?
         nil
@@ -66,6 +69,23 @@ module Digger
       nil
     end
 
+    def json_fetch(json, keys)
+      if keys.is_a? String
+        # parse json keys like '$.k1.k2[0]'
+        parts = keys.match(/^\$[\S]*$/)[0].scan(/(\.([\w]+)|\[([\d]+)\])/).map do |p|
+          p[1].nil? ? { index: p[2].to_i  } : { key: p[1] }
+        end
+        json_fetch(json, parts)
+      elsif keys.is_a? Array
+        if keys.length == 0
+          json
+        else
+          pt = keys.shift
+          json_fetch(json[pt[:index] || pt[:key]], keys)
+        end
+      end
+    end
+
     class Nokogiri::XML::Node
       %w{one many}.each do |name|
         define_method "inner_#{name}" do |css, &block| 

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = "0.1.4"
+  VERSION = "0.1.5"
 end

data/spec/page_spec.rb

@@ -0,0 +1,14 @@
+require 'digger'
+require 'json'
+
+describe Digger::Page do
+  it 'page json' do
+    json_str = '{"a":1,"b":[1,2,3]}'
+    j1 = Digger::Page.new('', body: json_str)
+    j2 = Digger::Page.new('', body: "hello(#{json_str});")
+    expect(j1.json['a']).to eq(1)
+    expect(j2.jsonp['a']).to eq(1)
+    expect(j1.json['b'][0]).to eq(1)
+    expect(j2.jsonp['b'][1]).to eq(2)
+  end
+end

data/spec/pattern_spec.rb

@@ -0,0 +1,15 @@
+require 'digger'
+require 'json'
+
+describe Digger::Pattern do
+  it 'json fetch' do
+    json = JSON.parse('{"a":1,"b":[1,2,3]}')
+    pt = Digger::Pattern.new
+    expect(pt.json_fetch(json, '$')['a']).to eq(1)
+    expect(pt.json_fetch(json, '$.a')).to eq(1)
+    expect(pt.json_fetch(json, '$.b').length).to eq(3)
+    expect(pt.json_fetch(json, '$.b[2]')).to eq(3)
+  end
+
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - binz
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-01 00:00:00.000000000 Z
+date: 2021-12-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -87,12 +87,14 @@ files:
 - lib/digger/pattern.rb
 - lib/digger/version.rb
 - spec/digger_spec.rb
+- spec/page_spec.rb
+- spec/pattern_spec.rb
 - spec/validate_spec.rb
 homepage: ''
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -107,11 +109,12 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.2
-signing_key: 
+rubygems_version: 3.2.32
+signing_key:
 specification_version: 4
 summary: Dig need stractual infomation from web page.
 test_files:
 - spec/digger_spec.rb
+- spec/page_spec.rb
+- spec/pattern_spec.rb
 - spec/validate_spec.rb