digger 0.1.9 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed96af1c5ae92569e1de4885958ac5852864c045f6c6337d5f17d91747d8ed80
-  data.tar.gz: 82003ae80f54cd3f9b805757e5dcb4c7894bba91c4c376cf08ebd43e6de6e80b
+  metadata.gz: 025bfc4f8d6dbf55994363070b67a450fd421457c4f12bf9b714fe965aefaad6
+  data.tar.gz: a8464f07a53c332ddb40457998b830b1dff99661431a7608d9dc075175f5e9f1
 SHA512:
-  metadata.gz: b7aad69fd46c7d1688026ece2e1efe14d7dea29b42f94656d794655e12a92677bd3e1034f0c776bf197bcd75c96bd49377df399433bd2e1c13507520af1addc5
-  data.tar.gz: 60055a69ec3ad77e80fc4f1b50bb3a6c298e2274827ae21b64d1d82dae53a1d5338ccc99bcdb09c7d2e946abd007a488ff3034a753bc54586e9b651aeb3c5ce7
+  metadata.gz: d84a2f8ee2d65c16d79b66795fc618b07fb5ba4517d902eec7e17107b69c240b36f68728de49f451fd143ddeb872743a906b608d413446971e5f38a632cf6fbe
+  data.tar.gz: 4f36e035bab956bd6702024d951eb9963f2f97812f355e00b0840148eaa4fef07949abac0f874f889c88703addd13ea2bc69efa61bac3872b14f371f7fa47ded

data/lib/digger/pattern.rb

@@ -6,7 +6,9 @@ module Digger
     attr_accessor :type, :value, :block
 
     def initialize(hash = {})
-      hash.each_pair { |key, value| send("#{key}=", value) if %w[type value block].include?(key.to_s)}
+      hash.each_pair do |key, value|
+        send("#{key}=", value) if %w[type value block].include?(key.to_s)
+      end
     end
 
     def safe_block(&default_block)
@@ -28,11 +30,12 @@ module Digger
 
     MATCH_MAX = 3
 
-    TYPES_REGEXP = 0.upto(MATCH_MAX).map { |i| "match_#{i}" } + %w[match_many]
-    TYPES_CSS = %w[css_one css_many].freeze
+    TYPES_REGEXP = 0.upto(MATCH_MAX).map { |i| "match_#{i}" } + %w[match_many match_all]
+    TYPES_CSS = %w[css_one css_many css_all].freeze
     TYPES_JSON = %w[json jsonp].freeze
+    TYPES_OTHER = %w[cookie plain lines header body].freeze
 
-    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON + ['cookie']
+    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON + TYPES_OTHER
 
     def match_page(page)
       return unless page.success?
@@ -43,13 +46,31 @@ module Digger
         css_match(page.doc)
       elsif TYPES_JSON.include?(type)
         json_match(page)
-      else
-        cookie_get(page.cookies)
+      elsif TYPES_OTHER.include?(type)
+        send("get_#{type}", page)
       end
     end
 
-    def cookie_get(cookies)
-      cookie = cookies.find { |c| c.name == value }&.value
+    def get_header(page)
+      header = (page.headers[value.to_s.downcase] || []).first
+      safe_block.call(header)
+    end
+
+    def get_body(page)
+      safe_block.call(page.body)
+    end
+
+    def get_plain(page)
+      safe_block.call(page.doc&.text)
+    end
+
+    def get_lines(page)
+      block = safe_block
+      page.body.split("\n").map(&:strip).filter { |line| !line.empty? }.map { |line| block.call(line) }
+    end
+
+    def get_cookie(page)
+      cookie = page.cookies.find { |c| c.name == value }&.value
       safe_block.call(cookie)
     end
 
@@ -61,24 +82,36 @@ module Digger
     end
 
     def css_match(doc)
-      block = safe_block { |node| node.content.strip }
       # content is Nokogiri::HTML::Document
       contents = doc.css(value)
       if type == 'css_many'
-        contents.map { |node| block.call(node) }.uniq
+        block = safe_block { |node| node&.content&.strip }
+        contents.map { |node| block.call(node) }
+      elsif type == 'css_all'
+        block = safe_block
+        block.call(contents)
       else
+        block = safe_block { |node| node&.content&.strip }
         block.call(contents.first)
       end
     end
 
     def regexp_match(body)
-      block = safe_block(&:strip)
       # content is String
-      if type == 'match_many'
-        body.gsub(value).to_a.map { |node| block.call(node) }.uniq
+      if %w[match_many match_all].include? type
+        regexp = value.is_a?(Regexp) ? value : Regexp.new(value.to_s)
+        matches = body.gsub(regexp).to_a
+        if type == 'match_many'
+          block = safe_block(&:strip)
+          matches.map { |node| block.call(node) }
+        else
+          block = safe_block
+          block.call(matches)
+        end
       else
         index = TYPES_REGEXP.index(type)
         matches = body.match(value)
+        block = safe_block(&:strip)
         block.call(matches[index]) unless matches.nil?
       end
     end
@@ -100,21 +133,24 @@ module Digger
 
     private :json_index_keys, :json_fetch
 
-    # Nokogiri node methods
-    class Nokogiri::XML::Node
-      %w[one many].each do |name|
-        define_method "inner_#{name}" do |css, &block|
-          callback = ->(node) { (block || ->(n) { n.text.strip }).call(node) if node }
-          if name == 'one' # inner_one
-            callback.call(self.css(css).first)
-          else # inner_many
-            self.css(css).map { |node| callback.call(node) }
-          end
-        end
+    class ::Nokogiri::XML::Node
+      def inner_one(expr, &block)
+        fn = block || ->(node) { node&.content&.strip }
+        fn.call(css(expr)&.first)
+      end
+
+      def inner_many(expr, &block)
+        fn = block || ->(node) { node&.content&.strip }
+        css(expr)&.map { |node| fn.call(node) }
       end
+
       def source
         to_xml
       end
+
+      def inner_number
+        content&.match(/\d+/).to_s.to_i
+      end
     end
   end
 end

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = '0.1.9'.freeze
+  VERSION = '0.2.2'.freeze
 end

data/spec/pattern_spec.rb

@@ -11,10 +11,23 @@ describe Digger::Pattern do
   #   expect(pt.json_fetch(json, '$[0].b[2]')).to eq(3)
   # end
 
-  it 'parse cookoe' do
+  it 'parse cookie & others' do
     page = Digger::HTTP.new.fetch_page('https://xueqiu.com/')
-    pt = Digger::Pattern.new({ type: 'cookie', value: 'xq_a_token', block: ->(v) { "!!#{v}" } })
-    result = pt.match_page(page)
+    p1 = Digger::Pattern.new({ type: 'cookie', value: 'xq_a_token', block: ->(v) { "!!#{v}" } })
+    # cookie
+    result = p1.match_page(page)
     expect(result.length).to eq(42)
+    # header
+    p2 = Digger::Pattern.new({ type: 'header', value: 'transfer-encoding' })
+    expect(p2.match_page(page)).to eq('chunked')
+    # get_plain
+    p3 = Digger::Pattern.new({ type: 'plain' })
+    expect(p3.match_page(page).length).to be > 100
+  end
+
+  it 'match_all & css_all' do
+    p = Digger::Pattern.new({ type: 'match_all', value: '[\d]+' })
+    m = p.regexp_match('123,12,1')
+    expect(m.length).to eq(3)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.2
 platform: ruby
 authors:
 - binz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-29 00:00:00.000000000 Z
+date: 2022-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake