digger 0.1.9 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ed96af1c5ae92569e1de4885958ac5852864c045f6c6337d5f17d91747d8ed80
-  data.tar.gz: 82003ae80f54cd3f9b805757e5dcb4c7894bba91c4c376cf08ebd43e6de6e80b
+  metadata.gz: 29c3945e9377348e1152eea7f46e0f11aa2e59cc5568fad57d25ecd3d271a9df
+  data.tar.gz: 1e4862f9939aa9c62e175a39df078fe12a2f51190af1f280f6d418cbab7e6390
 SHA512:
-  metadata.gz: b7aad69fd46c7d1688026ece2e1efe14d7dea29b42f94656d794655e12a92677bd3e1034f0c776bf197bcd75c96bd49377df399433bd2e1c13507520af1addc5
-  data.tar.gz: 60055a69ec3ad77e80fc4f1b50bb3a6c298e2274827ae21b64d1d82dae53a1d5338ccc99bcdb09c7d2e946abd007a488ff3034a753bc54586e9b651aeb3c5ce7
+  metadata.gz: 177e393de76bc35e31d6cc0eeda839d543d13fd81c40eba7d08704a131ec01396872734ea689c49fb1d09ceb4ba604fae76c3c201a2706dc9c889161038e0323
+  data.tar.gz: da76004a179aaed5cf75a96f90da3ebc739416e0ec162ff95c6aa27625590826fcc179b749156ea96937e364fb6a251515cb335a14b317001f3a47ea30330aeb

data/lib/digger/pattern.rb

@@ -6,7 +6,9 @@ module Digger
     attr_accessor :type, :value, :block
 
     def initialize(hash = {})
-      hash.each_pair { |key, value| send("#{key}=", value) if %w[type value block].include?(key.to_s)}
+      hash.each_pair do |key, value|
+        send("#{key}=", value) if %w[type value block].include?(key.to_s)
+      end
     end
 
     def safe_block(&default_block)
@@ -31,8 +33,9 @@ module Digger
     TYPES_REGEXP = 0.upto(MATCH_MAX).map { |i| "match_#{i}" } + %w[match_many]
     TYPES_CSS = %w[css_one css_many].freeze
     TYPES_JSON = %w[json jsonp].freeze
+    TYPES_OTHER = %w[cookie plain lines header body].freeze
 
-    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON + ['cookie']
+    TYPES = TYPES_REGEXP + TYPES_CSS + TYPES_JSON + TYPES_OTHER
 
     def match_page(page)
       return unless page.success?
@@ -43,13 +46,31 @@ module Digger
         css_match(page.doc)
       elsif TYPES_JSON.include?(type)
         json_match(page)
-      else
-        cookie_get(page.cookies)
+      elsif TYPES_OTHER.include?(type)
+        send("get_#{type}", page)
       end
     end
 
-    def cookie_get(cookies)
-      cookie = cookies.find { |c| c.name == value }&.value
+    def get_header(page)
+      header = (page.headers[value.to_s.downcase] || []).first
+      safe_block.call(header)
+    end
+
+    def get_body(page)
+      safe_block.call(page.body)
+    end
+
+    def get_plain(page)
+      safe_block.call(page.doc.text)
+    end
+
+    def get_lines(page)
+      block = safe_block
+      page.body.split("\n").map(&:strip).filter { |line| !line.empty? }.map { |line| block.call(line) }
+    end
+
+    def get_cookie(page)
+      cookie = page.cookies.find { |c| c.name == value }&.value
       safe_block.call(cookie)
     end
 
@@ -75,7 +96,8 @@ module Digger
       block = safe_block(&:strip)
       # content is String
       if type == 'match_many'
-        body.gsub(value).to_a.map { |node| block.call(node) }.uniq
+        regexp = value.is_a?(Regexp) ? value : Regexp.new(value.to_s)
+        body.gsub(regexp).to_a.map { |node| block.call(node) }.uniq
       else
         index = TYPES_REGEXP.index(type)
         matches = body.match(value)

data/lib/digger/version.rb

@@ -1,3 +1,3 @@
 module Digger
-  VERSION = '0.1.9'.freeze
+  VERSION = '0.2.0'.freeze
 end

data/spec/pattern_spec.rb

@@ -11,10 +11,17 @@ describe Digger::Pattern do
   #   expect(pt.json_fetch(json, '$[0].b[2]')).to eq(3)
   # end
 
-  it 'parse cookoe' do
+  it 'parse cookie & others' do
     page = Digger::HTTP.new.fetch_page('https://xueqiu.com/')
-    pt = Digger::Pattern.new({ type: 'cookie', value: 'xq_a_token', block: ->(v) { "!!#{v}" } })
-    result = pt.match_page(page)
+    p1 = Digger::Pattern.new({ type: 'cookie', value: 'xq_a_token', block: ->(v) { "!!#{v}" } })
+    # cookie
+    result = p1.match_page(page)
     expect(result.length).to eq(42)
+    # header
+    p2 = Digger::Pattern.new({ type: 'header', value: 'transfer-encoding' })
+    expect(p2.match_page(page)).to eq('chunked')
+    # get_plain
+    p3 = Digger::Pattern.new({ type: 'plain' })
+    expect(p3.match_page(page).length).to be > 100
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: digger
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.2.0
 platform: ruby
 authors:
 - binz
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-29 00:00:00.000000000 Z
+date: 2021-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake