husc 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 70da435ce2b15bb485ce958997a91488cf19f00b7faf5da3f25c92a891028508
-  data.tar.gz: b363fa06b547c1a5612889739465af7ea46cbeea6b6922aab0d484fa21169ce6
+  metadata.gz: a8d3839225b1c0ae71db2371b3858620ac34408c0a209739dbe96e1683437a6f
+  data.tar.gz: 4effb7652a7d0121b835e4ac8f8255305c9eb078ee7acaa245dccd19c819843b
 SHA512:
-  metadata.gz: 2190d0269954730626eca1d142c8ffd00614bf98fb19d7ba4d595668e944d769636f26c151211a664ba775b2d9f1c3b5594bbd7d5cde05eec7af31e11f82dfff
-  data.tar.gz: 5f0ea49baf5d2fbf6a707f3b580e95364686c2a3e1439d8c3469c090335790ebeef062176e3d67e3afd7592b23e2256f92f6421a14e4b208c94b4e5a1e737a7f
+  metadata.gz: 29d4486ab7cf5de9fb3e757b0b495ac9fbcd88ed389d6696f04a42a8b4e6dfbb841e2435afb3af996d883bcde8ecdca9dde9911be20a7b36cf4009dea4dbdec8
+  data.tar.gz: be53729a52309e9b4dbc36cd82673e0d4f20ba8d9631390776a65c5fe623b6395b03636cffa498c4aed1df8cacd767d0ed6435a2f527cd393b68b4b7694c82f1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    husc (0.1.0)
+    husc (0.1.1)
       mechanize
       nokogiri
 

data/README.md

@@ -1,5 +1,5 @@
-Husc
-=======
+husc
+====
 
 A simple crawling utility for Ruby.
 
@@ -19,7 +19,7 @@ This project enables site crawling and data extraction with xpath and css select
 require 'husc'
 
 url = 'http://www.example.com/'
-doc = Husc(url)
+doc = Husc.new(url)
 
 # access another url
 doc.get('another url')
@@ -32,7 +32,6 @@ doc.html
 
 # get <table> tags as dict
 doc.tables
-# ex) doc.tables['予約・お問い合わせ'] => 050-5596-6465
 ```
 
 ### Scraping Example
@@ -50,14 +49,14 @@ doc.xpath('//*[@id="top"]/div[1]')
 
 # other example
 doc.css('div').css('a')[2].attr('href') # => string object
-doc.css('p').innerText() # => string object
+doc.css('p').inner_text() # => string object
 # You do not need to specify "[]" to access the first index
 ```
 
 ### Submitting Form Example
 1. Specify target node's attribute
 2. Specify value(int or str) / check(bool) / file_name(str)
-3. call submit() with form attribute specified
+3. Call submit() with form attribute specified
 ```ruby
 # login
 doc.send(id:'id attribute', value:'value to send')
@@ -77,8 +76,6 @@ doc.send(class:'class attribute', value:100)
 ```
 
 
-
-
 ## Installation
 ```sh
 $ gem install husc
@@ -86,4 +83,4 @@ $ gem install husc
 
 
 ## Contributing
-Bug reports and pull requests are welcome on GitHub at [https://github.com/AjxLab/PyCrawl](https://github.com/AjxLab/PyCrawl).
+Bug reports and pull requests are welcome on GitHub at [https://github.com/AjxLab/husc](https://github.com/AjxLab/husc).

data/lib/husc/version.rb

@@ -1,3 +1,3 @@
-module Husc
-  VERSION = "0.1.1"
+class Husc
+  VERSION = "0.2.0"
 end

data/lib/husc.rb

@@ -5,211 +5,204 @@ require 'net/http'
 require 'kconv'
 require 'husc/version'
 
-module Husc
+
+class Husc
   class Error < StandardError; end
 
-  class Crawler
-    attr_reader :url, :html, :tables, :params
+  attr_reader :url, :html, :tables, :params
 
-    # 特殊配列
-    class CrawlArray < Array
+  # 特殊配列
+  class CrawlArray < Array
 
-      def find(search)
-        ## -----*----- 検索 -----*----- ##
-        self.each do |e|
-          if search.keys[0].to_s == 'inner'
-            # innerTextが一致するか
-            return e if e.innerText == search.values[0]
-          else
-            # 属性が一致するか
-            return e if e.attr(search.keys[0].to_s) == search.values[0]
-          end
-        end
-      end
-
-      def method_missing(method, *args)
-        if self == []
-          return eval("Crawler.new(doc: nil).#{method}(*#{args})")
+    def find(search)
+      ## -----*----- 検索 -----*----- ##
+      self.each do |e|
+        if search.keys[0].to_s == 'inner'
+          # inner_textが一致するか
+          return e if e.inner_text == search.values[0]
+        else
+          # 属性が一致するか
+          return e if e.attr(search.keys[0].to_s) == search.values[0]
         end
-
-        return eval("self[0].#{method}(*#{args})")
       end
     end
 
-    def initialize(url = nil, doc: nil, html: nil)
-      ## -----*----- コンストラクタ -----*----- ##
-      @agent = Mechanize.new
-      @agent.keep_alive = false
-
-      if !url.nil?
-        get(url)
-      elsif !doc.nil?
-        @html = doc.to_html
-        @doc = doc
-        table_to_hash
-      else
-        update_params(html)
-        @html = html
+    def method_missing(method, *args)
+      if self == []
+        return eval("Husc.new(doc: nil).#{method}(*#{args})")
       end
 
-      @params = []
+      return eval("self[0].#{method}(*#{args})")
     end
+  end
+
+  def initialize(url = nil, doc: nil, html: nil)
+    ## -----*----- コンストラクタ -----*----- ##
+    @agent = Mechanize.new
+    @agent.keep_alive = false
 
-    def get(url)
-      ## -----*----- ページ推移 -----*----- ##
-      @url = url
-      page = @agent.get(@url)
-      html = page.content.toutf8
+    if !url.nil?
+      get(url)
+    elsif !doc.nil?
+      @html = doc.to_html
+      @doc = doc
+      table_to_hash
+    else
       update_params(html)
+      @html = html
     end
 
-    def send(opts)
-      ## -----*----- フォームデータ指定 -----*----- ##
-      #
-      # テキスト，数値など　  => value（String）を指定
-      # チェックボックス　　  => check（Bool）を指定
-      # ファイルアップロード  => file（String）を指定
-      @params << {}
-      opts.each {|key, value| @params[-1][key.to_sym] = value}
-    end
+    @params = []
+  end
 
-    def submit(url = @url, opt)
-      ## -----*----- フォーム送信 -----*----- ##
-      @agent.get(url) do |page|
-        # フォーム指定
-        if opt.kind_of?(Integer)
-          form = page.forms[opt]
-        else
-          form = page.form(**opt)
-        end
-        return if form.nil?
-
-        @params.each do |param|
-          # テキスト，数値など
-          if param.include?(:value) && !param.include?(:check)
-            value = param.delete(:value)
-            next if value.nil?
-            form.field_with(**param).value = value unless form.field_with(**param).nil?
-          end
+  def get(url)
+    ## -----*----- ページ推移 -----*----- ##
+    @url = url
+    page = @agent.get(@url)
+    html = page.content.toutf8
+    update_params(html)
+  end
 
-          # チェックボックス
-          if param.include?(:check)
-            check = param.delete(:check)
-            next if check.nil?
-            if check
-              form.checkbox_with(**param).check unless form.checkbox_with(**param).nil?
-            else
-              form.checkbox_with(**param).uncheck unless form.checkbox_with(**param).nil?
-            end
-          end
+  def send(opts)
+    ## -----*----- フォームデータ指定 -----*----- ##
+    #
+    # テキスト，数値など　  => value（String）を指定
+    # チェックボックス　　  => check（Bool）を指定
+    # ファイルアップロード  => file（String）を指定
+    @params << {}
+    opts.each {|key, value| @params[-1][key.to_sym] = value}
+  end
 
-          # ファイルアップロード
-          if param.include?(:file)
-            file = param.delete(:file)
-            next if file.nil? || !File.exist?(file)
-            form.file_upload_with(**param).file_name = file unless form.file_upload_with(**param).nil?
-          end
+  def submit(url = @url, opt)
+    ## -----*----- フォーム送信 -----*----- ##
+    @agent.get(url) do |page|
+      # フォーム指定
+      if opt.kind_of?(Integer)
+        form = page.forms[opt]
+      else
+        form = page.form(**opt)
+      end
+      return if form.nil?
+
+      @params.each do |param|
+        # テキスト，数値など
+        if param.include?(:value) && !param.include?(:check)
+          value = param.delete(:value)
+          next if value.nil?
+          form.field_with(**param).value = value unless form.field_with(**param).nil?
         end
 
-        form = form.submit
-        update_params(form.content.toutf8)
-      end
-      @params = []
-    end
+        # チェックボックス
+        if param.include?(:check)
+          check = param.delete(:check)
+          next if check.nil?
+          if check
+            form.checkbox_with(**param).check unless form.checkbox_with(**param).nil?
+          else
+            form.checkbox_with(**param).uncheck unless form.checkbox_with(**param).nil?
+          end
+        end
 
-    def xpath(locator, single = false)
-      ## -----*----- HTMLからXPath指定で要素取得 -----*----- ##
-      elements = CrawlArray.new(@doc.xpath(locator).map {|el| Crawler.new(doc: el)})
-      if single
-        # シングルノード
-        if elements[0] == nil
-          return CrawlArray.new()
-        else
-          return elements[0]
+        # ファイルアップロード
+        if param.include?(:file)
+          file = param.delete(:file)
+          next if file.nil? || !File.exist?(file)
+          form.file_upload_with(**param).file_name = file unless form.file_upload_with(**param).nil?
         end
-      else
-        # 複数ノード
-        return elements
       end
+
+      form = form.submit
+      update_params(form.content.toutf8)
     end
+    @params = []
+  end
 
-    def css(locator, single = false)
-      ## -----*----- HTMLからCSSセレクタで要素取得 -----*----- ##
-      elements = CrawlArray.new(@doc.css(locator).map {|el| Crawler.new(doc: el)})
-      if single
-        # シングルノード
-        if elements[0] == nil
-          return CrawlArray.new()
-        else
-          return elements[0]
-        end
+  def xpath(locator, single = false)
+    ## -----*----- HTMLからXPath指定で要素取得 -----*----- ##
+    elements = CrawlArray.new(@doc.xpath(locator).map {|el| Husc.new(doc: el)})
+    if single
+      # シングルノード
+      if elements[0] == nil
+        return CrawlArray.new()
       else
-        # 複数ノード
-        return elements
+        return elements[0]
       end
+    else
+      # 複数ノード
+      return elements
     end
+  end
 
-    def innerText(shaping = true)
-      ## -----*----- タグ内の文字列を取得 -----*----- ##
-      if shaping
-        return shaping_string(@doc.inner_text)
+  def css(locator, single = false)
+    ## -----*----- HTMLからCSSセレクタで要素取得 -----*----- ##
+    elements = CrawlArray.new(@doc.css(locator).map {|el| Husc.new(doc: el)})
+    if single
+      # シングルノード
+      if elements[0] == nil
+        return CrawlArray.new()
       else
-        @doc.inner_text
+        return elements[0]
       end
+    else
+      # 複数ノード
+      return elements
     end
+  end
 
-    def text(shaping = true)
-      ## -----*----- タグ内の文字列（その他タグ除去）を取得 -----*----- ##
-      if shaping
-        return shaping_string(@doc.text)
-      else
-        @doc.text
-      end
+  def inner_text(shaping = true)
+    ## -----*----- タグ内の文字列を取得 -----*----- ##
+    if shaping
+      return shaping_string(@doc.inner_text)
+    else
+      @doc.inner_text
     end
+  end
 
-    def attr(name)
-      ## -----*----- ノードの属性情報取得 -----*----- ##
-      ret = @doc.attr(name)
-      if ret.nil?
-        return ''
-      else
-        return ret
-      end
+  def text(shaping = true)
+    ## -----*----- タグ内の文字列（その他タグ除去）を取得 -----*----- ##
+    if shaping
+      return shaping_string(@doc.text)
+    else
+      @doc.text
     end
+  end
 
-    def url()
-      ## -----*----- カレントURLの取得 -----*----- ##
-      return @agent.get(@url).uri.to_s
+  def attr(name)
+    ## -----*----- ノードの属性情報取得 -----*----- ##
+    ret = @doc.attr(name)
+    if ret.nil?
+      return ''
+    else
+      return ret
     end
+  end
 
 
-    private
+  private
 
 
-    def update_params(html)
-      ## -----*----- パラメータを更新 -----*----- ##
-      @html = html
-      @doc = Nokogiri::HTML.parse(@html)
-      table_to_hash
-    end
+  def update_params(html)
+    ## -----*----- パラメータを更新 -----*----- ##
+    @html = html
+    @doc = Nokogiri::HTML.parse(@html)
+    table_to_hash
+  end
 
-    def table_to_hash
-      ## -----*----- テーブル内容をHashに変換 -----*----- ##
-      @tables = {}
-      @doc.css('tr').each do |tr|
-        @tables[tr.css('th').inner_text.gsub("\n", "").gsub(" ", "")] = shaping_string(tr.css('td').inner_text)
-      end
-      @doc.css('dl').each do |el|
-        @tables[el.css('dt').inner_text.gsub("\n", "").gsub(" ", "")] = shaping_string(el.css('dd').inner_text)
-      end
+  def table_to_hash
+    ## -----*----- テーブル内容をHashに変換 -----*----- ##
+    @tables = {}
+    @doc.css('tr').each do |tr|
+      @tables[tr.css('th').inner_text.gsub("\n", "").gsub(" ", "")] = shaping_string(tr.css('td').inner_text)
     end
-
-    def shaping_string(str)
-      ## -----*----- 文字例の整形 -----*----- ##
-      # 余計な改行，空白を全て削除
-      str = str.to_s
-      return str.gsub(" ", ' ').squeeze(' ').gsub("\n \n", "\n").gsub("\n ", "\n").gsub("\r", "\n").squeeze("\n").gsub("\t", "").strip
+    @doc.css('dl').each do |el|
+      @tables[el.css('dt').inner_text.gsub("\n", "").gsub(" ", "")] = shaping_string(el.css('dd').inner_text)
     end
   end
-end
 
+  def shaping_string(str)
+    ## -----*----- 文字例の整形 -----*----- ##
+    # 余計な改行，空白を全て削除
+    str = str.to_s
+    return str.gsub(" ", ' ').squeeze(' ').gsub("\n \n", "\n").gsub("\n ", "\n").gsub("\r", "\n").squeeze("\n").gsub("\t", "").strip
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: husc
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Tatsuya Abe