kimurai 1.3.2 → 1.4.0

data/lib/kimurai/capybara_ext/session.rb

@@ -88,7 +88,16 @@ module Capybara
     def current_response(response_type = :html)
       case response_type
       when :html
-        Nokogiri::HTML(body)
+        if config.encoding
+          if config.encoding == :auto
+            charset = body.force_encoding("ISO-8859-1").encode("UTF-8")[/<meta.*?charset=["]?([\w+\d+\-]*)/i, 1]
+            Nokogiri::HTML(body, nil, charset)
+          else
+            Nokogiri::HTML(body, nil, config.encoding)
+          end
+        else
+          Nokogiri::HTML(body)
+        end
       when :json
         JSON.parse(body)
       end

data/lib/kimurai/capybara_ext/session/config.rb

@@ -1,6 +1,6 @@
 module Capybara
   class SessionConfig
-    attr_accessor :cookies, :proxy, :user_agent
+    attr_accessor :cookies, :proxy, :user_agent, :encoding
     attr_writer :retry_request_errors, :skip_request_errors
 
     def retry_request_errors

data/lib/kimurai/cli.rb

@@ -116,7 +116,11 @@ module Kimurai
       end
 
       engine = options["engine"]&.delete(":")&.to_sym
-      klass.parse!(:console, engine, url: options["url"])
+      if url = options["url"]
+        klass.new(engine).request_to(:console, url: options["url"])
+      else
+        klass.new(engine).public_send(:console)
+      end
     end
 
     desc "list", "List all available spiders in the current project"

data/lib/kimurai/template/Gemfile

@@ -4,7 +4,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 ruby '>= 2.5'
 
 # Framework
-gem 'kimurai', '~> 1.0'
+gem 'kimurai', '~> 1.4'
 
 # Require files in directory and child directories recursively
 gem 'require_all'

data/lib/kimurai/template/spiders/application_spider.rb

@@ -100,6 +100,12 @@ class ApplicationSpider < Kimurai::Base
     # Format: same like for `skip_request_errors` option.
     # retry_request_errors: [Net::ReadTimeout],
 
+    # Handle page encoding while parsing html response using Nokogiri. There are two modes:
+    # Auto (`:auto`) (try to fetch correct encoding from <meta http-equiv="Content-Type"> or <meta charset> tags)
+    # Set required encoding manually, example: `encoding: "GB2312"` (Set required encoding manually)
+    # Default this option is unset.
+    # encoding: nil,
+
     # Restart browser if one of the options is true:
     restart_if: {
       # Restart browser if provided memory limit (in kilobytes) is exceeded (works for all engines)

data/lib/kimurai/version.rb

@@ -1,3 +1,3 @@
 module Kimurai
-  VERSION = "1.3.2"
+  VERSION = "1.4.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kimurai
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.4.0
 platform: ruby
 authors:
 - Victor Afanasev
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-08 00:00:00.000000000 Z
+date: 2019-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor