tanakai 1.5.0 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0363335680ba18ca855d2413e4efdce1957decab5f31c954e2b04f4f91660ac6'
-  data.tar.gz: 3fee8a56e284ef3bae724d1ffe3cc7f1614ad374efd59d60a002a7f71353cf06
+  metadata.gz: aefc92adbda49240ac69ad9d3163ec3bb54930d8288705943d269ccbba41c64a
+  data.tar.gz: c1c361906354aba4edb1dcef9e2737b4e827c794c5f61b4104ec309a67e3ca80
 SHA512:
-  metadata.gz: fabeeb2270349d0961294de34abe055906c38477cd4f744da9e033c626939e2672b86b30053a3d8c89bea1889f6392370e1b58296a0327f1a891d4915132478c
-  data.tar.gz: 8e34927825ef45893de6e00c676621823b4d3ca7c28210c73720409000c04bb228b1ea0dd75293144afd1dbff6f4f370ca0312847d781895434896374bb77f7b
+  metadata.gz: c744ae590cbb25e9dde914174be9fb003399bdc25408c9901c3898c296d4ff5428b97d3e4942c952979586946c75b4c7027edd594d438dc387dffb6afedda3bb
+  data.tar.gz: ac51ac208e71cd928aa402d62f51e465b11620b742f5ab8af208323e1f636503345ae910a4a6c0fe21f3d415bcff7bba600b6022bf879877695ba711463e97c0

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # CHANGELOG
 
+## 1.5.1
+### New
+* Add `response_type` to `in_parallel`
+
 ## 1.5.0
 ### New
 * First release as Tanakai

data/README.md

@@ -626,7 +626,7 @@ Check out **Capybara cheat sheets** where you can see all available methods **to
 
 ### `request_to` method
 
-For making requests to a particular method there is `request_to`. It requires minimum two arguments: `:method_name` and `url:`. An optional argument is `data:` (see above what for is it). Example:
+For making requests to a particular method there is `request_to`. It requires minimum two arguments: `:method_name` and `url:`. An optional argument is `data:` (see above what for is it) and `response_type` (defaults to `:html`). Example:
 
 ```ruby
 class Spider < Tanakai::Base
@@ -635,11 +635,12 @@ class Spider < Tanakai::Base
 
   def parse(response, url:, data: {})
     # Process request to `parse_product` method with `https://example.com/some_product` url:
-    request_to :parse_product, url: "https://example.com/some_product"
+    request_to :parse_product, url: "https://example.com/some_product.json", response_type: :json
   end
 
   def parse_product(response, url:, data: {})
-    puts "From page https://example.com/some_product !"
+    puts "JSON parsed from page https://example.com/some_product.json"
+    puts response
   end
 end
 ```
@@ -1194,6 +1195,7 @@ I, [2018-08-22 14:49:12 +0400#13033] [M: 46982297486840]  INFO -- amazon_spider:
 * `delay:` set delay between requests: `in_parallel(:method, urls, threads: 3, delay: 2)`. Delay can be `Integer`, `Float` or `Range` (`2..5`). In case of a Range, delay number will be chosen randomly for each request: `rand (2..5) # => 3`
 * `engine:` set custom engine than a default one: `in_parallel(:method, urls, threads: 3, engine: :poltergeist_phantomjs)`
 * `config:` pass custom options to config (see [config section](#crawler-config))
+* `response_type:` response should be returned as `:html` or `:json`, defaults to `:html`
 
 ### Active Support included
 

data/lib/tanakai/base.rb

@@ -286,7 +286,7 @@ module Tanakai
       end
     end
 
-    def in_parallel(handler, urls, threads:, data: {}, delay: nil, engine: @engine, config: {})
+    def in_parallel(handler, urls, threads:, data: {}, delay: nil, engine: @engine, config: {}, response_type: :html)
       parts = urls.in_sorted_groups(threads, false)
       urls_count = urls.size
 
@@ -304,12 +304,12 @@ module Tanakai
           part.each do |url_data|
             if url_data.class == Hash
               if url_data[:url].present? && url_data[:data].present?
-                spider.request_to(handler, delay, url_data)
+                spider.request_to(handler, delay, url_data, response_type: response_type)
               else
                 spider.public_send(handler, url_data)
               end
             else
-              spider.request_to(handler, delay, url: url_data, data: data)
+              spider.request_to(handler, delay, url: url_data, data: data, response_type: response_type)
             end
           end
         ensure

data/lib/tanakai/template/Gemfile

@@ -4,7 +4,7 @@ git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 ruby '>= 2.5'
 
 # Framework
-gem 'tanakai'
+gem 'tanakai', '~> 1.5'
 
 # Require files in directory and child directories recursively
 gem 'require_all'

data/lib/tanakai/version.rb

@@ -1,3 +1,3 @@
 module Tanakai
-  VERSION = "1.5.0"
+  VERSION = "1.5.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tanakai
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.5.1
 platform: ruby
 authors:
 - Victor Afanasev