tanakai 1.6.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7ea3cd20cfaedaebf473e853b66ebe58958e89b7525246444e3c8aeef46a4bf0
-  data.tar.gz: a2c51b86487d6392a58b533237731996639fe0037c9aca22a6140c3c968eaf7d
+  metadata.gz: f639e8c843d1effdd2fa5268fe01ee0fda5adcb741ecda49b5e8f2c8a51f55a3
+  data.tar.gz: d03287426dc9e1e802ef149ad9edf5f83813ea067d22f9f6fb8fa7ec33bb3c5a
 SHA512:
-  metadata.gz: 52d9a730a0a9e08c0a49ee4177a0370f5ed2a12ac9e3925f0a83b0c232dcedb1645d1b6860cb19c8453bbc5777cec02403654e2282e57ad75c5c2cb898b6dc1b
-  data.tar.gz: '0969ee651ec787b9fa1e47b8d776571b6f4751c29d3dd15bb0c696181ceab8bc826db6f54486df6426151f25f626f4725bf79c51ec8cc8cebebbe6cfa057bfa3'
+  metadata.gz: 12b9a122343c1599c87caf97cd527bf98c83db50e4f5fab40b4657932c41b41c5f3437a26297141a2e19f064f5083ba9b12c099d4bc20f537b0cf440aa92d9e2
+  data.tar.gz: 756e98178ef2c1fe80d9dfca936eab248e46e1c199665c23973fa9b96ad513a9bad2e4d0498e6ea3d3442a3aa76bf744109d91161e39c89aa0b29cdf83385ed0

data/.gitignore

@@ -11,3 +11,4 @@ Gemfile.lock
 *.retry
 .tags*
 *.gem
+.DS_Store

data/CHANGELOG.md

@@ -1,18 +1,32 @@
 # CHANGELOG
 
+## Next
+* Your contribution here
+
+## 1.7.1
+### Fixes
+* [#5](https://github.com/glaucocustodio/tanakai/pull/5): Replace `File.exists`/`Dir.exists` that have been removed on ruby 3.2 - [MrChriss](https://github.com/MrChriss)
+
+## 1.7.0
+### New
+* Allow passing `data:` to `crawl!` - [glaucocustodio](https://github.com/glaucocustodio)
+
+### Fixes
+* [#4](https://github.com/glaucocustodio/tanakai/pull/4): Fix keyword args on `crawl!` - [milk1000cc](https://github.com/milk1000cc)
+
 ## 1.6.0
 ### New
-* Add support to Ruby 3
+* Add support to Ruby 3 - [glaucocustodio](https://github.com/glaucocustodio)
 
 ## 1.5.1
 ### New
-* Add `response_type` to `in_parallel`
+* Add `response_type` to `in_parallel` - [glaucocustodio](https://github.com/glaucocustodio)
 
 ## 1.5.0
 ### New
-* First release as Tanakai
-* Add support to [Apparition](https://github.com/twalpole/apparition)
-* Add support to [Cuprite](https://github.com/rubycdp/cuprite)
+* First release as Tanakai - [glaucocustodio](https://github.com/glaucocustodio)
+* Add support to [Apparition](https://github.com/twalpole/apparition) - [glaucocustodio](https://github.com/glaucocustodio)
+* Add support to [Cuprite](https://github.com/rubycdp/cuprite) - [glaucocustodio](https://github.com/glaucocustodio)
 
 ## 1.4.0
 ### New
@@ -120,7 +134,6 @@
 * Fix Mechanize::Driver#proxy (there was a bug while using proxy for mechanize engine without authorization)
 * Fix requests retries logic
 
-
 ## 1.0.1
 * Add missing `logger` method to pipeline
 * Fix `set_proxy` in Mechanize and Poltergeist builders

data/README.md

@@ -1355,6 +1355,12 @@ end # =>
 # {:spider_name=>"example_spider", :status=>:completed, :environment=>"development", :start_time=>2018-08-22 18:49:22 +0400, :stop_time=>2018-08-22 18:49:23 +0400, :running_time=>0.801, :visits=>{:requests=>1, :responses=>1}, :items=>{:sent=>0, :processed=>0}, :error=>nil}
 ```
 
+You can also pass `data` to `crawl!`:
+
+```ruby
+ExampleSpider.crawl!(data: { foo: "bar" })
+```
+
 So what if you're don't care about stats and just want to process request to a particular spider method and get the returning value from this method? Use `.parse!` instead:
 
 #### `.parse!(:method_name, url:)` method

data/lib/tanakai/base/saver.rb

@@ -42,7 +42,7 @@ module Tanakai
       def save_to_json(item)
         data = JSON.generate([item])
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           file_content = File.read(path).sub(/\}\]\Z/, "\}\,")
           File.open(path, "w") do |f|
             f.write(file_content + data.sub(/\A\[/, ""))
@@ -55,7 +55,7 @@ module Tanakai
       def save_to_pretty_json(item)
         data = JSON.pretty_generate([item])
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           file_content = File.read(path).sub(/\}\n\]\Z/, "\}\,\n")
           File.open(path, "w") do |f|
             f.write(file_content + data.sub(/\A\[\n/, ""))
@@ -68,7 +68,7 @@ module Tanakai
       def save_to_jsonlines(item)
         data = JSON.generate(item)
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           File.open(path, "a") { |file| file.write("\n" + data) }
         else
           File.open(path, "w") { |file| file.write(data) }
@@ -78,7 +78,7 @@ module Tanakai
       def save_to_csv(item)
         data = flatten_hash(item)
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           CSV.open(path, "a+", force_quotes: true) do |csv|
             csv << data.values
           end
@@ -102,5 +102,3 @@ module Tanakai
     end
   end
 end
-
-

data/lib/tanakai/base.rb

@@ -100,7 +100,7 @@ module Tanakai
       end
     end
 
-    def self.crawl!(exception_on_fail: true)
+    def self.crawl!(exception_on_fail: true, data: {})
       logger.error "Spider: already running: #{name}" and return false if running?
 
       @storage = Storage.new
@@ -124,13 +124,13 @@ module Tanakai
       if start_urls
         start_urls.each do |start_url|
           if start_url.class == Hash
-            spider.request_to(:parse, start_url)
+            spider.request_to(:parse, url: start_url, data: data)
           else
-            spider.request_to(:parse, url: start_url)
+            spider.request_to(:parse, url: start_url, data: data)
           end
         end
       else
-        spider.parse
+        spider.parse(data: data)
       end
     rescue StandardError, SignalException, SystemExit => e
       @run_info.merge!(status: :failed, error: e.inspect)
@@ -160,7 +160,7 @@ module Tanakai
       if args.present?
         spider.public_send(handler, *args)
       elsif request.present?
-        spider.request_to(handler, request)
+        spider.request_to(handler, **request)
       else
         spider.public_send(handler)
       end

data/lib/tanakai/cli/ansible_command_builder.rb

@@ -31,7 +31,7 @@ module Tanakai
           "--extra-vars", "ansible_python_interpreter=/usr/bin/python3"
         ]
 
-        if File.exists? "config/automation.yml"
+        if File.exist? "config/automation.yml"
           require 'yaml'
           if config = YAML.load_file("config/automation.yml").dig(@playbook)
             config.each { |key, value| @vars[key] = value unless @vars[key] }

data/lib/tanakai/cli/generator.rb

@@ -17,7 +17,7 @@ module Tanakai
 
       def generate_spider(spider_name, in_project:)
         spider_path = in_project ? "spiders/#{spider_name}.rb" : "./#{spider_name}.rb"
-        raise "Spider #{spider_path} already exists" if File.exists? spider_path
+        raise "Spider #{spider_path} already exist?" if File.exist? spider_path
 
         spider_class = to_spider_class(spider_name)
         create_file spider_path do

data/lib/tanakai/cli.rb

@@ -174,7 +174,7 @@ module Tanakai
     private
 
     def inside_project?
-      Dir.exists?("spiders") && File.exists?("./config/boot.rb")
+      Dir.exist?("spiders") && File.exist?("./config/boot.rb")
     end
   end
 end

data/lib/tanakai/version.rb

@@ -1,3 +1,3 @@
 module Tanakai
-  VERSION = "1.6.0"
+  VERSION = "1.7.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tanakai
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.1
 platform: ruby
 authors:
 - Victor Afanasev
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-02-16 00:00:00.000000000 Z
+date: 2023-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor