tanakai 1.7.0 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a544dd8c9d448beccd646a4a536a5597bd9e6348717c4f8103f708798fbc21c5
-  data.tar.gz: '093f88b3185e3999f4bed6b6c9df472a1d7c32c4f1e95e044bd313ec4e444f50'
+  metadata.gz: 412cdf33eda65d65336652df303c4eb55cf395f5154e53f88a1c901ef4360c55
+  data.tar.gz: eb0217edc65c54bbe6b243596965b5073e0c3a308e5b52249d9de7027e8382e9
 SHA512:
-  metadata.gz: 0bf1db16739720f015902830f588a3a463331b6d36d241fdd0d397bd1d8cb2e8ef921bf8d10a552016b7b2e43e415f938b974001e63a347a8feec9bb8b9b270f
-  data.tar.gz: c080531acf9b98fb00c94ca994c3e32281c5eada7a38d15175a481f55e4b8cbe11b707950a9674fdfd09909b6ded2ece608f753b5fb5fb919ae79b6810c97547
+  metadata.gz: 87cdf732e266dc15d8a160d8ca7d4161f838b05f870d1fcc6b74f2c62f85a380ac98df778660206ab93927e22cd03dc5855b90062285461f48db041baeaa4cb4
+  data.tar.gz: 96b44cde5d58de7579dccd0394545f035412b01e26a46e2d686969176d6ea36589089ab3306a5972d2ae8cca4e8eb3d57b3c89173d96fe6531b5be8e528eae59

data/CHANGELOG.md

@@ -3,6 +3,14 @@
 ## Next
 * Your contribution here
 
+## 1.7.2
+### Fixes
+* [#6](https://github.com/glaucocustodio/tanakai/pull/6): Fix url validation - [MrChriss](https://github.com/MrChriss)
+
+## 1.7.1
+### Fixes
+* [#5](https://github.com/glaucocustodio/tanakai/pull/5): Replace `File.exists`/`Dir.exists` that have been removed on ruby 3.2 - [MrChriss](https://github.com/MrChriss)
+
 ## 1.7.0
 ### New
 * Allow passing `data:` to `crawl!` - [glaucocustodio](https://github.com/glaucocustodio)
@@ -130,7 +138,6 @@
 * Fix Mechanize::Driver#proxy (there was a bug while using proxy for mechanize engine without authorization)
 * Fix requests retries logic
 
-
 ## 1.0.1
 * Add missing `logger` method to pipeline
 * Fix `set_proxy` in Mechanize and Poltergeist builders

data/lib/tanakai/base/saver.rb

@@ -42,7 +42,7 @@ module Tanakai
       def save_to_json(item)
         data = JSON.generate([item])
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           file_content = File.read(path).sub(/\}\]\Z/, "\}\,")
           File.open(path, "w") do |f|
             f.write(file_content + data.sub(/\A\[/, ""))
@@ -55,7 +55,7 @@ module Tanakai
       def save_to_pretty_json(item)
         data = JSON.pretty_generate([item])
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           file_content = File.read(path).sub(/\}\n\]\Z/, "\}\,\n")
           File.open(path, "w") do |f|
             f.write(file_content + data.sub(/\A\[\n/, ""))
@@ -68,7 +68,7 @@ module Tanakai
       def save_to_jsonlines(item)
         data = JSON.generate(item)
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           File.open(path, "a") { |file| file.write("\n" + data) }
         else
           File.open(path, "w") { |file| file.write(data) }
@@ -78,7 +78,7 @@ module Tanakai
       def save_to_csv(item)
         data = flatten_hash(item)
 
-        if @index > 1 || append && File.exists?(path)
+        if @index > 1 || append && File.exist?(path)
           CSV.open(path, "a+", force_quotes: true) do |csv|
             csv << data.values
           end
@@ -102,5 +102,3 @@ module Tanakai
     end
   end
 end
-
-

data/lib/tanakai/base.rb

@@ -192,7 +192,9 @@ module Tanakai
     end
 
     def request_to(handler, delay = nil, url:, data: {}, response_type: :html)
-      raise InvalidUrlError, "Requested url is invalid: #{url}" unless URI.parse(url).kind_of?(URI::HTTP)
+      if %w[http https].exclude?(Addressable::URI.parse(url).scheme)
+        raise InvalidUrlError, "Requested url scheme is invalid: #{url}"
+      end
 
       if @config[:skip_duplicate_requests] && !unique_request?(url)
         add_event(:duplicate_requests) if self.with_info

data/lib/tanakai/cli/ansible_command_builder.rb

@@ -31,7 +31,7 @@ module Tanakai
           "--extra-vars", "ansible_python_interpreter=/usr/bin/python3"
         ]
 
-        if File.exists? "config/automation.yml"
+        if File.exist? "config/automation.yml"
           require 'yaml'
           if config = YAML.load_file("config/automation.yml").dig(@playbook)
             config.each { |key, value| @vars[key] = value unless @vars[key] }

data/lib/tanakai/cli/generator.rb

@@ -17,7 +17,7 @@ module Tanakai
 
       def generate_spider(spider_name, in_project:)
         spider_path = in_project ? "spiders/#{spider_name}.rb" : "./#{spider_name}.rb"
-        raise "Spider #{spider_path} already exists" if File.exists? spider_path
+        raise "Spider #{spider_path} already exist?" if File.exist? spider_path
 
         spider_class = to_spider_class(spider_name)
         create_file spider_path do

data/lib/tanakai/cli.rb

@@ -174,7 +174,7 @@ module Tanakai
     private
 
     def inside_project?
-      Dir.exists?("spiders") && File.exists?("./config/boot.rb")
+      Dir.exist?("spiders") && File.exist?("./config/boot.rb")
     end
   end
 end

data/lib/tanakai/version.rb

@@ -1,3 +1,3 @@
 module Tanakai
-  VERSION = "1.7.0"
+  VERSION = "1.7.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: tanakai
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.7.2
 platform: ruby
 authors:
 - Victor Afanasev
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-10-25 00:00:00.000000000 Z
+date: 2023-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor