url_verifier 0.0.1.pre.rc.01 → 2.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 173471784a4c797364efaee8b96757c992057df65e62d1b7373b65f4386abfe4
-  data.tar.gz: 591e35185ed9065cd2386912c04ea45fb891a08b9eda7da1415fb754b8bd4905
+  metadata.gz: 8393c5aa642d2da6e0552c011926170c29da0832237d9395632d5d5f6a4ac634
+  data.tar.gz: c5bcab090867ae584abc32c15d0c8a990432c94332a2743e7a68155474568369
 SHA512:
-  metadata.gz: 3981b75a7351316126d9ba85b6d5a349121a571e1119c45d02669a063ff704cc9d30669dd028884165ac1112b44dcc8d326415f858d44e3a0b256064ae6e7a5f
-  data.tar.gz: 7a65c63fd747f53357eb9b64903cdfe01b0a649ee8ea2e9a08aae52dba7c9e8ecd50ef5c8f99894fd84aa428e70f81fd28287d9ab572e2e4303f1fb1a5b18950
+  metadata.gz: ab1cb8b4a865ba81c12649872f26fe4916c163e511590f8352bd0cb4efbd6b554f0d97e3f2a624e328ef2fdbba7ff29e006b262b253cf2e5e26f9a3c6ee79933
+  data.tar.gz: 7184195ca8d85da4d161d769691b20572943f0fa33756a9c7caf06c0998b57769af51cb583b9dd2a492fb9467b46bd1744c3278ae82a704fc2cfb3a780bd0b84

data/.gitignore

@@ -1,11 +1,14 @@
 /.bundle/
 /.yardoc
+/Gemfile.lock
 /_yardoc/
 /coverage/
-/doc/
 /pkg/
 /spec/reports/
 /tmp/
 
-# rspec failure tracking
-.rspec_status
+utf8_sanitizer-*.gem
+.DS_Store
+.idea/
+.xlsx
+.txt

data/Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    url_verifier (0.0.1.pre.rc.01)
-      activesupport (~> 5.2, >= 5.2.0)
-      crm_formatter (~> 2.4)
-      utf8_sanitizer (~> 2.0)
+    url_verifier (2.10)
+      activesupport (~> 5.2)
+      crm_formatter (~> 2.62)
+      curb (~> 0.9.6)
+      net-ping (~> 1.7, >= 1.7.8)
+      utf8_sanitizer (~> 2.16)
 
 GEM
   remote: https://rubygems.org/
@@ -14,74 +16,21 @@ GEM
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    ast (2.4.0)
-    binding.repl (3.0.0)
-    byebug (10.0.2)
-    cd (1.0.1)
-    class_indexer (0.3.0)
-      json
-      parser
-    clipboard (1.1.2)
-    code (0.9.2)
-      coderay (~> 1.1)
-      method_source (~> 0.9)
     coderay (1.1.2)
     concurrent-ruby (1.0.5)
-    crm_formatter (2.4)
-      activesupport (~> 5.2, >= 5.2.0)
-      utf8_sanitizer (~> 2.0)
-    debugging (1.1.1)
-      binding.repl (~> 3.0)
-      paint (>= 0.9, < 3.0)
+    crm_formatter (2.62)
+      activesupport (~> 5.2)
+      utf8_sanitizer (~> 2.16)
+    curb (0.9.6)
     diff-lcs (1.3)
-    every_day_irb (2.0.0)
-      cd (~> 1.0)
-    fancy_irb (1.1.0)
-      paint (>= 0.9, < 3.0)
-      unicode-display_width (~> 1.1)
-    ffi (1.9.25)
-    hirb (0.7.3)
     i18n (1.0.1)
       concurrent-ruby (~> 1.0)
-    instance (0.2.0)
-    interactive_editor (0.0.11)
-      spoon (>= 0.0.1)
-    irbtools (2.2.1)
-      binding.repl (~> 3.0)
-      clipboard (~> 1.1)
-      code (~> 0.9)
-      coderay (~> 1.1)
-      debugging (~> 1.1)
-      every_day_irb (~> 2.0)
-      fancy_irb (~> 1.1)
-      hirb (~> 0.7, >= 0.7.3)
-      instance (~> 0.2)
-      interactive_editor (~> 0.0, >= 0.0.10)
-      method_locator (~> 0.0, >= 0.0.4)
-      methodfinder (~> 2.0)
-      ori (~> 0.1.0)
-      os
-      paint (>= 0.9, < 3.0)
-      ruby_engine (~> 1.0)
-      ruby_info (~> 1.0)
-      ruby_version (~> 1.0)
-      wirb (~> 2.0)
-    json (2.1.0)
-    method_locator (0.0.4)
     method_source (0.9.0)
-    methodfinder (2.2.1)
     minitest (5.11.3)
-    ori (0.1.0)
-    os (1.0.0)
-    paint (2.0.1)
-    parallel (1.12.1)
-    parser (2.5.1.0)
-      ast (~> 2.4.0)
-    powerpack (0.1.2)
+    net-ping (1.7.8)
     pry (0.11.3)
       coderay (~> 1.1.0)
       method_source (~> 0.9.0)
-    rainbow (3.0.0)
     rake (12.3.1)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
@@ -96,42 +45,20 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    rubocop (0.56.0)
-      parallel (~> 1.10)
-      parser (>= 2.5)
-      powerpack (~> 0.1)
-      rainbow (>= 2.2.2, < 4.0)
-      ruby-progressbar (~> 1.7)
-      unicode-display_width (~> 1.0, >= 1.0.1)
-    ruby-beautify (0.97.4)
-    ruby-progressbar (1.9.0)
-    ruby_engine (1.0.1)
-    ruby_info (1.0.1)
-    ruby_version (1.0.1)
-    spoon (0.0.6)
-      ffi
     thread_safe (0.3.6)
     tzinfo (1.2.5)
       thread_safe (~> 0.1)
-    unicode-display_width (1.4.0)
-    utf8_sanitizer (2.0)
-      activesupport (~> 5.2, >= 5.2.0)
-    wirb (2.1.2)
-      paint (>= 0.9, < 3.0)
+    utf8_sanitizer (2.16)
+      activesupport (~> 5.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.16, >= 1.16.2)
-  byebug (~> 10.0, >= 10.0.2)
-  class_indexer (~> 0.3.0)
-  irbtools (~> 2.2, >= 2.2.1)
   pry (~> 0.11.3)
   rake (~> 12.3, >= 12.3.1)
   rspec (~> 3.7)
-  rubocop (~> 0.56.0)
-  ruby-beautify (~> 0.97.4)
   url_verifier!
 
 BUNDLED WITH

data/README.md

@@ -1,5 +1,25 @@
 # UrlVerifier
-## Verify url validity and follow redirects.  Currently in Beta Version while Building.
+
+[![Gem Version](https://badge.fury.io/rb/url_verifier.svg)](https://badge.fury.io/rb/url_verifier)
+[![MIT License](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+
+### Format, Verify & Follow URL redirects with detailed reports.
+
+Following url redirects can sometimes take a few minutes and often creates various exceptions.  UrlVerifier is built with exceptional error handling, reformatting, and optional time limits you can set; default is set to 60 sec limit, but typically only takes 5-10 seconds per url.  UrlVerifier has been developed and improved upon for several years in an enterprise level app and is now available as an open source gem.  It is perfect for high-volume, yet smooth, uninterrupted url formatting and verification.
+
+Example:
+
+```
+{
+  url: 'blackwellford.com/staff',
+  verified_url: 'https://www.blackwellford.com',
+  url_path: '/staff',
+  response_code:'200',
+  url_redirected: true,
+  url_sts: 'Valid'
+}
+```
 
 ## Installation
 
@@ -19,13 +39,137 @@ Or install it yourself as:
 
 ## Usage
 
-Coming soon.  Currently in Beta Version while Developing.
+### Available Methods
+
+#### 1. Verify one URL as a string:
+
+```
+verifier = UrlVerifier::RunCurler.new
+verified_url_hashes = verifier.verify_urls(array_of_urls)
+```
+
+
+#### 2. Verify Array of URL strings:
+
+```
+verifier = UrlVerifier::RunCurler.new
+verified_url_hashes = verifier.verify_url('example.com')
+```
+
+
+### Usage Example
+
+```
+array_of_urls = %w[blackwellford.com/staff www.mccrea.subaru.com/inventory www.sofake.sofake https://www.century1chevy.com https://www.mccreasubaru.com]
+
+args = { timeout_limit: 30 }
+verifier = UrlVerifier::RunCurler.new(args)
+verified_url_hashes = verifier.verify_urls(array_of_urls)
+```
+Note: `:timeout_limit` default is 60 seconds per url, but most urls take under 10 sec each.  You can override the default by passing in your desired limit.  If 60 seconds is fine, no need to pass any args.  You could simply instantiate like this:
+
+```
+verifier = UrlVerifier::RunCurler.new
+verified_url_hashes = verifier.verify_urls(array_of_urls)
+```
+
+Returns Data in Hash Format with detailed report:
+
+Notice the URLs in the input array above were NOT uniformly formatted.  UrlVerifier leverages the `utf8_sanitizer gem` and `crm_formatter gem` to pre-format the URLs before verifying, so the results will be uniformly formatted, like in the hashes below `:url_f`.  
+
+`:verified_url` is the final verified URL.  `:url_redirected` indicates that the verified URL is different than `:url_f`.  
+
+If `url_sts: 'Invalid'`, `:wx_date` will be timestamped, which helps keep track of when it became invalid, incase you are running period database URL verifications and want to include these details in your reports.
+
+`:response_code` in the 200's is ideal.  If it has recently been forwarded it will be in the 300's, and 400's indicates an issue with the URL domain or server.  Some 400's could be run later when they resolve their issues, so don't always give up on them.  
+
+Here is a reference guide: [List of HTTP status codes](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes)
+
+```
+verified_url_hashes = [
+  {
+    :web_status=>"formatted",
+    :url=>"blackwellford.com/staff",
+    :url_f=>"http://www.blackwellford.com",
+    :url_path=>"/staff",
+    :web_neg=>nil,
+    :verified_url=>"https://www.blackwellford.com",
+    :url_redirected=>true,
+    :response_code=>"200",
+    :url_sts=>"Valid",
+    :url_date=>2018-07-02 09:16:19 -0500,
+    :wx_date=>nil,
+    :timeout=>0
+  },
+  {
+    :web_status=>"formatted",
+    :url=>"www.mccrea.subaru.com/inventory",
+    :url_f=>"http://www.mccrea.subaru.com",
+    :url_path=>nil,
+    :web_neg=>nil,
+    :verified_url=>"https://www.mccreasubaru.com",
+    :url_redirected=>true,
+    :response_code=>"200",
+    :url_sts=>"Valid",
+    :url_date=>2018-07-02 09:16:38 -0500,
+    :wx_date=>nil,
+    :timeout=>0
+  },
+  {
+    :web_status=>"invalid",
+    :url=>"www.sofake.sofake",
+    :url_f=>nil,
+    :url_path=>nil,
+    :web_neg=>"error: ext.invalid [sofake]",
+    :verified_url=>nil,
+    :url_redirected=>false,
+    :response_code=>nil,
+    :url_sts=>"Invalid",
+    :url_date=>2018-07-02 09:16:58 -0500,
+    :wx_date=>2018-07-02 09:16:58 -0500,
+    :timeout=>nil
+  },
+  {
+    :web_status=>"unchanged",
+    :url=>"https://www.century1chevy.com",
+    :url_f=>"https://www.century1chevy.com",
+    :url_path=>nil,
+    :web_neg=>nil,
+    :verified_url=>"http://www.centurychevy.com",
+    :url_redirected=>true,
+    :response_code=>"405",
+    :url_sts=>"Valid",
+    :url_date=>2018-07-02 09:16:58 -0500,
+    :wx_date=>nil,
+    :timeout=>0
+  },
+  {
+    :web_status=>"unchanged",
+    :url=>"https://www.mccreasubaru.com",
+    :url_f=>"https://www.mccreasubaru.com",
+    :url_path=>nil,
+    :web_neg=>nil,
+    :verified_url=>"https://www.mccreasubaru.com",
+    :url_redirected=>false,
+    :response_code=>"200",
+    :url_sts=>"Valid",
+    :url_date=>2018-07-02 09:16:59 -0500,
+    :wx_date=>nil,
+    :timeout=>0
+  }
+]
+```
+
+## Author
+
+Adam J Booth  - [4rlm](https://github.com/4rlm)
+
 
 ## Development
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+After checking out the repo, verify `bin/setup` to install dependencies. Then, verify `rake spec` to verify the tests. You can also verify `bin/console` for an interactive prompt that will allow you to experiment.
 
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+To install this gem onto your local machine, verify `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then verify `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 

data/Rakefile

@@ -1,6 +1,34 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require 'url_verifier'
+
 
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+task :test => :spec
+
+###################
+task :console do
+  require 'irb'
+  require 'irb/completion'
+  require 'url_verifier'
+  require "active_support/all"
+  ARGV.clear
+
+  verified_urls = run_verify_urls
+  # binding.pry
+
+  IRB.start
+end
+
+
+def run_verify_urls
+  # urls = %w[https://www.century1chevy.com www.sofake.sofake http://www.mccrea.subaru.com blackwellford.com minooka.subaru.com texarkana.mercedesdealer.com www.bobilya.com hammondautoplex.com www.harbinfordscottsboro.net http://www.lancaster.subaru.com loufusz.subaru.com www.mastro.subaru.com www.muller.subaru.com reinekefamilydealerships.com]
+
+  array_of_urls = %w[blackwellford.com/staff www.mccrea.subaru.com/inventory www.sofake.sofake https://www.century1chevy.com https://www.mccreasubaru.com]
+
+  args = { timeout_limit: 60 }
+  verifier = UrlVerifier::RunCurler.new(args)
+  verified_url_hashes = verifier.verify_urls(array_of_urls)
+end

data/lib/url_verifier/curler.rb

@@ -0,0 +1,102 @@
+# require 'check_int'
+# require 'timeout'
+# require 'net/ping'
+# gem 'net-ping', '~> 1.7', '>= 1.7.8'
+# gem 'curb', '~> 0.9.3'
+
+module UrlVerifier
+  class Curler
+
+    def initialize
+      @web_formatter = CrmFormatter::Web.new
+      @ran_again = false
+    end
+
+    def start_curl(url, timeout)
+      curl_result = { verified_url: nil, response_code: nil, curl_err: nil }
+      if url.present?
+        result = nil
+
+        begin # Curl Exception Handling
+          begin # Timeout Exception Handling
+            Timeout.timeout(timeout) do
+              pre_curl_msg(url, timeout)
+
+              result = Curl::Easy.perform(url) do |curl|
+                curl.follow_location = true
+                curl.useragent = "curb"
+                curl.connect_timeout = timeout
+                curl.enable_cookies = true
+                curl.head = true #testing - new
+              end # result
+
+              curl_result[:response_code] = result&.response_code.to_s
+              web_hsh = @web_formatter.format_url(result&.last_effective_url)
+
+              url_f = web_hsh[:url_f]
+              curl_result[:verified_url] = url_f if url_f.present?
+            end
+
+          rescue Timeout::Error # Timeout Exception Handling
+            curl_result[:curl_err] = "Error: Timeout"
+          end
+
+        # rescue LoadError => e  # Curl Exception Handling
+        rescue StandardError => e
+          curl_err = error_parser("Error: #{$!.message}")
+          # CheckInt.new.check_int if curl_err.include?('TCP')
+          curl_result[:curl_err] = curl_err
+        end
+      else ## If no url present?
+        curl_result[:curl_err] = 'URL Nil'
+      end
+
+      curl_result = run_again(curl_result, url, timeout)
+      curl_result
+    end
+
+    def run_again(curl_result, url, timeout)
+      if curl_result[:curl_err].present?
+        if @ran_again == false
+          @ran_again = true
+          url = https_to_http(url)
+          curl_result = start_curl(url, timeout)
+        else
+          @ran_again = false
+        end
+      else
+        @ran_again = false
+      end
+      curl_result
+    end
+
+    def https_to_http(url)
+      url = url.gsub('https://', 'http://')
+    end
+
+    def pre_curl_msg(url, timeout)
+      puts "\n\n#{'='*40}\nVERIFYING: #{url}\nMax Wait Set: #{timeout} Seconds\n\n"
+    end
+
+    def error_parser(curl_err)
+      if curl_err.include?("Couldn't connect to server")
+        curl_err = "Error: Expired Url"
+      elsif curl_err.include?("SSL connect error")
+        curl_err = "Error: SSL"
+      elsif curl_err.include?("Couldn't resolve host name")
+        curl_err = "Error: Host"
+      elsif curl_err.include?("Peer certificate")
+        curl_err = "Error: Certificate"
+      elsif curl_err.include?("Failure when receiving data")
+        curl_err = "Error: Transfer"
+      elsif curl_err.include?("TCP connection")
+        curl_err = "Error: TCP"
+      else
+        curl_err = "Error: Undefined"
+      end
+
+      curl_err
+    end
+
+  end
+end

data/lib/url_verifier/iterate.rb

@@ -0,0 +1,47 @@
+
+
+## INACTIVE - CONSIDER LATER FOR ADVANCED OPTION ###
+module UrlVerifier
+  module Iterate
+    #
+    # def iterate_query(query)
+    #   # Call: VerUrl.new.start_ver_url
+    #   # Delayed::Worker.max_run_time = 2.seconds
+    #   query.in_groups(@group_count).each do |batch_of_ids|
+    #     @query_count -= batch_of_ids&.count
+    #     pause_iteration
+    #     format_query_results(batch_of_ids)
+    #   end
+    # end
+    #
+    #
+    # def pause_iteration
+    #   until get_dj_count <= @dj_count_limit
+    #     puts "\nCurrent Process: #{@current_process}"
+    #     puts "Waiting on #{get_dj_count} Queued Jobs | Queue Limit: #{@dj_count_limit}"
+    #     puts "Total Query Count: #{@query_count}, Refresh Rate: #{@dj_refresh_interval} seconds\n\n"
+    #     sleep(@dj_refresh_interval)
+    #   end
+    # end
+    #
+    #
+    # def get_dj_count
+    #   Delayed::Job.all.count
+    # end
+    #
+    #
+    # def format_query_results(batch_of_ids)
+    #   batch_of_ids.in_groups(@dj_workers).each do |group_of_ids|
+    #     standard_iterator(group_of_ids)
+    #     # delay.standard_iterator(group_of_ids)
+    #   end
+    # end
+    #
+    #
+    # def standard_iterator(ids)
+    #   # ids.each { |id| template_starter(id) if id }
+    #   ids.each { |id| delay(priority: 10).template_starter(id) if id }
+    # end
+
+  end
+end

data/lib/url_verifier/query.rb

@@ -0,0 +1,130 @@
+
+## INACTIVE - CONSIDER LATER FOR ADVANCED OPTION ###
+
+module UrlVerifier
+  class Query
+    # include Curler
+
+  # def initialize
+  #   @dj_on = false
+  #   @dj_count_limit = 0
+  #   @dj_workers = 3
+  #   @obj_in_grp = 10
+  #   @dj_refresh_interval = 10
+  #   @db_timeout_limit = 120
+  #   @cut_off = 10.days.ago
+  #   @formatter = Formatter.new
+  #   @mig = Mig.new
+  #   @current_process = "VerUrl"
+  # end
+
+
+  # def get_query
+  #   err_sts_arr = ['Error: Timeout', 'Error: Host', 'Error: TCP']
+  #   query = Web.select(:id)
+  #     .where(url_sts: ['Valid', nil])
+  #     .where('url_date < ? OR url_date IS NULL', @cut_off)
+  #       .or(Web.select(:id)
+  #         .where(url_sts: err_sts_arr)
+  #         .where('timeout < ?', @db_timeout_limit)
+  #       ).order("timeout ASC").pluck(:id)
+  # end
+
+
+  # def start_ver_url
+  #   query = get_query[0..20]
+  #   while query.any?
+  #     setup_iterator(query)
+  #     query = get_query[0..20]
+  #     break unless query.any?
+  #   end
+  # end
+
+
+  # def setup_iterator(query)
+  #   @query_count = query.count
+  #   (@query_count & @query_count > @obj_in_grp) ? @group_count = (@query_count / @obj_in_grp) : @group_count = 2
+  #   @dj_on ? iterate_query(query) : query.each { |id| template_starter(id) }
+  # end
+
+  #
+  # #Call: VerUrl.new.start_ver_url
+  # def template_starter(id)
+  #   web = Web.find(id)
+  #   web_url = web.url
+  #   db_timeout = web.timeout
+  #   db_timeout == 0 ? timeout = @dj_refresh_interval : timeout = (db_timeout * 3)
+  #
+  #   begin
+  #     url_f = @formatter.format_url(web_url)
+  #     if !url_f.present?
+  #         web.update!(response_code: nil, url_sts: 'Invalid', url_date: Time.now, wx_date: Time.now, timeout: timeout)
+  #     elsif url_f != web_url
+  #       fwd_web_obj = Web.find_by(url: url_f)
+  #       AssocWeb.transfer_web_associations(web, fwd_web_obj) if fwd_web_obj&.url.present?
+  #     end
+  #
+  #     ####### CURL-BEGINS - FORMATTED URLS ONLY!! #######
+  #     #Call: VerUrl.new.start_ver_url
+  #     if url_f.present?
+  #       curl_hsh = start_curl(url_f, timeout)
+  #       curl_err = curl_hsh[:curl_err]
+  #       if !curl_err.present?
+  #         update_db(web, curl_hsh)
+  #       elsif curl_err == "Error: Timeout" || curl_err == "Error: Host"
+  #         puts "curl_err: #{curl_err}"
+  #         web.update!(url_sts: curl_err, url_date: Time.now, timeout: timeout)
+  #       else
+  #         web.update!(response_code: nil, url_sts: curl_err, url_date: Time.now, timeout: 0)
+  #       end
+  #     end
+  #   rescue
+  #     web = delete_duplicates(web_url)
+  #   end
+  #
+  # end
+
+
+  # def update_db(web, curl_hsh)
+  #   web_url = web.url
+  #   response_code = curl_hsh[:response_code]
+  #   last_effective_url = curl_hsh[:last_effective_url]
+  #   print_curl_results(web_url, last_effective_url, response_code)
+  #
+  #   begin
+  #     if !last_effective_url.present?
+  #       web.update!(url_sts: "Error: Nil", response_code: nil, url_date: Time.now, timeout: 0)
+  #     elsif last_effective_url.present? && last_effective_url == web_url
+  #         web.update!(url_sts: 'Valid', response_code: response_code, url_date: Time.now, timeout: 0)
+  #     elsif last_effective_url.present? && last_effective_url != web_url
+  #       fwd_web_obj = Web.find_or_create_by(url: last_effective_url)
+  #       AssocWeb.transfer_web_associations(web, fwd_web_obj) if fwd_web_obj&.url.present?
+  #     end
+  #   rescue
+  #     original_web_obj = delete_duplicates(original_web_obj.url)
+  #   end
+  # end
+
+
+
+  # def delete_duplicates(web_url)
+  #   duplicate_web_objs = Web.where(url: web_url).order("id ASC")
+  #   duplicate_web_objs.last.destroy if duplicate_web_objs.count > 1
+  #   non_duplicate_web_obj = duplicate_web_objs.first
+  #   non_duplicate_web_obj
+  # end
+
+
+  # def print_curl_results(web_url, last_effective_url, response_code)
+  #   puts "=================================="
+  #   puts "W: #{web_url}"
+  #   puts "C: #{last_effective_url}"
+  #   puts "S: #{response_code}\n\n\n"
+  # end
+
+
+
+
+  end
+
+end

data/lib/url_verifier/run_curler.rb

@@ -0,0 +1,129 @@
+
+module UrlVerifier
+  class RunCurler
+
+    def initialize(args={})
+      @timeout_limit = args.fetch(:timeout_limit, 60)
+      @web_formatter = CrmFormatter::Web.new
+      @curler = UrlVerifier::Curler.new
+
+      # @dj_on = false
+      # @dj_count_limit = 0
+      # @dj_workers = 3
+      # @obj_in_grp = 10
+      # @dj_refresh_interval = 10
+      # @cut_off = 10.days.ago
+      # @current_process = "VerUrl"
+      # @url_hash = {}
+    end
+
+    def verify_urls(urls=[])
+      url_hashes = urls.map { |url| verify_url(url) }
+    end
+
+    def verify_url(url)
+      url_hash = @web_formatter.format_url(url)
+      url_hash = merge_url_hash(url_hash)
+
+      if url_hash[:url_f].present?
+        url_hash = send_to_curl(url_hash)
+        url_hash = check_for_redirect(url_hash)
+      else
+        url_hash = evaluate_formatted_url(url_hash)
+      end
+
+      puts url_hash.inspect
+      url_hash
+    end
+
+    def merge_url_hash(url_hash)
+      url_hash_fields = {
+        verified_url: nil,
+        url_redirected: false,
+        response_code: nil,
+        url_sts: nil,
+        url_date: Time.now,
+        wx_date: nil,
+        timeout: 0
+      }
+      url_hash.merge(url_hash_fields)
+    end
+
+    def evaluate_formatted_url(url_hash)
+      url_hash = url_hash.merge({url_sts: 'Invalid', wx_date: Time.now })
+    end
+
+    def check_for_redirect(url_hash)
+      ver = url_hash[:verified_url]
+      form = url_hash[:url_f]
+      url_hash[:url_redirected] = ver.present? && ver != form
+      url_hash
+    end
+
+    def send_to_curl(url_hash)
+      curl_result = @curler.start_curl(url_hash[:url_f], @timeout_limit)
+      curl_err = curl_result[:curl_err]
+
+      if curl_err.present?
+        url_hash = url_hash.merge({ url_sts: curl_err, timeout: evaluate_curl_err(curl_err) })
+      else
+        url_hash = process_valid_curl_response(url_hash, curl_result)
+      end
+
+      url_hash
+    end
+
+    def evaluate_curl_err(curl_err)
+      curl_err == "Error: Timeout" || curl_err == "Error: Host" ? timeout = @timeout_limit : timeout = 0
+    end
+
+    def process_valid_curl_response(url_hash, curl_result)
+      curl_result[:verified_url].present? ? url_sts = 'Valid' : url_sts = "Error: Nil"
+
+      valid_hash = {
+        verified_url: curl_result[:verified_url],
+        url_sts: url_sts,
+        response_code: curl_result[:response_code],
+        timeout: 0
+      }
+      url_hash = url_hash.merge(valid_hash)
+    end
+
+
+    ##### ADVANCED USAGE - REVISIT LATER #####
+    # def get_query
+    #   err_sts_arr = ['Error: Timeout', 'Error: Host', 'Error: TCP']
+    #   query = Web.select(:id)
+    #     .where(url_sts: ['Valid', nil])
+    #     .where('url_date < ? OR url_date IS NULL', @cut_off)
+    #       .or(Web.select(:id)
+    #         .where(url_sts: err_sts_arr)
+    #         .where('timeout < ?', @timeout_limit)
+    #       ).order("timeout ASC").pluck(:id)
+    # end
+
+    # def start_ver_url
+    #   query = get_query[0..20]
+    #   while query.any?
+    #     setup_iterator(query)
+    #     query = get_query[0..20]
+    #     break unless query.any?
+    #   end
+    # end
+
+    # def setup_iterator(query)
+    #   @query_count = query.count
+    #   (@query_count & @query_count > @obj_in_grp) ? @group_count = (@query_count / @obj_in_grp) : @group_count = 2
+    #   @dj_on ? iterate_query(query) : query.each { |id| template_starter(id) }
+    # end
+
+    # def template_starter(url)
+      # web = Web.find(id)
+      # web_url = web.url
+      # db_timeout = web.timeout
+      # db_timeout == 0 ? timeout = @dj_refresh_interval : timeout = (db_timeout * 3)
+    # end
+
+  end
+
+end

data/lib/url_verifier/version.rb

@@ -1,3 +1,3 @@
 module UrlVerifier
-  VERSION = "0.0.1.pre.rc.01"
+  VERSION = "2.10"
 end

data/lib/url_verifier.rb

@@ -1,4 +1,15 @@
-require "url_verifier/version"
+require 'url_verifier/version'
+require 'url_verifier/curler'
+require 'url_verifier/run_curler'
+
+require 'pry'
+require 'utf8_sanitizer'
+require 'crm_formatter'
+
+require 'timeout'
+require 'net/ping'
+require 'curb'
+
 
 module UrlVerifier
   # Your code goes here...

data/url_verifier.gemspec

@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
   spec.authors       = ["Adam Booth"]
   spec.email         = ["4rlm@protonmail.ch"]
 
-  spec.summary       = %q{Beta Version - Verify url validity and follow redirects. Note: Reserving Namespace for Gem in Development.}
-  spec.description   = %q{Beta Version - Verify url validity and follow redirects. Note: Reserving Namespace for Gem in Development. - Coming Soon.}
+  spec.summary       = %q{Format, Verify & Follow URL redirects with detailed reports. (url: 'blackwellford.com/staff', verified_url: 'https://www.blackwellford.com', response_code:'200', url_redirected: true)  }
+  spec.description   = %q{Format, Verify & Follow URL redirects with detailed reports. (Ex => url: 'blackwellford.com/staff', verified_url: 'https://www.blackwellford.com', response_code:'200', url_redirected: true, url_sts: 'Valid', url_path: '/staff'). Following url redirects can sometimes take a few minutes and often creates various exceptions.  UrlVerifier is built with exceptional error handling, reformatting, and optional time limits you can set; default is set to 60 sec limit, but typically only takes 5-10 seconds per url.  UrlVerifier has been developed and improved upon for several years in an enterprise level app and is now available as an open source gem.  It is perfect for high-volume, yet smooth, uninterrupted url formatting and verification.}
   spec.homepage      = 'https://github.com/4rlm/url_verifier'
   spec.license       = "MIT"
 
@@ -36,18 +36,43 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.required_ruby_version = '~> 2.5.1'
-  spec.add_dependency 'activesupport', '~> 5.2', '>= 5.2.0'
+  spec.add_dependency 'activesupport', '~> 5.2'
+  spec.add_dependency 'utf8_sanitizer', '~> 2.16'
+  spec.add_dependency 'crm_formatter', '~> 2.62'
 
-  spec.add_dependency "utf8_sanitizer", "~> 2.0"
-  spec.add_dependency "crm_formatter", "~> 2.4"
+  spec.add_dependency 'net-ping', '~> 1.7', '>= 1.7.8'
+  spec.add_dependency 'curb', '~> 0.9.6'
 
+  # spec.add_dependency "activesupport-inflector", ['~> 0.1.0']
   spec.add_development_dependency 'bundler', '~> 1.16', '>= 1.16.2'
-  spec.add_development_dependency 'byebug', '~> 10.0', '>= 10.0.2'
-  spec.add_development_dependency 'class_indexer', '~> 0.3.0'
-  spec.add_development_dependency 'irbtools', '~> 2.2', '>= 2.2.1'
   spec.add_development_dependency 'pry', '~> 0.11.3'
   spec.add_development_dependency 'rake', '~> 12.3', '>= 12.3.1'
   spec.add_development_dependency 'rspec', '~> 3.7'
-  spec.add_development_dependency 'rubocop', '~> 0.56.0'
-  spec.add_development_dependency 'ruby-beautify', '~> 0.97.4'
+  # spec.add_development_dependency 'byebug', '~> 10.0', '>= 10.0.2'
+  # spec.add_development_dependency 'class_indexer', '~> 0.3.0'
+  # spec.add_development_dependency 'irbtools', '~> 2.2', '>= 2.2.1'
+  # spec.add_development_dependency 'rubocop', '~> 0.56.0'
+  # spec.add_development_dependency 'ruby-beautify', '~> 0.97.4'
+  # spec.add_runtime_dependency 'library', '~> 2.2'
+  # spec.add_dependency 'activerecord', '>= 3.0'
+  # spec.add_dependency 'actionpack', '>= 3.0'
+  # spec.add_dependency 'polyamorous', '~> 1.3.2'
+  # spec.add_development_dependency 'machinist', '~> 1.0.6'
+  # spec.add_development_dependency 'faker', '~> 0.9.5'
+  # spec.add_development_dependency 'sqlite3', '~> 1.3.3'
+  # spec.add_development_dependency 'pg', '~> 0.21'
+  # spec.add_development_dependency 'mysql2', '0.3.20'
+
+  # spec.requirements << 'libmagick, v6.0'
+  # spec.requirements << 'A good graphics card'
+  # # This gem will work with 1.8.6 or greater...
+  # spec.required_ruby_version = '>= 1.8.6'
+  #
+  # # Only with ruby 2.0.x
+  # spec.required_ruby_version = '~> 2.0'
+  #
+  # # Only with ruby between 2.2.0 and 2.2.2
+  # spec.required_ruby_version = ['>= 2.2.0', '< 2.2.3']
+
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: url_verifier
 version: !ruby/object:Gem::Version
-  version: 0.0.1.pre.rc.01
+  version: '2.10'
 platform: ruby
 authors:
 - Adam Booth
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-27 00:00:00.000000000 Z
+date: 2018-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -17,9 +17,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -27,111 +24,88 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.2'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 5.2.0
 - !ruby/object:Gem::Dependency
   name: utf8_sanitizer
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.16'
 - !ruby/object:Gem::Dependency
   name: crm_formatter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
+        version: '2.62'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.4'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.16'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.16.2
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.16'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.16.2
+        version: '2.62'
 - !ruby/object:Gem::Dependency
-  name: byebug
+  name: net-ping
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 10.0.2
-  type: :development
+        version: 1.7.8
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.7'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 10.0.2
+        version: 1.7.8
 - !ruby/object:Gem::Dependency
-  name: class_indexer
+  name: curb
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
-  type: :development
+        version: 0.9.6
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: 0.9.6
 - !ruby/object:Gem::Dependency
-  name: irbtools
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '1.16'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: 1.16.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.2'
+        version: '1.16'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 2.2.1
+        version: 1.16.2
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
@@ -180,36 +154,16 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.7'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.56.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.56.0
-- !ruby/object:Gem::Dependency
-  name: ruby-beautify
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.97.4
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 0.97.4
-description: 'Beta Version - Verify url validity and follow redirects. Note: Reserving
-  Namespace for Gem in Development. - Coming Soon.'
+description: 'Format, Verify & Follow URL redirects with detailed reports. (Ex =>
+  url: ''blackwellford.com/staff'', verified_url: ''https://www.blackwellford.com'',
+  response_code:''200'', url_redirected: true, url_sts: ''Valid'', url_path: ''/staff'').
+  Following url redirects can sometimes take a few minutes and often creates various
+  exceptions.  UrlVerifier is built with exceptional error handling, reformatting,
+  and optional time limits you can set; default is set to 60 sec limit, but typically
+  only takes 5-10 seconds per url.  UrlVerifier has been developed and improved upon
+  for several years in an enterprise level app and is now available as an open source
+  gem.  It is perfect for high-volume, yet smooth, uninterrupted url formatting and
+  verification.'
 email:
 - 4rlm@protonmail.ch
 executables: []
@@ -228,6 +182,10 @@ files:
 - bin/console
 - bin/setup
 - lib/url_verifier.rb
+- lib/url_verifier/curler.rb
+- lib/url_verifier/iterate.rb
+- lib/url_verifier/query.rb
+- lib/url_verifier/run_curler.rb
 - lib/url_verifier/version.rb
 - url_verifier.gemspec
 homepage: https://github.com/4rlm/url_verifier
@@ -246,14 +204,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.1
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: 'Beta Version - Verify url validity and follow redirects. Note: Reserving
-  Namespace for Gem in Development.'
+summary: 'Format, Verify & Follow URL redirects with detailed reports. (url: ''blackwellford.com/staff'',
+  verified_url: ''https://www.blackwellford.com'', response_code:''200'', url_redirected:
+  true)'
 test_files: []