deadfinder 1.0.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3a2e86be2607ea1af0d52ca1ed57055869cb457d58aa0b3773154d8de15f7a08
-  data.tar.gz: ffb4fa247f7f388c9e5f71144f8b9728118627feead5159380bac65d2af4420f
+  metadata.gz: 248abe3bb3c321a29cb164930132c0a61ad9e3f7987a8cfb863eebe614d0350f
+  data.tar.gz: bbd932e255500d98dbd2171d5fe3fb66a820846bb5801f4b6b1b1b74905675e7
 SHA512:
-  metadata.gz: 237f99c06741f5b509af6ede98be469e20d9a60f7be2a5d615568eb7707686a40998ee49ce41143cb6af72beb3fe181ad9cebaed114d11f7a8e9408b3fd54fee
-  data.tar.gz: 8cfd465a45ae8b4646e82810e4aa595c809a44a11107c0845f06e11a2eddeee394bb8e01b53a412f33b13bc05aaca49bd0a890ed6a702138d3b7b327634712f1
+  metadata.gz: 4b722245eafc6373f2776db13377cbb391ab9a4c4cffb7b226e206fb8c8f7a58d1474345a7fef41b7cc14cc350ef3549db95f3359b7df7b3bcd43d0d3d580dac
+  data.tar.gz: 5b9706ea35279e5f7c881d1839bb030ab73fb412a474a3cc9ac01837a256ff121b17154246a9ef6173a7410f29e611f52cf66dff9eff7a14c7176b27a9ab00c1

data/lib/deadfinder/logger.rb

@@ -2,20 +2,20 @@
 
 require 'colorize'
 
-class Logger     
-    def self.info text
-        puts "ℹ ".colorize(:blue) + "#{text}".colorize(:light_blue)
-    end
+class Logger
+  def self.info(text)
+    puts 'ℹ '.colorize(:blue) + text.to_s.colorize(:light_blue)
+  end
 
-    def self.target text 
-        puts "► ".colorize(:green) + "#{text}".colorize(:light_green)
-    end
+  def self.target(text)
+    puts '► '.colorize(:green) + text.to_s.colorize(:light_green)
+  end
 
-    def self.sub_info text
-        puts "  ● ".colorize(:blue) + "#{text}".colorize(:light_blue)
-    end
+  def self.sub_info(text)
+    puts '  ● '.colorize(:blue) + text.to_s.colorize(:light_blue)
+  end
 
-    def self.found text
-        puts "  ✘ #{text}".colorize(:red)
-    end
-end
+  def self.found(text)
+    puts "  ✘ #{text}".colorize(:red)
+  end
+end

data/lib/deadfinder/version.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+VERSION = '1.1.1'

data/lib/deadfinder.rb

@@ -6,13 +6,14 @@ require 'open-uri'
 require 'nokogiri'
 require 'deadfinder/utils'
 require 'deadfinder/logger'
+require 'deadfinder/version'
 require 'concurrent-edge'
 require 'sitemap-parser'
 
 Channel = Concurrent::Channel
 
 class DeadFinderRunner
-  def run(target)
+  def run(target, options)
     page = Nokogiri::HTML(URI.open(target))
     nodeset = page.css('a')
     link_a = nodeset.map { |element| element['href'] }.compact
@@ -22,7 +23,7 @@ class DeadFinderRunner
     jobs    = Channel.new(buffer: :buffered, capacity: 100)
     results = Channel.new(buffer: :buffered, capacity: 100)
 
-    (1..20).each do |w|
+    (1..options['concurrency']).each do |w|
       Channel.go { worker(w, jobs, results) }
     end
 
@@ -32,78 +33,82 @@ class DeadFinderRunner
     end
     jobs.close
 
-    (1..link_a.uniq.length).each do 
+    (1..link_a.uniq.length).each do
       ~results
     end
     Logger.sub_info 'Done'
   end
 
-  def worker(id, jobs, results)
+  def worker(_id, jobs, results)
     jobs.each do |j|
       begin
         URI.open(j)
-      rescue => exception
-        if exception.to_s.include? '404 Not Found'
-          Logger.found "[#{exception}] #{j}"
-        end
-      end  
+      rescue StandardError => e
+        Logger.found "[#{e}] #{j}" if e.to_s.include? '404 Not Found'
+      end
       results << j
     end
   end
 end
 
-def run_pipe
+def run_pipe(options)
   app = DeadFinderRunner.new
   while $stdin.gets
     target = $LAST_READ_LINE.gsub("\n", '')
-    app.run target
+    app.run target, options
   end
 end
 
-def run_file(filename)
+def run_file(filename, options)
   app = DeadFinderRunner.new
   File.open(filename).each do |line|
     target = line.gsub("\n", '')
-    app.run target
+    app.run target, options
   end
 end
 
-def run_url(url)
+def run_url(url, options)
   app = DeadFinderRunner.new
-  app.run url
+  app.run url, options
 end
 
-def run_sitemap(sitemap_url)
+def run_sitemap(sitemap_url, options)
   app = DeadFinderRunner.new
-  sitemap = SitemapParser.new sitemap_url, {recurse: true}
+  sitemap = SitemapParser.new sitemap_url, { recurse: true }
   sitemap.to_a.each do |url|
-    app.run url
+    app.run url, options
   end
 end
 
 class DeadFinder < Thor
-  
+  class_option :concurrency, aliases: :c, default: 20, type: :numeric
+
   desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
   def pipe
     Logger.info 'Pipe mode'
-    run_pipe
+    run_pipe options
   end
 
   desc 'file', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
   def file(filename)
     Logger.info 'File mode'
-    run_file filename
+    run_file filename, options
   end
 
   desc 'url', 'Scan the Single URL.'
   def url(url)
     Logger.info 'Single URL mode'
-    run_url url
+    run_url url, options
   end
 
   desc 'sitemap', 'Scan the URLs from sitemap.'
   def sitemap(sitemap)
     Logger.info 'Sitemap mode'
-    run_sitemap sitemap
+    run_sitemap sitemap, options
+  end
+
+  desc 'version', 'Show version.'
+  def version
+    Logger.info "deadfinder #{VERSION}"
   end
 end

metadata

@@ -1,16 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.1
 platform: ruby
 authors:
 - hahwul
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-24 00:00:00.000000000 Z
-dependencies: []
-description: Find dead-links (broken links)
+date: 2022-09-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby-edge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+- !ruby/object:Gem::Dependency
+  name: open-uri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: sitemap-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+description: Dead link (broken link) means a link within a web page that cannot be
+  connected. These links can have a security negative impact with SEO. This tool makes
+  it easy to identify and modify.
 email: hahwul@gmail.com
 executables:
 - deadfinder
@@ -21,11 +107,13 @@ files:
 - lib/deadfinder.rb
 - lib/deadfinder/logger.rb
 - lib/deadfinder/utils.rb
+- lib/deadfinder/version.rb
 homepage: https://www.hahwul.com
 licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/hahwul/deadfinder
 post_install_message:
 rdoc_options: []
 require_paths: