deadfinder 1.1.0 → 1.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3dd7587737b26129d64fcd4887eb39041b8abb27c4b8e1a6c125c7c128b77fb3
-  data.tar.gz: cf6b6dd2b663df8ffe3aaae9810deb39be18a15957f0b37cb3f70f7584bfb39d
+  metadata.gz: 1e08daa4368e4c3e041e642f1708619637ea618c2937c6e36f2aba841f152ec4
+  data.tar.gz: 5162fb9eb973b7638ece6bcac4e6b2c731f3a6bd654f842f1aeeeed286cee4ed
 SHA512:
-  metadata.gz: 79df6bc3e4f38da2ceda5f4704dbb69f5bb11af7b0e1637096e930c929ed065f2811ca879956ef9819f8a4afde342eb49757d5fb7bd05124d6fffd5aae0cd64e
-  data.tar.gz: 263eeae2ebaa5e4eceee0182ca1721f8d51936b3921060180cfb7efa2422e5f8f3b78f6eebc83d01214ffec83b1f6bd14ae7f79ad3e9b34306b19bef3f6f0826
+  metadata.gz: aa82c3b87b81f8f5293819dbd0e242e57e7fda14fe8820197aa4e32d6c774ea0c379adda7ef49e88865fc00f32cb5cce36c6fae711ac019e2d5be921e1747d65
+  data.tar.gz: 952de90ed7b150e106fe33ec203140c162f2f74c941400328fc3da9c41866111f940043b53d9560903fdb822337ff8973bfc70740ad937cdc04428e5598e37fb

data/lib/deadfinder/logger.rb

@@ -15,6 +15,10 @@ class Logger
     puts '  ● '.colorize(:blue) + text.to_s.colorize(:light_blue)
   end
 
+  def self.sub_done(text)
+    puts '  ✓ '.colorize(:blue) + text.to_s.colorize(:light_blue)
+  end
+
   def self.found(text)
     puts "  ✘ #{text}".colorize(:red)
   end

data/lib/deadfinder/version.rb

@@ -1 +1,3 @@
-VERSION = '1.1.0'
+# frozen_string_literal: true
+
+VERSION = '1.1.2'

data/lib/deadfinder.rb

@@ -9,22 +9,34 @@ require 'deadfinder/logger'
 require 'deadfinder/version'
 require 'concurrent-edge'
 require 'sitemap-parser'
+require 'set'
 
 Channel = Concurrent::Channel
+CacheSet = Set.new
+CacheQue = {}
 
 class DeadFinderRunner
   def run(target, options)
     page = Nokogiri::HTML(URI.open(target))
-    nodeset = page.css('a')
-    link_a = nodeset.map { |element| element['href'] }.compact
+
+    nodeset_a = page.css('a')
+    link_a = nodeset_a.map { |element| element['href'] }.compact
+    nodeset_script = page.css('script')
+    link_script = nodeset_script.map { |element| element['src'] }.compact
+    nodeset_link = page.css('link')
+    link_link = nodeset_link.map { |element| element['href'] }.compact
+
+    link_merged = []
+    link_merged = link_merged.concat link_a, link_script, link_link
+
     Logger.target target
-    Logger.sub_info "Found #{link_a.length} point"
+    Logger.sub_info "Found #{link_merged.length} point. [a:#{link_a.length}/s:#{link_script.length}/l:#{link_link.length}]"
     Logger.sub_info 'Checking'
-    jobs    = Channel.new(buffer: :buffered, capacity: 100)
-    results = Channel.new(buffer: :buffered, capacity: 100)
+    jobs    = Channel.new(buffer: :buffered, capacity: 1000)
+    results = Channel.new(buffer: :buffered, capacity: 1000)
 
     (1..options['concurrency']).each do |w|
-      Channel.go { worker(w, jobs, results) }
+      Channel.go { worker(w, jobs, results, options) }
     end
 
     link_a.uniq.each do |node|
@@ -36,17 +48,27 @@ class DeadFinderRunner
     (1..link_a.uniq.length).each do
       ~results
     end
-    Logger.sub_info 'Done'
+    Logger.sub_done 'Done'
   end
 
-  def worker(_id, jobs, results)
+  def worker(_id, jobs, results, options)
     jobs.each do |j|
-      begin
-        URI.open(j)
-      rescue StandardError => e
-        Logger.found "[#{e}] #{j}" if e.to_s.include? '404 Not Found'
+      if !CacheSet.include? j
+        CacheSet.add j
+        begin
+          CacheQue[j] = true
+          URI.open(j, :read_timeout => options['timeout'])
+        rescue StandardError => e
+          Logger.found "[#{e}] #{j}" if e.to_s.include? '404 Not Found'
+          CacheQue[j] = false
+        end
+        results << j
+      else 
+        if !CacheQue[j] 
+          Logger.found "[404 Not Found] #{j}"
+        end
+        results << j
       end
-      results << j
     end
   end
 end
@@ -59,7 +81,7 @@ def run_pipe(options)
   end
 end
 
-def run_file(filename,options)
+def run_file(filename, options)
   app = DeadFinderRunner.new
   File.open(filename).each do |line|
     target = line.gsub("\n", '')
@@ -67,12 +89,12 @@ def run_file(filename,options)
   end
 end
 
-def run_url(url,options)
+def run_url(url, options)
   app = DeadFinderRunner.new
   app.run url, options
 end
 
-def run_sitemap(sitemap_url,options)
+def run_sitemap(sitemap_url, options)
   app = DeadFinderRunner.new
   sitemap = SitemapParser.new sitemap_url, { recurse: true }
   sitemap.to_a.each do |url|
@@ -82,6 +104,7 @@ end
 
 class DeadFinder < Thor
   class_option :concurrency, aliases: :c, default: 20, type: :numeric
+  class_option :timeout, aliases: :t, default: 10, type: :numeric
 
   desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
   def pipe

metadata

@@ -1,16 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: deadfinder
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.2
 platform: ruby
 authors:
 - hahwul
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-25 00:00:00.000000000 Z
-dependencies: []
-description: Find dead-links (broken links)
+date: 2022-09-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby-edge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.6.0
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.13.0
+- !ruby/object:Gem::Dependency
+  name: open-uri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: sitemap-parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.2.0
+description: Dead link (broken link) means a link within a web page that cannot be
+  connected. These links can have a security negative impact with SEO. This tool makes
+  it easy to identify and modify.
 email: hahwul@gmail.com
 executables:
 - deadfinder