deadfinder 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 3a2e86be2607ea1af0d52ca1ed57055869cb457d58aa0b3773154d8de15f7a08
+  data.tar.gz: ffb4fa247f7f388c9e5f71144f8b9728118627feead5159380bac65d2af4420f
+SHA512:
+  metadata.gz: 237f99c06741f5b509af6ede98be469e20d9a60f7be2a5d615568eb7707686a40998ee49ce41143cb6af72beb3fe181ad9cebaed114d11f7a8e9408b3fd54fee
+  data.tar.gz: 8cfd465a45ae8b4646e82810e4aa595c809a44a11107c0845f06e11a2eddeee394bb8e01b53a412f33b13bc05aaca49bd0a890ed6a702138d3b7b327634712f1

data/bin/deadfinder

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'deadfinder'
+DeadFinder.start(ARGV)

data/lib/deadfinder/logger.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'colorize'
+
+class Logger     
+    def self.info text
+        puts "ℹ ".colorize(:blue) + "#{text}".colorize(:light_blue)
+    end
+
+    def self.target text 
+        puts "► ".colorize(:green) + "#{text}".colorize(:light_green)
+    end
+
+    def self.sub_info text
+        puts "  ● ".colorize(:blue) + "#{text}".colorize(:light_blue)
+    end
+
+    def self.found text
+        puts "  ✘ #{text}".colorize(:red)
+    end
+end

data/lib/deadfinder/utils.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+def generate_url(text, base_url)
+  node = text.to_s
+  begin
+    unless node.start_with?('http://', 'https://')
+      uri = URI(base_url)
+      if node.start_with? '//'
+        return "#{uri.scheme}:#{node}"
+      elsif node.start_with? '/'
+        return "#{uri.scheme}://#{uri.host}#{node}"
+      else
+        return "#{uri}#{node}"
+      end
+    end
+  rescue StandardError => e
+    # puts e
+  end
+  node
+end

data/lib/deadfinder.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'English'
+require 'thor'
+require 'open-uri'
+require 'nokogiri'
+require 'deadfinder/utils'
+require 'deadfinder/logger'
+require 'concurrent-edge'
+require 'sitemap-parser'
+
+Channel = Concurrent::Channel
+
+class DeadFinderRunner
+  def run(target)
+    page = Nokogiri::HTML(URI.open(target))
+    nodeset = page.css('a')
+    link_a = nodeset.map { |element| element['href'] }.compact
+    Logger.target target
+    Logger.sub_info "Found #{link_a.length} point"
+    Logger.sub_info 'Checking'
+    jobs    = Channel.new(buffer: :buffered, capacity: 100)
+    results = Channel.new(buffer: :buffered, capacity: 100)
+
+    (1..20).each do |w|
+      Channel.go { worker(w, jobs, results) }
+    end
+
+    link_a.uniq.each do |node|
+      result = generate_url node, target
+      jobs << result
+    end
+    jobs.close
+
+    (1..link_a.uniq.length).each do 
+      ~results
+    end
+    Logger.sub_info 'Done'
+  end
+
+  def worker(id, jobs, results)
+    jobs.each do |j|
+      begin
+        URI.open(j)
+      rescue => exception
+        if exception.to_s.include? '404 Not Found'
+          Logger.found "[#{exception}] #{j}"
+        end
+      end  
+      results << j
+    end
+  end
+end
+
+def run_pipe
+  app = DeadFinderRunner.new
+  while $stdin.gets
+    target = $LAST_READ_LINE.gsub("\n", '')
+    app.run target
+  end
+end
+
+def run_file(filename)
+  app = DeadFinderRunner.new
+  File.open(filename).each do |line|
+    target = line.gsub("\n", '')
+    app.run target
+  end
+end
+
+def run_url(url)
+  app = DeadFinderRunner.new
+  app.run url
+end
+
+def run_sitemap(sitemap_url)
+  app = DeadFinderRunner.new
+  sitemap = SitemapParser.new sitemap_url, {recurse: true}
+  sitemap.to_a.each do |url|
+    app.run url
+  end
+end
+
+class DeadFinder < Thor
+  
+  desc 'pipe', 'Scan the URLs from STDIN. (e.g cat urls.txt | deadfinder pipe)'
+  def pipe
+    Logger.info 'Pipe mode'
+    run_pipe
+  end
+
+  desc 'file', 'Scan the URLs from File. (e.g deadfinder file urls.txt)'
+  def file(filename)
+    Logger.info 'File mode'
+    run_file filename
+  end
+
+  desc 'url', 'Scan the Single URL.'
+  def url(url)
+    Logger.info 'Single URL mode'
+    run_url url
+  end
+
+  desc 'sitemap', 'Scan the URLs from sitemap.'
+  def sitemap(sitemap)
+    Logger.info 'Sitemap mode'
+    run_sitemap sitemap
+  end
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification
+name: deadfinder
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- hahwul
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-09-24 00:00:00.000000000 Z
+dependencies: []
+description: Find dead-links (broken links)
+email: hahwul@gmail.com
+executables:
+- deadfinder
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/deadfinder
+- lib/deadfinder.rb
+- lib/deadfinder/logger.rb
+- lib/deadfinder/utils.rb
+homepage: https://www.hahwul.com
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.3
+signing_key:
+specification_version: 4
+summary: Find dead-links (broken links)
+test_files: []