finddups 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cb43cbb4ba1ea49a71b5d369cffbd0decf21f0c77764b2252d4211b7b1ccb65
-  data.tar.gz: cc04d2a23bcb19c8b8849e815e847df2472f681d46e6e7007d8cf8880acae1c6
+  metadata.gz: d1bdd387a3145a5a49a979e946be3668e2c8da432928f69f4e6e22518b2bd8ba
+  data.tar.gz: 78de1056705cfb1afee0e3c6e5dd4a54b80107e8466e875d23aaeaaeaaad1a2a
 SHA512:
-  metadata.gz: 1c46b2d368ba239f703bf20a4c5297cf03d9f9275ea5e924a87112a2d9d92d8488e4b1e37453aec7b81b4b294629d2fe39a0d6e76114981eefa8b1c23e4b61d7
-  data.tar.gz: ec31d5d9d66ac0081c8d0571a9a7783006df85f63f5a0711b1fbc10e3e2249fb2eb758804eabe6cc4db740c07c4517586686d1e9c1e5a140f9d766314ceb9dca
+  metadata.gz: 60c7b54ca17cfc307bdd9f0eb1102b2c3a8b041a4b314de6fbcd3e2f5aa61fb9a598e651a9614a03f0589f5ad2257db504890bedc1b5f1119ea41b96c9605d6d
+  data.tar.gz: 65f26108b4480c66f5b0e80f4927873438eeb2b486b6c04f5637eb06aba30b4c384e3b9584cf22a4ef03cabb2193c7b3f15271104132d92b2b62c0381a7a043a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    finddups (0.1.0)
+    finddups (0.2.0)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -1,6 +1,7 @@
 # Finddups
 
 Shows duplicate files within a list of directories and outputs as JSON.
+This is accomplished by generating a hash digest each file and comparing the hashes.
 
 ## Installation
 
@@ -9,17 +10,17 @@ Shows duplicate files within a list of directories and outputs as JSON.
 ## Usage
 
 ```
-finddups (version 0.1.0)
+finddups 0.2.0
 Usage: finddups [dirs] [options]
     -i, --ignore path                ignore paths
-        --atime                      (default) Use file access time to sort duplicates
-        --mtime                      Use file modification time to sort duplicates
-        --ctime                      Use file change time to sort duplicates
-(the time at which directory information about the file was changed, not the file itself)
-    -t, --threads threads            Number of threads to use (default 16)
     -d, --depth depth                Max depth to search
+        --[no-]cache                 Perform caching
+        --[no-]cache-to-tmp          Save cache files to /tmp/file_hashes
+    -o, --output path                Output file path
+        --alg alg                    Hashing algorithm (SHA1, MD5)
+        --[no-]ignore-empty          Ignore empty files
     -h, --help                       Show this help
-    -v                               Show version
+    -v, --version                    Show version
 ```
 
 Example:

data/exe/finddups

@@ -2,53 +2,54 @@
 
 require 'digest/sha1'
 require 'json'
+require 'yaml'
 require 'fileutils'
-require 'thread'
 require 'optparse'
 
 $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
 
 require 'finddups'
+require 'dup_finder'
 
-@options = {
-  sort: 'atime',
-  depth: Float::INFINITY,
-  ignore: [],
-  threads: 16,
-}
+options = {}
 
 optparser = OptionParser.new do |opts|
   opts.banner = <<~BANNER
-    finddups (version #{Finddups::VERSION})
+    finddups #{Finddups::VERSION}
     Usage: #{File.basename(__FILE__)} [dirs] [options]
   BANNER
 
   opts.on("-i path", "--ignore path", "ignore paths") do |path|
-    @options[:ignore] << path
+    options[:ignore] ||= []
+    options[:ignore] << path
   end
 
-  opts.on("--atime", "(default) Use file access time to sort duplicates") do
-    @options[:sort] = 'atime'
+  opts.on("-d depth", "--depth depth", "Max depth to search") do |depth|
+    options[:depth] = depth.to_i
   end
 
-  opts.on("--mtime", "Use file modification time to sort duplicates") do
-    @options[:sort] = 'mtime'
+  opts.on("--[no-]cache", "Perform caching") do |value|
+    options[:cache] = value
   end
 
-  desc = <<~DESC
-    Use file change time to sort duplicates
-    (the time at which directory information about the file was changed, not the file itself)
-  DESC
-  opts.on("--ctime", desc) do
-    @options[:sort] = 'ctime'
+  opts.on("--[no-]cache-to-tmp", "Save cache files to /tmp/file_hashes") do |value|
+    options[:cache_to_tmp] = value
   end
 
-  opts.on("-t threads", "--threads threads", "Number of threads to use (default 16)") do |threads|
-    @options[:threads] = threads.to_i
+  opts.on("-o path", "--output path", "Output file path") do |path|
+    options[:output] = path
   end
 
-  opts.on("-d depth", "--depth depth", "Max depth to search") do |depth|
-    @options[:depth] = depth.to_i
+  opts.on("--alg alg", "Hashing algorithm (SHA1, MD5)") do |alg|
+    options[:alg] = alg.downcase.to_sym
+    unless %i[md5 sha1].include?(options[:alg])
+      $stderr.puts "Unsupported algorithm: #{options[:alg]}"
+      exit 1
+    end
+  end
+
+  opts.on("--[no-]ignore-empty", "Ignore empty files") do |value|
+    options[:ignore_empty] = value
   end
 
   opts.on("-h", "--help", "Show this help") do
@@ -56,8 +57,8 @@ optparser = OptionParser.new do |opts|
     exit
   end
 
-  opts.on("-v", "Show version") do
-    puts "finddups (version #{Finddups::VERSION})"
+  opts.on("-v", "--version", "Show version") do
+    puts "finddups #{Finddups::VERSION}"
     exit
   end
 end
@@ -69,83 +70,16 @@ if ARGV.empty? || ARGV.any? { |entry| !File.directory?(entry) }
   exit 1
 end
 
-# ========
-# = Prog =
-# ========
-
-search_dirs = ARGV
-trash_dir = "/tmp/duplicates/"
-
-@mutex = Mutex.new
-@queue = []
-
-def search(directory, depth = 0)
-  # puts "Searching: #{directory}"
-
-  # Skips
-  return @duplicates if @options[:ignore].include?(File.basename(directory))
-
-  Dir.entries(directory).each do |entry|
-    next if entry.start_with?('.')
-    path = File.join(directory, entry)
-
-    if File.directory?(path)
-      if depth < @options[:depth]
-        @queue.push -> { search(path, depth + 1) }
-      end
-    elsif File.symlink?(path)
-      next
-    else
-      begin
-        digest = Digest::SHA1.hexdigest(File.read(path))
-        @mutex.synchronize do
-          @duplicates[digest] ||= []
-          @duplicates[digest] << path
-        end
-      rescue Errno::EINVAL => e
-        $stderr.puts "#{path}: #{e}"
-      end
-    end
-  end
-  @duplicates
-end
+# Prog ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-@duplicates = {}
+dup_finder = DupFinder.new(options: options)
+ARGV.each { |path| dup_finder.queue(path) }
+dup_finder.search
 
-search_dirs.each do |search_dir|
-  @queue.push -> { search(search_dir) }
-end
-
-until @queue.empty?
-  threads = []
-  @options[:threads].times do
-    _proc = @queue.shift
-    threads << Thread.new { _proc.call } if _proc
-  end
-  threads.each(&:join)
-end
-
-# Trim non dups
-@duplicates = @duplicates
-  .values
-  .reject do |files|
-    files.length < 2
-  end
-
-# Stort
-@duplicates.each do |dups|
-  dups = dups.sort do |a, b|
-    case @options[:sort]
-    when 'atime'
-      File.atime(a) <=> File.atime(a)
-    when 'mtime'
-      File.mtime(a) <=> File.mtime(a)
-    when 'ctime'
-      File.ctime(a) <=> File.ctime(a)
-    else
-      a.length <=> b.length
-    end
+if options[:output]
+  File.open(options[:output], 'w') do |file|
+    file.write(JSON.pretty_generate(dup_finder.duplicate_entries))
   end
+else
+  $stdout.puts JSON.pretty_generate(dup_finder.duplicate_entries)
 end
-
-$stdout.puts JSON.pretty_generate(@duplicates)

data/lib/dup_finder.rb

@@ -0,0 +1,142 @@
+class DupFinder
+  attr_accessor :options
+
+  def initialize(hashed: {}, options: {})
+    @queue = []
+    @hashed = hashed
+    @options = {
+      depth: Float::INFINITY,
+      ignore: [],
+      ignore_empty: true,
+      cache: true,
+      alg: :md5,
+      cache_to_tmp: true,
+    }.merge(options)
+    @cache = {}
+  end
+
+  def queue(directory, depth = 0)
+    @queue << [directory, depth]
+  end
+
+  def search
+    until @queue.empty?
+      directory, depth = @queue.shift
+
+      unless depth > @options[:depth]
+        hash_entries(directory)
+      end
+    end
+
+    self
+  end
+
+  def duplicate_entries
+    dups = {}
+
+    @hashed.each do |(path, hash)|
+      dups[hash] ||= []
+      dups[hash] << path
+    end
+
+    dups.values.reject { |paths| paths.length < 2 }
+  end
+
+  private # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+  def ignored?(path)
+    return true if @options[:ignore_empty] && File.zero?(path)
+
+    @options[:ignore].any? do |pattern|
+      File.fnmatch(pattern, path, File::FNM_EXTGLOB)
+    end
+  end
+
+  def hash_entries(directory, depth: 0)
+    Dir.entries(directory).each do |entry|
+      next if entry.start_with?('.')
+      path = File.join(directory, entry)
+
+      Signal.trap("INT") do
+        write_cache(directory) if @options[:cache]
+        exit 1
+      end
+
+      next if File.symlink?(path)
+      next if ignored?(path)
+
+      if File.directory?(path)
+        queue(path, depth + 1)
+      else
+        if @options[:cache]
+          digest = cached_hash(path)
+        else
+          digest = hash_entry(path)
+        end
+      end
+    end
+
+    write_cache(directory) if @options[:cache]
+  end
+
+  def cached_hash(path)
+    directory = File.dirname(path)
+    file_name = File.basename(path)
+    @cache[directory] ||= load_cache(directory)
+    @cache[directory][file_name] ||= {}
+    meta_data = @cache[directory][file_name]
+
+    if meta_data.key?(options[:alg])
+      meta_data[options[:alg]]
+    else
+      meta_data[options[:alg]] = hash_entry(path)
+    end
+  end
+
+  def cache_path(directory)
+    if @options[:cache_to_tmp]
+      File.join("/tmp/file_hashes", File.expand_path(directory), "hashes.yml")
+    else
+      File.join(directory, "hashes.yml")
+    end
+  end
+
+  def load_cache(directory)
+    File.exist?(cache_path(directory)) ? YAML.load_file(cache_path(directory)) : {}
+  end
+
+  def write_cache(directory)
+    @cache[directory] ||= {}
+    @cache[directory].keep_if do |file_name, hash|
+      File.exist?(File.join(directory, file_name))
+    end
+
+    FileUtils.mkdir_p(File.dirname(cache_path(directory)))
+
+    File.open(cache_path(directory), 'w') do |file|
+      file.write(YAML.dump(@cache[directory]))
+    end
+  end
+
+  def hash_entry(path)
+    return @hashed[path] if @hashed.key?(path)
+
+    digest =
+      case @options[:alg]
+      when :md5  then Digest::MD5.new
+      when :sha1 then Digest::SHA1.new
+      end
+
+    File.open(path, 'rb') do |file|
+      while data = file.read(1024 * 1024)
+        digest.update(data)
+      end
+    end
+
+    @hashed[path] = digest.hexdigest
+    digest.hexdigest
+  rescue Errno::EINVAL => e
+    $stderr.puts "#{path}: #{e}"
+    nil
+  end
+end

data/lib/finddups/version.rb

@@ -1,3 +1,3 @@
 module Finddups
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: finddups
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Alex Clink
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-06-03 00:00:00.000000000 Z
+date: 2023-05-02 00:00:00.000000000 Z
 dependencies: []
-description: 
+description:
 email:
 - code@alexclink.com
 executables:
@@ -30,6 +30,7 @@ files:
 - bin/setup
 - exe/finddups
 - finddups.gemspec
+- lib/dup_finder.rb
 - lib/finddups.rb
 - lib/finddups/version.rb
 homepage: https://github.com/SleepingInsomniac/finddups
@@ -38,7 +39,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/SleepingInsomniac/finddups
   source_code_uri: https://github.com/SleepingInsomniac/finddups
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -53,8 +54,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.3
-signing_key: 
+rubygems_version: 3.4.9
+signing_key:
 specification_version: 4
 summary: Shows duplicate files within a list of directories and outputs as JSON.
 test_files: []