rdup 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b46c4a3517e675b2710af4baea234d33ff235520
+  data.tar.gz: 9a5f237afaee2c2d3c184c3d07b0e543e2d5ce6b
+SHA512:
+  metadata.gz: 98a8e3114fa9e4081037a0eca3e4cee6d6e8ad485f2562a25cb3e07a22f77636f701b39612586f94cbc619f10b199f250dcd42f59c96113e5a97c695d6342b6a
+  data.tar.gz: c4dd4443d3a022d19a03f3eed0f3e5518d9af9e7c6f7df8309b6a02b238f0f84c7a50d482530cfb8d376464605c8bb10d82a084a3334faad48605e2a11302bc2

data/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 physacco
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

data/README.md

@@ -0,0 +1,72 @@
+# RDup
+
+This program finds duplicate files in multiple directories and interactively remove any of them as you wish.
+
+It is inspired by [fdupes](https://github.com/adrianlopezroche/fdupes) and [fastdupes](https://github.com/ssokolow/fastdupes).
+
+Written in pure Ruby. No external dependencies. Cross-platform.
+
+Tested on Ruby 2.2.2, but it should be able to run on Ruby >= 2.0.
+
+## Algorithm
+
+Files with the same SHA1 digests are considered as duplicates.
+
+RDup implements the Duplicates Finding Algorithm used by fastdupes:
+
+1. The given paths are recursively walked to gather a list of files.
+2. Files are grouped by size and single-entry groups are pruned away.
+3. Groups are subdivided and pruned by hashing the first *16KiB* of each file.
+4. Groups are subdivided and pruned again by hashing full contents.
+5. Any groups which remain are sets of duplicates.
+
+By using this algorithm, RDup performs much faster than fdupes.
+
+## Installation
+
+`gem install rdup`
+
+## Usage
+
+```
+Usage: rdup [options] dir1 [dir2 ...]
+
+Options:
+  -h, --help                Print this help message and exit
+  -v, --version             Print version information and exit
+  -t, --mtime               Show each file's mtime
+  -d, --delete              Delete duplicated files (with prompt)
+  -n, --dry-run             Don't actually delete any files
+  --min-size=NUM            Files below this size will be ignored
+```
+
+## Example
+
+```
+$ rdup --mtime --delete --dry-run foo/ bar/
+Found 5 files to be compared for duplication.
+Found 2 sets of files with identical sizes. (5 files in total)
+Found 2 sets of files with identical header hashes. (5 files in total)
+Found 2 sets of files with identical hashes. (5 files in total)
+
+[1/2] SHA1: c56351f9f9eb825c743141dd4acc870166838e3c, Size: 880 bytes
+  1) 2015-12-13 17:07:52 +0800  foo/abc/abc.dat
+  2) 2015-12-13 17:08:04 +0800  bar/abc.dat
+Which to preserve (1,2 or all): 1
+  [+] foo/abc/abc.dat
+  [-] bar/abc.dat
+
+[2/2] SHA1: 500fe2c2d2018bbe97a1341cf826335aaafab3d9, Size: 1,076 bytes
+  1) 2015-12-13 17:07:13 +0800  foo/abc/foo.txt
+  2) 2015-12-13 17:06:49 +0800  foo/foo.txt
+  3) 2015-12-13 17:07:25 +0800  bar/bar.txt
+Which to preserve (1,2,3 or all): 2
+  [-] foo/abc/foo.txt
+  [+] foo/foo.txt
+  [-] bar/bar.txt
+```
+
+## Known issues
+
+* RDup doesn't follow Windows [Shortcut files](https://en.wikipedia.org/wiki/File_shortcut). Shortcuts are treated as normal files.
+

data/bin/rdup

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'getoptlong'
+require_relative '../lib/rdup'
+
+Usage = <<EOF
+Usage: rdup [options] dir1 [dir2 ...]
+
+Options:
+  -h, --help                Print this help message and exit
+  -v, --version             Print version information and exit
+  -t, --mtime               Show each file's mtime
+  -d, --delete              Delete duplicated files (with prompt)
+  -n, --dry-run             Don't actually delete any files
+  --min-size=NUM            Files below this size will be ignored
+EOF
+
+VersionInfo = "rdup v#{RDup::VERSION}"
+
+def parse_arguments
+  opts = GetoptLong.new(
+    ['--help',     '-h', GetoptLong::NO_ARGUMENT],
+    ['--version',  '-v', GetoptLong::NO_ARGUMENT],
+    ['--mtime',    '-t', GetoptLong::NO_ARGUMENT],
+    ['--delete',   '-d', GetoptLong::NO_ARGUMENT],
+    ['--dry-run',  '-n', GetoptLong::NO_ARGUMENT],
+    ['--min-size',       GetoptLong::REQUIRED_ARGUMENT],
+  )
+
+  show_mtime = false
+  deletion = false
+  dry_run = false
+  min_size = 0
+
+  begin
+    opts.each do |opt, arg|
+      case opt
+        when '--help'
+          puts Usage
+          exit 0
+        when '--version'
+          puts VersionInfo
+          exit 0
+        when '--mtime'
+          show_mtime = true
+        when '--delete'
+          deletion = true
+        when '--dry-run'
+          dry_run = true
+        when '--min-size'
+          if arg =~ /^\d+$/
+            min_size = arg.to_i
+          else
+            STDERR.puts "rdup: --min-size requires a number"
+            exit 1
+          end
+      end
+    end
+  rescue GetoptLong::Error
+    exit 1
+  end
+
+  if ARGV.size == 0
+    STDERR.puts 'rdup: no directories specified'
+    exit 1
+  end
+
+  return {
+    :show_mtime => show_mtime,
+    :deletion   => deletion,
+    :dry_run    => dry_run,
+    :min_size   => min_size,
+    :arguments  => ARGV
+  }
+end
+
+args = parse_arguments
+RDup::Scanner.new(args).run

data/lib/rdup.rb

@@ -0,0 +1,252 @@
+# encoding: utf-8
+
+require 'digest/sha1'
+require_relative 'rdup/version'
+
+module RDup
+  Defaults = {
+    :show_mtime  => false,
+    :deletion    => false,
+    :dry_run     => false,
+    :min_size    => 0,
+    :header_size => 2 ** 14,
+  }
+
+  class FileStat
+    attr_reader :size, :mtime
+    attr_accessor :header_hash, :full_hash
+
+    def initialize(size, mtime)
+      @size = size
+      @mtime = mtime
+      @header_hash = nil
+      @full_hash = nil
+    end
+  end
+
+  class Scanner
+    attr_accessor :opts, :files, :dirs
+    attr_reader :stats, :header_hashes, :full_hashes
+    attr_reader :size_map, :header_hash_map, :full_hash_map
+
+    def initialize(opts)
+      @opts = Defaults.dup
+      @files = []
+      @dirs = []
+      @stats = {}
+      @header_hashes = {}
+      @full_hashes = {}
+      @size_map = {}
+      @header_hash_map = {}
+      @full_hash_map = {}
+
+      @opts.update(opts)
+
+      opts[:arguments].each do |path|
+        if File.file?(path)
+          @files << path
+        elsif File.directory?(path)
+          @dirs << path
+        else
+          STDERR.puts "Warning: skip `#{dir}' because it's neither a file nor a directory"
+        end
+      end
+    end
+
+    def run
+      find_all_files
+      fcount = @stats.size
+      puts "Found #{fcount} files to be compared for duplication."
+      if fcount == 0
+        return
+      end
+
+      build_size_map
+      reduce_groups(@size_map)
+      gcount = @size_map.size
+      fcount = count_files(@size_map)
+      puts "Found #{gcount} sets of files with identical sizes. (#{fcount} files in total)"
+      if fcount == 0
+        return
+      end
+
+      build_header_hash_map
+      reduce_groups(@header_hash_map)
+      gcount = @header_hash_map.size
+      fcount = count_files(@header_hash_map)
+      puts "Found #{gcount} sets of files with identical header hashes. (#{fcount} files in total)"
+      if fcount == 0
+        return
+      end
+
+      build_full_hash_map
+      reduce_groups(@full_hash_map)
+      gcount = @full_hash_map.size
+      fcount = count_files(@full_hash_map)
+      puts "Found #{gcount} sets of files with identical hashes. (#{fcount} files in total)"
+      if fcount == 0
+        return
+      end
+
+      @full_hash_map.each_with_index do |pair, i|
+        full_hash, group = pair
+        size = @stats[group[0]].size
+        puts "\n[#{i + 1}/#{gcount}] SHA1: #{full_hash}, Size: #{csf(size)} bytes"
+        group.each_with_index do |path, j|
+          stat = @stats[path]
+          if @opts[:show_mtime]
+            puts "  #{j + 1}) #{stat.mtime}  #{path}"
+          else
+            puts "  #{j + 1}) #{path}"
+          end
+        end
+
+        if @opts[:deletion]
+          survivals = which_to_preserve(group)
+          group.each_with_index do |path, index|
+            if survivals.include?(index + 1)
+              puts "  [+] #{path}"
+            else
+              puts "  [-] #{path}"
+              remove_file(path) unless @opts[:dry_run]
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    def find_all_files
+      @files.each do |path|
+        stat = File.stat(path)
+        if stat.size >= @opts[:min_size]
+          @stats[path] = FileStat.new(stat.size, stat.mtime)
+        else
+          @files.delete(path)
+        end
+      end
+
+      pwd = Dir.pwd
+      @dirs.each do |dir|
+        begin
+          Dir.chdir(dir)
+          Dir['**/*'].each do |path|
+            stat = File.stat(path)
+            if stat.file? and stat.size >= @opts[:min_size]
+              path = File.join(dir, path)
+              @files << path
+              @stats[path] = FileStat.new(stat.size, stat.mtime)
+            end
+          end
+        rescue => e
+          STDERR.puts "Error: #{e}"
+        ensure
+          Dir.chdir(pwd)
+        end
+      end
+    end
+
+    # Group the files by size
+    # @size_map: file_size => [file1, file2, ...]
+    def build_size_map
+      @stats.each do |path, stat|
+        size = stat.size
+        if @size_map.has_key?(size)
+          @size_map[size] << path
+        else
+          @size_map[size] = [path]
+        end
+      end
+    end
+
+    # @header_hash_map: header_hash => [file1, file2, ...]
+    def build_header_hash_map
+      header_size = @opts[:header_size]
+      @size_map.each do |size, paths|
+        paths.each do |path|
+          header = File.open(path, 'rb'){|f| f.read(header_size)}
+          header = '' if header.nil?  # empty file
+
+          header_hash = Digest::SHA1.new.hexdigest(header)
+          @stats[path].header_hash = header_hash
+          @stats[path].full_hash = header_hash if size <= header_size
+
+          if @header_hash_map.has_key?(header_hash)
+            @header_hash_map[header_hash] << path
+          else
+            @header_hash_map[header_hash] = [path]
+          end
+        end
+      end
+    end
+
+    # @header_hash_map: full_hash => [file1, file2, ...]
+    def build_full_hash_map
+      @header_hash_map.each_value do |paths|
+        paths.each do |path|
+          stat = @stats[path]
+          if stat.size <= @opts[:header_size]
+            full_hash = stat.full_hash
+          else
+            full_hash = Digest::SHA1.new.file(path).hexdigest
+            @stats[path].full_hash = full_hash
+          end
+
+          if @full_hash_map.has_key?(full_hash)
+            @full_hash_map[full_hash] << path
+          else
+            @full_hash_map[full_hash] = [path]
+          end
+        end
+      end
+    end
+
+    def reduce_groups(map)
+      map.delete_if {|key, paths| paths.size == 1}
+    end
+
+    def count_files(map)
+      map.values.flatten.size
+    end
+
+    # Comma-separated format
+    def csf(number)
+      number.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+    end
+
+    # Ask the user which files to preserve.
+    # Return an array of numbers
+    def which_to_preserve(group)
+      while true
+        all = 1.upto(group.size).to_a
+        print "Which to preserve (#{all.join(',')} or all): "
+        input = STDIN.readline.strip
+        if input.empty?
+          # continue
+        elsif ['a', 'all'].include?(input.downcase)
+          return all
+        elsif input =~ /^[\d\s,]+$/
+          nums = input.split(/[,\s]+/).delete_if(&:empty?).map(&:to_i)
+          if nums.empty?
+            STDERR.puts 'Illegal answer. Please input some numbers.'
+          elsif nums.min < 1 || nums.max > group.size
+            STDERR.puts "Illegal number. Allowed range: [1, #{group.size}]"
+          else # good answer
+            return nums
+          end
+        else
+          STDERR.puts 'Illegal answer. Only numbers/commas/spaces allowed.'
+        end
+      end
+    end
+
+    def remove_file(path)
+      begin
+        File.unlink(path)
+      rescue => e
+        STDERR.puts "Error: #{e}"
+      end
+    end
+  end
+end

data/lib/rdup/version.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+module RDup
+  VERSION = '0.1.0'
+end

data/rdup.gemspec

@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+require_relative 'lib/rdup/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'rdup'
+  s.version     = RDup::VERSION
+  s.date        = '2015-12-05'
+
+  s.summary     = 'Find and remove duplicate files in multiple directories.'
+  s.description = <<EOF
+This program finds duplicate files in multiple directories and
+interactively remove any of them as you wish. It is similar to fdupes,
+but much faster. Written in pure Ruby. No external dependencies.
+EOF
+
+  s.authors     = ['physacco']
+  s.email       = ['physacco@gmail.com']
+  s.homepage    = 'https://github.com/physacco/rdup'
+  s.license     = 'MIT'
+
+  s.files       = Dir['lib/**/*.rb'] + Dir['bin/*'] +
+                  ['README.md', 'LICENSE', 'rdup.gemspec']
+  s.executables = ['rdup']
+
+  s.platform    = Gem::Platform::RUBY
+  s.required_ruby_version = '>= 2.0.0'
+end

metadata

@@ -0,0 +1,55 @@
+--- !ruby/object:Gem::Specification
+name: rdup
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- physacco
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-05 00:00:00.000000000 Z
+dependencies: []
+description: |
+  This program finds duplicate files in multiple directories and
+  interactively remove any of them as you wish. It is similar to fdupes,
+  but much faster. Written in pure Ruby. No external dependencies.
+email:
+- physacco@gmail.com
+executables:
+- rdup
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/rdup
+- lib/rdup.rb
+- lib/rdup/version.rb
+- rdup.gemspec
+homepage: https://github.com/physacco/rdup
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Find and remove duplicate files in multiple directories.
+test_files: []
+has_rdoc: