find-duplicates 0.0.1

data/bin/find-duplicates

@@ -0,0 +1,80 @@
+#!/usr/bin/ruby
+
+# Copyright 2013 Daneel S. Yaitskov
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this tool is written directly for 1 usecase.
+# There is 2 folders containing relativly similar files
+# but these files are in different pathes.
+# And you need to merge to folder structures
+# without space overhead.
+
+# Solution. Merge file sets with cp or rsync.
+# Start this script over 1 directory structure.
+# Script builds hashes for all files.
+# Groups pathes to files with the same hash.
+
+require 'optparse'
+require 'ruby-progressbar'
+require 'find-duplicates'
+include FindDuplicates
+
+options = {
+  :bar => lambda { |size| ProgressBar.create(:title => "Hashing",
+                                             :format => '%t: %e %P |%b%i|',
+                                             :total => size) },
+  :dry => true,
+  :remove_policy => PolicyLeaveLongest.new
+}
+
+paths = []
+begin
+  OptionParser.new do |opts|
+    opts.banner = 'Usage: find-duplicates.rb [ options ] <path-to-dir>'
+    opts.on('-d', '--dry',
+            'default. dry run. just show groups duplicated files 
+                                      and which one will be left') do |v|
+      options[:dry] = true
+    end
+    opts.on('-B', '--no-bar',
+            'hide progress bar') do |v|
+      options[:bar] = lambda { |size| DummyBar.new }
+    end
+    opts.on('-r', '--run',
+            'delete duplicates by the specified policy') do |v|
+      options[:dry] = false
+    end
+    opts.on('-l', '--longest',
+            'policy leaves a file with longest name among duplicates') do |v|
+      options[:remove_policy] = PolicyLeaveLongest.new
+    end
+  end.parse!
+
+  if ARGV.empty?
+    raise "there isn't any path given"  
+  end
+
+  paths = ARGV.map do |path|
+    raise "path '#{ path }' isn't a directory" if !File.directory?(path)
+    path.sub(/\/+ *$/, '')        
+  end
+rescue 
+  $stderr.puts "Error #{ $! }"
+  exit 1
+end
+
+find_duplicates options paths
+
+
+

data/find-duplicates.gemspec

@@ -0,0 +1,19 @@
+Gem::Specification.new do |s|
+  s.name    = 'find-duplicates'
+  s.version = '0.0.1'
+  s.date    = '2013-12-07'
+  s.summary = 'Find duplicated files'
+  s.description = 'Find and remove duplicated files'
+  s.authors = [ 'Daneel S. Yaitskov' ]
+  s.email   = 'rtfm.rtfm.rtfm@gmail.com'
+  s.files = [ 'bin/find-duplicates',              
+              'lib/find-duplicates.rb',
+              'find-duplicates.gemspec'
+            ]
+  s.require_paths = [ 'bin', 'lib' ]
+  s.executable = 'find-duplicates'
+  s.license = 'Apache 2'
+  s.homepage = 'https://github.com/yaitskov/find-duplicates'
+  s.add_dependency "ruby-progressbar"
+  s.add_dependency "bundler"  
+end

data/lib/find-duplicates.rb

@@ -0,0 +1,80 @@
+# Copyright 2013 Daneel S. Yaitskov
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'digest/md5'
+
+module FindDuplicates
+  class PolicyLeaveLongest
+    def choose_to_die(files)
+      files.sort { |a,b| b.size - a.size }
+    end
+  end
+
+  class DummyBar
+    def increment
+    end
+  end
+
+  class Groups
+    def initialize
+      @hashes = {}    
+    end
+    def add(path)
+      md5 = Digest::MD5.new
+      begin
+        File.open(path, 'rb') do |file_h|
+          file_h.each(8192) do |block|
+            md5.update block
+          end
+        end
+      rescue
+        $stderr.puts "problem: #{ $! }"
+        return
+      end
+      digest = md5.digest
+      if @hashes.key?(digest)
+        @hashes[digest] << path
+      else
+        @hashes[digest] = [ path ]
+      end
+    end
+    def duplicates
+      @hashes.find_all { |k,v| v.size > 1 }    
+    end
+  end
+
+  def find_duplicates(options, paths)
+    groups = Groups.new
+    files = paths.map { |path| Dir.glob(path + '/**/*') }.flatten
+
+    bar = options[:bar].call files.size
+
+    files.each do |file|
+      bar.increment
+      next if File.directory?(file)
+      groups.add file
+    end
+
+    groups.duplicates.each do |k,group|
+      death_queue = options[:remove_policy].choose_to_die(group)
+      survivor = [death_queue.first]
+      death_queue = death_queue.slice(1..-1)
+      if options[:dry]
+        puts (survivor + death_queue.map { |s| "   " + s }).join("\n")
+      else
+        death_queue.each { |file| File.delete file }
+      end
+    end    
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: find-duplicates
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Daneel S. Yaitskov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-12-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Find and remove duplicated files
+email: rtfm.rtfm.rtfm@gmail.com
+executables:
+- find-duplicates
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/find-duplicates
+- lib/find-duplicates.rb
+- find-duplicates.gemspec
+homepage: https://github.com/yaitskov/find-duplicates
+licenses:
+- Apache 2
+post_install_message: 
+rdoc_options: []
+require_paths:
+- bin
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Find duplicated files
+test_files: []