rivalry 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rivalry.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Anthony Cook
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,53 @@
+# Rivalry
+
+Rivalry is the (fast!) duplicate file finder written in Ruby.
+
+It supports specifically targeting audio and image files and ignores SCM folders such as `.git` and `.hg` by default.
+
+## Installation
+
+Using rubygems just:
+
+    $ gem install rivalry
+
+If you wish to use Rivalry's functionality inside your app your can instead add this line to your application's Gemfile:
+
+    gem 'rivalry'
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+On the commandline the most basic usage is:
+
+```bash
+$ rivalry ~/directory/with/duplicates
+```
+
+It will display the duplicate files on the screen, which can then be grepped or otherwise used to remove the unwanted duplicates.
+
+Example output:
+
+```bash
+$ rivalry ..
+Scanning all files...
+-- Total Size  : 173 MB
+-- Total Count : 95 files
+-- Similar     : 34 files with the same size
+
+Determining duplicates...
+-- Dupes Count : 2 files
+
+/Users/acook/Dropbox/Projects/Negutyv Xeiro/Audio/Cytokine Storm/Negutyv Xeiro - Cytokine Storm [Mørch Mix].mp3
+/Users/acook/Dropbox/Projects/Negutyv Xeiro/Audio/Cytokine Storm/alexcyto/Cytokine Storm m.mp3
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/rivalry

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'rivalry'
+rescue LoadError
+  $: << File.expand_path(File.join(File.dirname(__FILE__), '../lib'))
+  require 'rivalry'
+end
+
+Rivalry.run ARGV
+

data/lib/rivalry.rb

@@ -0,0 +1,27 @@
+require 'pry' rescue LoadError
+
+require 'rivalry/version'
+require 'rivalry/display'
+require 'rivalry/base'
+require 'rivalry/scanner'
+require 'rivalry/file_path'
+require 'rivalry/files_with_data'
+require 'rivalry/find_files'
+require 'rivalry/hash_files'
+
+module Rivalry
+  extend Display
+  module_function
+
+  def run args
+    path = args.first || usage
+
+    scanner = Rivalry::Scanner.new path
+    files = scanner.scan
+
+    out
+    files.each do |file|
+      puts file.to_s
+    end
+  end
+end

data/lib/rivalry/base.rb

@@ -0,0 +1,5 @@
+module Rivalry
+  class Base
+    include Display
+  end
+end

data/lib/rivalry/display.rb

@@ -0,0 +1,86 @@
+module Rivalry
+  module Display
+    def self.included object
+      unless $silent then
+        object.send :include, Methods
+      else
+        object.send :include, NoopMethods
+      end
+    end
+
+    def self.extended object
+      unless $silent then
+        object.send :extend, Methods
+      else
+        object.send :extend, NoopMethods
+      end
+    end
+
+    module Methods
+
+      def usage
+        puts "usage: #{$0} path/to/search/for/duplicates"
+        exit 1
+      end
+
+      def out text = ''
+        if $verbose then
+          puts text
+        else
+          puts text[0..(width - 1)]
+        end
+      end
+
+      def progress type, file, count = nil, total = nil
+        if count && total then
+          tally = " (#{count}/#{total})"
+        elsif count then
+          tally = " (#{count})"
+        else
+          tally = ''
+        end
+
+        text = "-- #{type}#{tally}: #{file}"
+
+        if $verbose then
+          puts text
+        else
+          clear_line
+          print text[0..(width - 1)]
+        end
+      end
+
+      def clear_line
+        clear_line = "\e[2K"
+        start_of_line = "\e[0G"
+        print start_of_line, clear_line
+      end
+
+      def width
+        tiocgwinsz = 0x40087468
+        str = [0, 0, 0, 0].pack('SSSS')
+        if $stdin.ioctl(tiocgwinsz, str) >= 0 then
+          str.unpack('SSSS')[1]
+        else
+          80
+        end
+      end
+
+      BYTE_UNITS =[[1073741824, 'GB'], [1048576, 'MB'], [1024, 'KB'], [-1, 'B']]
+
+      def humanize bytes
+        unit = BYTE_UNITS.find{|u| bytes > u[0] }
+        "#{bytes / unit[0]} #{unit[1]}"
+      end
+
+    end
+
+    module NoopMethods
+      def noop; end
+
+      Methods.instance_methods(false).each do |method|
+        alias_method method, :noop
+      end
+    end
+  end
+end

data/lib/rivalry/file_path.rb

@@ -0,0 +1,6 @@
+module Rivalry
+  class FilePath < Pathname
+    alias_method :each, :each_child
+    alias_method :exists?, :exist?
+  end
+end

data/lib/rivalry/files_with_data.rb

@@ -0,0 +1,7 @@
+module Rivalry
+  class FilesWithData < Hash
+    def initialize
+      super { |hash, key| hash[key] = Array.new }
+    end
+  end
+end

data/lib/rivalry/find_files.rb

@@ -0,0 +1,63 @@
+module Rivalry
+  class FindFiles < Base
+    def initialize pathname, do_want = {}, do_not_want = {}
+      @pathname = pathname
+      @do_want = do_want
+      @do_not_want = do_not_want
+    end
+    attr :pathname, :do_want, :do_not_want, :count, :size
+
+    def find
+      self.size  = 0
+      self.count = 0
+
+      files_with_sizes = FilesWithData.new
+
+      pathname.find do |file|
+
+        if file.directory? then
+          if do_not_want? file then
+            progress 'SKIP DIR', file
+            Find.prune
+          else
+            progress 'DIRECTORY', file
+          end
+        elsif valid? file then
+          if do_want? file then
+            progress 'FILE', file
+
+            file_size = file.size
+
+            files_with_sizes[file_size] << file
+
+            self.size += file_size
+            self.count += 1
+          else
+            progress 'SKIP FILE', file
+          end
+        end
+
+      end
+
+      files_with_sizes
+    end
+
+    protected
+
+    attr_writer :count, :size
+
+    def do_want? path
+      extension = File.extname path
+      do_want.find{|name, pattern| extension =~ pattern} && true
+    end
+
+    def do_not_want? path
+      basename = File.basename path
+      do_not_want.find{|name, pattern| basename =~ pattern} && true
+    end
+
+    def valid? file
+      file.exists?
+    end
+  end
+end

data/lib/rivalry/hash_files.rb

@@ -0,0 +1,32 @@
+module Rivalry
+  class HashFiles < Base
+    def initialize similar_files
+      @similar_files = similar_files
+    end
+    attr :similar_files
+
+    def hash
+      out
+      out "Determining duplicates..."
+
+      similar_count = 0
+      file_hashes = FilesWithData.new
+
+      similar_files.each do |file|
+        similar_count += 1
+
+        progress 'HASHING', file, similar_count, total_similar
+
+        hash = Digest::SHA256.file(file).to_s
+
+        file_hashes[hash] << file
+      end
+
+      file_hashes
+    end
+
+    def total_similar
+      similar_files.length
+    end
+  end
+end

data/lib/rivalry/scanner.rb

@@ -0,0 +1,57 @@
+require 'find'
+require 'pathname'
+require 'digest'
+
+module Rivalry
+  class Scanner < Base
+    def initialize pathname
+      @pathname = FilePath.new File.expand_path(pathname)
+    end
+    attr :pathname
+
+    def scan
+
+      out "Scanning all files..."
+
+      finder = FindFiles.new pathname, wants, ignores
+      files_with_sizes = finder.find
+
+      similar_files = dedup files_with_sizes
+
+      clear_line
+      out "-- Total Size  : #{humanize finder.size}"
+      out "-- Total Count : #{finder.count} files"
+      out "-- Similar     : #{similar_files.length} files with the same size"
+
+      hasher = HashFiles.new similar_files
+      files_with_hashes = hasher.hash
+
+      duplicate_files = dedup files_with_hashes
+
+      clear_line
+      out "-- Dupes Count : #{duplicate_files.length} files"
+
+      duplicate_files
+    end
+
+    def dedup files_with_data
+      files_with_data.map do |_, file_list|
+        file_list if file_list && file_list.length > 1
+      end.flatten.compact
+    end
+
+    def ignores
+      {
+        dir: /^\.$/,
+        scm: /^\.(git|hg|svn|gitkeep)$/
+      }
+    end
+
+    def wants
+      {
+        audio:  /^\.(mp3|ogg|flac|wav|aiff|mid)$/i,
+        images: /^\.(png|jpg|gif|bmp|tga|jpeg|tif|tiff)/i
+      }
+    end
+  end
+end

data/lib/rivalry/version.rb

@@ -0,0 +1,3 @@
+module Rivalry
+  VERSION = "0.0.1"
+end

data/rivalry.gemspec

@@ -0,0 +1,19 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rivalry/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "rivalry"
+  gem.version       = Rivalry::VERSION
+  gem.authors       = ["Anthony Cook"]
+  gem.email         = ["anthonymichaelcook@gmail.com"]
+  gem.description   = %q{The (fast!) duplicate file finder for Ruby! Supports media file and ignoring SCM directories.}
+  gem.summary       = %q{The (fast!) duplicate file finder for Ruby!}
+  gem.homepage      = "https://github.com/acook/rivalry#readme"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,63 @@
+--- !ruby/object:Gem::Specification
+name: rivalry
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Anthony Cook
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-01-27 00:00:00.000000000 Z
+dependencies: []
+description: The (fast!) duplicate file finder for Ruby! Supports media file and ignoring
+  SCM directories.
+email:
+- anthonymichaelcook@gmail.com
+executables:
+- rivalry
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.markdown
+- Rakefile
+- bin/rivalry
+- lib/rivalry.rb
+- lib/rivalry/base.rb
+- lib/rivalry/display.rb
+- lib/rivalry/file_path.rb
+- lib/rivalry/files_with_data.rb
+- lib/rivalry/find_files.rb
+- lib/rivalry/hash_files.rb
+- lib/rivalry/scanner.rb
+- lib/rivalry/version.rb
+- rivalry.gemspec
+homepage: https://github.com/acook/rivalry#readme
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: The (fast!) duplicate file finder for Ruby!
+test_files: []