right_data 0.5.0

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Jonathan Siegel
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,21 @@
+= right_data
+
+Description goes here.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+rake version:bump:minor
+rake build
+gem install pkg/...
+
+== Copyright
+
+Copyright (c) 2010 Jonathan Siegel. See LICENSE for details.

data/lib/FileSystemItem.rb

@@ -0,0 +1,153 @@
+module RightData
+  class FileSystemItem
+    
+    attr_reader :relativePath
+    attr_reader :parent
+
+    attr_reader :ignore_children
+    attr_reader :duplicate_children
+    attr_accessor :duplicates
+    attr_accessor :ignorable
+    
+    def initialize path, args
+      if args[:parent]
+        @relativePath = File.basename(path)
+        @parent = args[:parent]
+      else
+        @relativePath = path
+        @parent = nil
+      end
+      @ignorable = false
+      @duplicates = [] # for this node
+      @duplicate_children = 0 # counts for children
+      @ignore_children    = 0
+      self
+    end
+
+    def files
+      return 0 if leaf? && File.directory?(fullPath)
+      return 1 if leaf?
+      return children.map {|n| n.files}.inject {|sum, n| sum + n } 
+    end
+    def ignore_files
+      return 0 if leaf? && File.directory?(fullPath)
+      return ignorable? ? 1 : 0 if leaf?
+      return children.map {|n| n.ignore_files}.inject {|sum, n| sum + n } 
+    end
+    def duplicate_files
+      return 0 if leaf? && File.directory?(fullPath)
+      return duplicate? ? 1 : 0 if leaf?
+      return children.map {|n| n.duplicate_files}.inject {|sum, n| sum + n } 
+    end
+
+
+    def basename; @relativePath; end
+    
+    def self.rootItem
+      @rootItem ||= self.new '/', :parent => nil
+    end
+    
+    def children
+      unless @children
+         if File.directory?(fullPath) and File.readable?(fullPath)
+           @children = Dir.entries(fullPath).select { |path|
+              path != '.' and path != '..'
+           }.map { |path|
+              FileSystemItem.new path, :parent => self
+           }
+         else
+           @children = nil
+         end
+      end
+      @children
+    end
+    
+    def path; fullPath; end
+    def fullPath
+      @parent ? File.join(@parent.fullPath, @relativePath) : @relativePath
+    end
+    
+    def childAtIndex n
+      children[n]
+    end
+    
+    def numberOfChildren
+      children == nil ? -1 : children.size
+    end
+
+    def children?; !children.nil? && !children.empty?; end
+
+    def duplicate?
+      if leaf?
+        !duplicates.empty?
+      else # Dup if all ignored / dup children
+        ((@ignore_children + @duplicate_children) == numberOfChildren)
+      end
+    end
+
+    def ignorable?; ignorable; end
+
+    def increment_ignorable_children
+      @ignore_children += 1
+      update_duplicate_ignorable_status
+    end
+
+    def update_duplicate_ignorable_status
+      parent.increment_duplicate_children if((@ignore_children + @duplicate_children) == numberOfChildren)
+    end
+
+    def increment_duplicate_children
+      @duplicate_children += 1
+      update_duplicate_ignorable_status
+    end
+
+    def leaf?; !children?; end
+
+    def traverse(&block) # Allow proc to decide if we traverse
+      if block.call(self) && children?
+        children.each { |c| c.traverse(&block) }
+      end
+    end
+
+    def other_children
+      children.size - ignore_children - duplicate_children
+    end
+
+    def to_param; to_s; end
+    def to_s
+      "<Tree :path => #{self.path}, :files => #{self.files}>" 
+    end
+
+    # Inspect the nodes:
+    def report(pre="")
+      pre += " " if !pre.empty?
+      self.traverse do |n|
+        # Is this a leaf (e.g. a file)?
+        if n.leaf?
+          msg = nil
+          msg = "# dup(#{n.duplicates.count})" if n.duplicate?
+          msg = "# ign" if n.ignorable?
+          if msg
+            puts "#{pre}#{n.path} #{msg}" # Remove the dups/igns!
+          else
+            puts "# #{n.path} unique"
+          end
+          false # Don't traverse deeper!
+        else
+          if n.duplicate_children + n.ignore_children == n.children.size
+            puts "#{pre}#{n.path} # #{n.duplicate_children} dups / #{n.ignore_children} ignores"
+            false # Don't traverse deeper!
+          elsif n.children.size == 0
+            puts "#{pre}#{n.path} # Empty... "
+            false
+          else
+            puts "# #{n.path} # Not #{n.duplicate_children} dup/ #{n.ignore_children} ign / #{n.other_children} other "
+            true
+          end
+        end
+      end
+      puts "# #{self.ignore_files} ignores, #{self.duplicate_files} dups of #{self.files} files"
+    end
+    
+  end
+end

data/lib/main.rb

@@ -0,0 +1,270 @@
+#!/usr/bin/env ruby
+require 'FileSystemItem'
+# require 'unicode'
+
+$KCODE = 'UTF-8'   # only used when encoding is not specified.
+
+# crawler = Index.new
+# crawler.crawl
+
+=begin
+  #!/usr/bin/ruby
+  # find_duplicates.rb
+  
+  require 'find'
+  require 'digest/md5'
+
+  def each_set_of_duplicates(*paths)
+    sizes = {}
+    Find.find(*paths) do |f|
+     (sizes[File.size(f)] ||= []) << f if File.file? f
+    end
+    sizes.each do |size, files|
+      next unless files.size > 1
+      md5s = {}
+      files.each do |f|
+        digest = Digest::MD5.hexdigest(File.read(f))
+        (md5s[digest] ||= []) << f
+      end
+      md5s.each { |sum, files| yield files if files.size > 1 }
+    end
+  end
+
+  each_set_of_ 
+duplicates(*ARGV) do |f|
+    puts " 
+Duplicates: #{f.join(", ")}"
+  end
+=end
+
+# http://codeidol.com/other/rubyckbk/System-Administration/Finding-Duplicate-Files/
+
+#!/usr/bin/ruby
+# find_duplicates2.rb
+
+require 'find'
+
+module RightData
+
+  BLOCK_SIZE = 1024*8
+
+  def self.each_set_of_duplicates(*paths, &block)
+    sizes = Hash.new {|h, k| h[k] = [] }
+    Find.find(*paths) { |f| sizes[File.size(f)] << f if File.file? f }
+
+    sizes.each_pair do |size, files|
+    # puts files.inspect
+      next unless files.size > 1
+      offset = 0
+      files = [files]
+      while !files.empty? && offset <= size
+        files = eliminate_non_duplicates(files, size, offset, &block)
+        offset += BLOCK_SIZE
+      end
+    end
+  end
+
+  def self.eliminate_non_duplicates(partition, size, offset)
+    possible_duplicates = []
+    partition.each do |possible_duplicate_set|
+      blocks = Hash.new {|h, k| h[k] = [] }
+      possible_duplicate_set.each do |f|
+        block = open(f, 'rb') do |file|
+          file.seek(offset)
+          file.read(BLOCK_SIZE)
+        end
+        blocks[block || ''] << f
+      end
+      blocks.each_value do |files|
+        if files.size > 1
+          if offset+BLOCK_SIZE >= size
+            # We know these are duplicates.
+            yield files
+          else
+            # We suspect these are duplicates, but we need to compare
+            # more blocks of data.
+            possible_duplicates << files
+          end
+        end
+      end
+    end
+   return possible_duplicates
+  end
+
+  IGNORE_FILES = [".DS_Store", ".typeAttributes.dict", "empty-file"]
+
+  def self.index_by_size(*paths)
+    sizes = Hash.new {|h, k| h[k] = [] }
+    count = 0
+    Find.find(*paths) { |f| 
+    sizes[File.size(f)] << f if File.file?(f) && !IGNORE_FILES.include?(File.basename(f)) && (File.size(f) != 0) # Ignore empty files
+    count += 1
+    }
+    puts "Indexed #{count} files."
+    sizes
+  end
+
+  def self.cache_not_working_on_write(master)
+    master_cache = File.join(master,".rightPruneCache")
+    if File.exist?(master_cache)
+      puts "Master cache FOUND at #{master_cache}."
+      master_index = File.open(master_cache) do |f| 
+        YAML::load(f)
+      end
+    else
+      puts "Master cache not found at #{master_cache}."
+      master_index = index_by_size(master)
+      puts "Writing #{master_cache}."
+      File.open(master_cache, "w") do |f| 
+        YAML.dump(master_index, f)
+      end  
+      puts "Wrote #{master_cache}."
+    end
+    master_index
+  end
+
+
+  def self.cache_serializing_on_write(master)
+    master_cache = File.join(master,".rightPruneCache")
+    if File.exist?(master_cache)
+      puts "Master cache FOUND at #{master_cache}."
+      master_index = File.open(master_cache) do |f| 
+        rval = {}
+        f.each_line do |l|
+          kv = Marshal.load(l)
+          rval[kv.first] = kv.last
+        end
+        rval
+      end
+    else
+      puts "Master cache not found at #{master_cache}."
+      master_index = index_by_size(master)
+      puts "Writing #{master_cache}."
+      File.open(master_cache, "w") do |f| 
+        master_index.each_pair do |k,v|
+          Marshal.dump([k,v], f)
+        end
+        # f.write(master_index.inspect)
+      end  
+      puts "Wrote #{master_cache}."
+    end
+  end
+
+  def self.get_block(file,offset)
+    open(file, 'r') do |f|
+      f.seek(offset); f.read(BLOCK_SIZE)
+    end
+  end
+
+  def self.check_file_in_index(master_index, file_to_check, &block)
+    size = File.size(file_to_check)
+    return [] if size == 0 # Ignore empty files
+    possible_master_dups = master_index[size] || []
+      offset = 0
+      while !possible_master_dups.empty? && offset <= size
+      file_to_check_block = get_block(file_to_check, offset)
+        new_possible_master_dups = []
+      possible_master_dups.each do |master|
+      block = get_block(master,offset)
+      if(block == file_to_check_block)
+        new_possible_master_dups << master
+      end
+      end
+        possible_master_dups = new_possible_master_dups
+        offset += BLOCK_SIZE
+      end
+    # puts possible_master_dups.inspect
+    possible_master_dups
+  end
+
+  def self.test
+    master = "/Users/jonathan/Dropbox"
+    prune  = "/Users/jonathan/Desktop/Old"
+    scan_for_prunable(master,prune) { |a,b| puts "#{b.size} : #{a}" }
+  end
+
+  # tree = scan_for_prunable(master,prune) { |a,b| puts "#{b.size} : #{a}" }; nil
+  def self.scan_for_prunable(master,prune, &block)
+    puts "Ignoring: #{IGNORE_FILES.inspect}"
+
+    master_index = cache_not_working_on_write(master)
+    # master_index = index_by_size(master)
+    puts "Found #{master_index.size} unique sizes."
+
+    # dups = check_file_in_index(master_index, "/Users/jonathan/Dropbox/2261093437_fac9fa9008_b.jpg")
+
+    count = 0
+
+    # Recursively compare the files in the filesystem.
+    # When a parent node gets a response from all its children
+    # that they are dups OR ignorable, that NODE becomes dup_or_ignorable too.
+    # This propagates.
+    # Then, there is a traversal that grabs all base nodes that are non_dup like:
+    # rm -rf /a_path_duped/here     # 14 dups / 9 ignores
+    # rm -rf /b_path_duped/way/here # 1 dup
+    tree = FileSystemItem.new(prune, :parent => nil)
+    # Mark the nodes:
+    tree.traverse do |n|
+      next true if File.directory?(n.path)
+      count += 1
+      if IGNORE_FILES.include?(n.basename)
+        n.ignorable = true
+        n.parent.increment_ignorable_children
+      else
+        # puts n.path
+        duplicates = check_file_in_index(master_index, n.path)
+        if(!duplicates.empty?) 
+          n.duplicates = duplicates
+          n.parent.increment_duplicate_children
+        end
+      end
+      true
+    end
+    puts "We counted #{count} files. Tree thinks it has #{tree.files}."
+    return tree
+
+    if nil
+    Find.find(prune) { |f|
+      if File.directory? f
+        puts "Dir: #{f}"
+        prunable_dirs[f] = {}
+        next
+      end
+      # next unless File.file? f
+      count += 1
+      duplicates = check_file_in_index(master_index, f)
+      if(!duplicates.empty?) 
+        dups[f] = duplicates
+        prunable_files[f] = duplicates
+        block.call(f, duplicates) unless block.nil?
+      else
+        prunable_files[f] = false
+      end
+    }
+
+    puts "After check. Found #{dups.size} / #{count} dups in master."
+    puts "After check. Found #{dups.first.inspect}"
+    end
+
+    # puts "Dirs scanned."
+    #prunable_dirs.each_pair do |file,prunable|
+      #puts "#{'#' if !prunable} #{file}"
+    #end
+
+    # puts "Files scanned."
+    # prunable_files.keys.sort.each do |file|
+      # prunable = prunable_files[file]
+      # puts "#{'#' if !prunable} #{file}"
+    # end
+    # prunable_files
+  end
+
+  # each_set_of_duplicates(dirs) do |f|
+  #  puts "Duplicates: #{f.join(", ")}"
+  #end
+
+  # With YAML cache:
+  # Master cache FOUND at /Users/jonathan/Dropbox/.rightPruneCache.
+  # Found 37765 unique sizes.
+  # After check. Found 1240 / 1940 dups in master.
+end

data/lib/right_data.rb

@@ -0,0 +1,5 @@
+require 'main'
+
+module RightData
+  def self.hello; "Hi!"; end
+end

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'right_data'
+
+class Test::Unit::TestCase
+end

data/test/test_right_data.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestRightData < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification 
+name: right_data
+version: !ruby/object:Gem::Version 
+  hash: 11
+  prerelease: false
+  segments: 
+  - 0
+  - 5
+  - 0
+  version: 0.5.0
+platform: ruby
+authors: 
+- Jonathan Siegel
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-08-30 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id001
+description: RightData helpers
+email: usiegj00@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- lib/FileSystemItem.rb
+- lib/main.rb
+- lib/right_data.rb
+- LICENSE
+- README.rdoc
+- test/helper.rb
+- test/test_right_data.rb
+has_rdoc: true
+homepage: http://github.com/usiegj00/right_data
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: RightData helpers
+test_files: 
+- test/helper.rb
+- test/test_right_data.rb