archivededup 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 832d1375b57959d1b4ed4704261478f15d80611f3e8a59eeca5365e2c188c56b
+  data.tar.gz: 3f93fc4ba314347ee306a45554239cfb8e1ed7ffa0ef0da2a1f88abe54d1540e
+SHA512:
+  metadata.gz: 715f06a1e280d06873d5ce900c7368b5f87d4509fa0fd1a4c3efc46c30a333675e40980a3297f6584eefdb90e3319f1954b29d4ab58468e8d281133324ec9d3e
+  data.tar.gz: 344bfd59b8af9d48430d8e97cc4e2f3594fea58061e5230afc6b901b142ca8c2bb9fa9a1c3234cf4a1ad1b0423ae011217eb8e2dd53c8089341fec605c744a2c

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/Gemfile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem "dbm", "~> 1.1"
+gem "rake", "~> 13.0"
+gem "rspec", "~> 3.12"
+

data/Gemfile.lock

@@ -0,0 +1,36 @@
+PATH
+  remote: .
+  specs:
+    archivededup (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    dbm (1.1.0)
+    diff-lcs (1.5.0)
+    rake (13.0.6)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.2)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.1)
+
+PLATFORMS
+  x86_64-darwin-21
+
+DEPENDENCIES
+  archivededup!
+  dbm (~> 1.1)
+  rake (~> 13.0)
+  rspec (~> 3.12)
+
+BUNDLED WITH
+   2.3.18

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Sam Baskinger
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,33 @@
+# Archivededup
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/archivededup`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add archivededup
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install archivededup
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/archivededup.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/archivededup.gemspec

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative "lib/archivededup/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "archivededup"
+  spec.version = Archivededup::VERSION
+  spec.authors = ["Sam Baskinger"]
+  spec.email = ["basking2@yahoo.com"]
+
+  spec.summary = "Deduplicate files from an archive directory."
+  spec.description = "Deduplicate files from an archive directory using a database."
+  spec.homepage = "https://github.com/basking2/archivededup"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.6.0"
+
+  #spec.metadata["allowed_push_host"] = "TODO: Set to your gem server 'https://example.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/basking2/archivededup"
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(__dir__) do
+    `git ls-files -z`.split("\x0").reject do |f|
+      (f == __FILE__) || f.match(%r{\A(?:(?:bin|test|spec|features)/|\.(?:git|travis|circleci)|appveyor)})
+    end
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "example-gem", "~> 1.0"
+
+  # For more information and examples about making a new gem, check out our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

data/exe/archivededup

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+#
+
+require 'dbm'
+require 'digest'
+require 'thread'
+require 'yaml'
+require 'optparse'
+
+require 'archivededup/db'
+require 'archivededup/paralleltask'
+require 'archivededup/filepicker'
+
+archive = File.join(ENV['HOME'], 'Archive')
+dbfile = 'archivededup.db'
+threads = 16
+action = 'check'
+
+OptionParser.new do |opts|
+
+  opts.on('-b', '--build', "Build a database.") do 
+    action = 'build'
+  end
+
+  opts.on('-c', '--check', "Check a database and report all duplicates.") do 
+    action = 'check'
+  end
+
+  opts.on('--removedups', "Check a database and remove all duplicates --check reports.") do 
+    action = 'remove'
+  end
+
+  opts.on('-d', '--dir=directory', 'Directory to dedup.') do |o|
+    archive = o
+  end
+
+  opts.on('-D', '--db=database_file', 'Dup database file.') do |o|
+    dbfile = o
+  end
+
+  opts.on('-t', '--threads=count', Integer, 'How many threads to use.') do |o|
+    threads = o
+  end
+end.parse! 
+
+
+db = Archivededup::Db.new(dbfile)
+
+case action
+when 'build'
+
+  parallelTask = Archivededup::ParallelTask.new
+
+  parallelTask.start(threads) do |d|
+    db.add_file(d)
+  end
+
+  parallelTask.scatter do |queue|
+    # Fill the queue and feed the threads.
+    db.add_directory(archive) do |d|
+      queue.enq d
+    end
+  end
+
+when /check|remove/
+  # Check and remove are nearly the same except for the actual delete operation.
+
+  do_delete = action == 'remove'
+
+  filepicker = Archivededup::FilePicker.new
+  
+  db.each_dup do |k, r|
+
+    # Skip files in the Documents directory. It's special.
+    files = r['files'].filter { |f| f !~ /\/(?:Documents|Archive\/Prairie Music)\// }
+
+    # If the filter removed all files, well, do nothing!
+    next if files.length < 2
+
+    # Skip files with unexpected file extensions.
+    unless files.find { |f| f =~ /\.(?:png|jpeg|jpg|mp4|mov|mp3|mod|xfc|gif|m4v|3g2|mpg)$/i }
+      files.each { |f| puts "Not deduping #{f}."}
+      next
+    end
+
+    filekeep = filepicker.pick(files)
+
+    puts "Keep #{filekeep}."
+    files.
+      select { |f| f != filekeep }.
+      each do |f|
+        puts "   Remove #{f}."
+
+        if do_delete
+          # Bit of an excessive check commented out below.
+          # raise Exception.new("Files differ!!!! #{f} #{filekeep}") if File.read(f) != File.read(filekeep)
+          raise Exception.new("File #{f} is the same as keep file #{filekeep}. Will not delete.") if File.identical?(filekeep, f)
+          File.unlink(f)
+        end
+      end
+  end
+end
+
+db.close

data/lib/archivededup/db.rb

@@ -0,0 +1,113 @@
+
+require 'dbm'
+require 'digest'
+require 'thread'
+require 'yaml'
+
+module Archivededup
+  class Db
+    def initialize(dbfile)
+      @db = DBM.new(dbfile, 0755, DBM::WRCREAT)
+    end
+
+    def each_hash
+      @db.select do |k, v|
+        k.start_with? 'hash - '
+      end.map do |k, v|
+        [k, YAML::load(v)]
+      end.each do |k, r|
+        yield k, r
+      end
+    end
+
+    def each_dup
+      @db.select do |k, v|
+        k.start_with? 'hash - '
+      end.map do |k, v|
+        [k, YAML::load(v)]
+      end.select do |k, r|
+        r['files'].length > 1
+      end.each do |k, r|
+        yield k, r
+      end
+    end
+
+    def add_directory(dir, &blk)
+      Dir.new(dir).each do |dent|
+        d = File.join(dir, dent)
+    
+        next if dent == '.'
+        next if dent == '..'
+        next if dent.start_with? '.'
+    
+        if File.directory? d
+          add_directory(d, &blk)
+        else
+          yield d if block_given?
+        end
+      end
+    end
+
+    def add_file(d)
+      # puts "Got #{d}"
+      s = File.stat(d)
+      # noimplemented - s.birthtime
+      s.mtime
+      s.ctime
+      # unused - s.atime
+      # puts d, s
+      hash = Digest::MD5.new
+      outbuf = ''
+
+      File.open(d) do |io|
+        while io.read(4096, outbuf) do
+          hash.update outbuf
+        end
+      end
+      
+      unless @db.has_key? "name - #{d}"
+
+        hash = hash.hexdigest
+
+        @db["name - #{d}"] = YAML::dump({
+          dir: d,
+          hash: hash,
+        })
+
+        if @db.has_key?('hash - '+hash)
+          o = YAML::load(@db['hash - '+hash])
+          buf1 = ''
+          buf2 = ''
+          File.open(d) do |io1|
+            File.open(o['files'][0]) do |io2|
+              v1 = ''
+              v2 = ''
+              while v1 && v2 && v1 == v2 do
+                v1 = io1.read(4096, buf1)
+                v2 = io2.read(4096, buf2)
+              end
+
+              if v1 == v2
+                o['files'] << d
+                puts "Dups found: #{o}"
+                @db['hash - '+hash] = YAML::dump(o)
+              end
+            end
+          end
+        else
+          puts "Storing #{d}."
+          @db['hash - '+hash] = YAML::dump({
+            'files' => [ d ],
+            'hash' => hash,
+          })
+        end
+      else
+        puts "Had #{d}."
+      end
+    end
+
+    def close()
+      @db.close()
+    end
+  end
+end

data/lib/archivededup/filepicker.rb

@@ -0,0 +1,90 @@
+
+require 'time'
+
+
+module Archivededup
+
+  module FileNamePickerByDate
+
+    @@times = [
+      /(\d\d\d\d)-(\d?\d)-\d?\d/, # eg 2021-09-08
+      /(\d\d\d\d)(\d\d)\d\d/,     # eg 20210908
+      /(\d\d\d\d)-(\d\d)/,        # eg 2021-09
+      /(\d\d\d\d)(\d\d)/,         # eg 202109
+      /(\d\d\d\d)/,               # eg 2021
+    ]
+
+    def self.goodmatch?(m)
+      return false if m.nil?
+      year = m.length > 1? m[1].to_i : 0
+      month = m.length > 2? m[2].to_i : 0
+
+      return false if year < 1979
+      return false if year > 2100
+      return false if month < 0
+      return false if month > 12
+
+      true
+    end
+    
+    def self.call(a, b)
+      last_compare = 0
+      @@times.find do |re|
+        ma = goodmatch?(re.match(a))
+        mb = goodmatch?(re.match(b))
+
+        if ma 
+          if mb
+            false
+          else
+            # Pick a as better.
+            last_compare = -1
+            true
+          end
+        else
+          if mb
+            # Pick b as better.
+            last_compare = 1
+            true
+          else
+            false
+          end
+        end
+      end
+
+      last_compare
+    end
+  end
+
+  # From a list of file names, pick the file that provides the most information
+  # and so should be kept.
+  #
+  class FilePicker
+    def initialize()
+      # A list of ways to compare file names.
+      # These are applied in-order until one returns non-zero.
+      @comparators = []
+      @comparators << FileNamePickerByDate
+      @comparators << proc { |a, b| b.length <=> a.length }
+    end
+
+    def pick(filelist)
+
+      filelist.sort do |a,b|
+
+        # Default the sort to be equal, no sort.
+        last_compare = 0
+
+        # Run comparators until one selcts a better file name.
+        @comparators.find do |c|
+          last_compare = c.call(a, b)
+          last_compare != 0
+        end
+
+        # Return the result of the last comparison.
+        last_compare
+      end.
+      first
+    end
+  end
+end

data/lib/archivededup/paralleltask.rb

@@ -0,0 +1,61 @@
+require 'thread'
+
+module Archivededup
+
+  class QueueAppender
+    def initialize(queue)
+      @queue = queue
+    end
+
+    def enq d
+      # Fill the queue and feed the threads.
+      while @queue.length > 100
+        sleep 1
+      end
+      @queue.enq d
+    end
+
+    alias :<< :enq
+  end
+
+  class ParallelTask
+    def initialize
+      @task
+    end
+
+    def start(threads=16)
+
+      @queue = Queue.new
+      
+      #Start the threads.
+      @threads = threads.times.map do 
+        Thread.new(@queue) do |queue|
+          run = true
+
+          while run
+            d = queue.deq
+
+            if d.nil?
+              run = false
+            else
+              yield d
+            end
+          end
+        end
+      end
+
+    end
+
+    def scatter
+      yield QueueAppender.new(@queue)
+
+      @queue.close
+
+      # Join the threads before exiting.
+      @threads.each do |t|
+        t.join
+      end
+    end
+
+  end
+end

data/lib/archivededup/version.rb

@@ -0,0 +1,3 @@
+module Archivededup
+  VERSION = "0.1.0"
+end

data/lib/archivededup.rb

@@ -0,0 +1,6 @@
+require "archivededup/version"
+
+module Archivededup
+  class Error < StandardError; end
+  # Your code goes here...
+end

data/sig/archivededup.rbs

@@ -0,0 +1,4 @@
+module Archivededup
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: archivededup
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sam Baskinger
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-07-04 00:00:00.000000000 Z
+dependencies: []
+description: Deduplicate files from an archive directory using a database.
+email:
+- basking2@yahoo.com
+executables:
+- archivededup
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- archivededup.gemspec
+- exe/archivededup
+- lib/archivededup.rb
+- lib/archivededup/db.rb
+- lib/archivededup/filepicker.rb
+- lib/archivededup/paralleltask.rb
+- lib/archivededup/version.rb
+- sig/archivededup.rbs
+homepage: https://github.com/basking2/archivededup
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/basking2/archivededup
+  source_code_uri: https://github.com/basking2/archivededup
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Deduplicate files from an archive directory.
+test_files: []