itunes-rm-dups 0.1.1-universal-darwin-9

data/README

@@ -0,0 +1,21 @@
+ITunes Duplicate Track Remover 
+
+After installation, run with this command:
+
+itunes-rm-dups
+
+and follow the prompts, which should be self-explanatory.
+
+NOTE: This program is designed to detect and remove only exact duplicates.
+Some apparent duplicates may be left intact in the iTunes library after the
+program runs. These are not true duplicates because their MD5 hashes do not
+match for one reason or another. It could be that one version of the track is
+protected and the other is not; that one version is compressed more than the
+other, or that one version uses different protection encryption, or that the
+versions simply have a few bits set differently for whatever reason. Please
+check the files manually to be sure. 
+
+
+Daniel Choi
+Cambridge, MA
+dhchoi@gmail.com

data/bin/itunes-rm-dups

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'itunes_dup_handler'
+run

data/lib/itunes_dup_handler.rb

@@ -0,0 +1,299 @@
+# To run, make sure this file is on the RUBYLIB, and type this on the command line:
+#
+# > ruby -r itunes_dup_handler -e 'run'
+#
+# And follow the prompts. 
+#
+# NOTE: This program is designed to detect and remove only exact duplicates.
+# Some apparent duplicates may be left intact in the iTunes library after the
+# program runs. These are not true duplicates because their MD5 hashes do not
+# match for one reason or another. It could be that one version of the track is
+# protected and the other is not; that one version is compressed more than the
+# other, or that one version uses different protection encryption, or that the
+# versions simply have a few bits set differently for whatever reason. Please
+# check the files manually to be sure. 
+
+require 'digest/md5'
+require 'osx/cocoa'
+require 'yaml'
+require 'cgi'
+include OSX
+require 'fileutils'
+include FileUtils
+OSX.require_framework 'ScriptingBridge'
+
+
+
+Itunes = SBApplication.applicationWithBundleIdentifier_("com.apple.iTunes")
+SkippedFiles = []
+
+class PossibleDuplicateTrack
+  # track is the iTunes track object that will be moved or deleted
+  attr_accessor :unix_path, :database_id, :md5
+
+  def initialize(itunesTrack, md5)
+    @database_id = itunesTrack.databaseID.to_i
+    @unix_path = unixify_path( itunesTrack.location.to_s ) 
+    @md5 = md5
+  end
+
+  def remove_from_itunes
+    predicate = OSX::NSPredicate.predicateWithFormat("databaseID == #{@database_id}")
+    #library_track = Itunes.sources.first.playlists[1].tracks.filteredArrayUsingPredicate(predicate).first
+    library_track = Itunes.sources.first.playlists.first.tracks.filteredArrayUsingPredicate(predicate).first
+    if library_track
+      library_track.delete
+      puts "  Removed from iTunes."
+      return true
+    else
+      puts "  ERROR trying to remove this track from iTunes. Please handle manually."
+      # NOTE This is probably a podcast
+      puts "  Continuing..."
+      sleep 2
+      return false
+    end
+  end
+
+  def delete_file
+    rm @unix_path
+    puts "  Deleted from file system."
+  end
+
+  def delete!
+    puts "Deleting: #@unix_path"
+    if remove_from_itunes
+      delete_file
+    end
+  end
+end
+
+def calculate_md5( file_path )
+  print "calculating md5 for file: #{file_path}"
+  print " --> "
+  STDOUT.flush
+  
+  #md5 = Digest::MD5.hexdigest( File.read( file_path ) )
+  raw = `openssl dgst -md5 "#{file_path}"`
+  md5 = raw.split[-1]
+  print md5
+  puts
+  md5
+end
+
+def unixify_path(path)
+  # CGI strips out literal +, etc. signs in the path, but we need to preserve them.
+  ['+', "'", '(', ')'].each do |symbol|
+    path.gsub!(symbol, CGI.escape(symbol))
+  end
+  path.gsub!('$', '\$')
+  CGI.unescape( path.sub(/^(file:\/\/localhost)/, '') ).strip
+end
+alias :unixify :unixify_path
+
+def normalize_path(path)
+  path.sub(/\s\d(.\w{3})$/, '\1')
+end
+
+# Takes an array of tracks and returns a hash where the keys are an MD5 digest
+# and the values are arrays which contain one or more PossibleDuplicateTrack
+# objects.
+def find_duplicates( tracks, run_limit = nil, mb_limit=nil )
+  stop_counter = 0
+  md5_tracks = Hash.new {|hash, key| hash[key] = []} # will be keyed by MD5 hash
+  tracks.each do |track|
+    track_file_location = unixify_path( track.location.to_s ) 
+    unless track_file_location && track_file_location.strip != ''
+      puts ">>> No file location for #{track.name}. Removing this track from iTunes..."
+      x = PossibleDuplicateTrack.new(track, nil)
+      x.remove_from_itunes
+      next
+    end
+    unless File.exist?(track_file_location)
+      puts ">>> No file found for #{track.name} at #{track_file_location}" 
+      SkippedFiles << track_file_location
+      next
+    end
+    mb = File.size(track_file_location) / (1024 * 1024)
+    if mb_limit && mb > mb_limit
+      puts "SKIPPING FILE DUE TO EXCESS SIZE (#{mb}MB): #{track.name} : #{track_file_location}"
+      SkippedFiles << track_file_location
+      puts "Please deal with any duplicates of this file manually"
+      next
+    end
+
+    # We will use the MD5 digest to identify identical files.
+    md5 = calculate_md5( track_file_location )
+    if md5.nil? # missing file or bad file path
+      SkippedFiles << track_file_location
+      next
+    end
+    # This might be a duplicate files
+    md5_tracks[md5] << PossibleDuplicateTrack.new( track, md5 )
+    stop_counter += 1
+    if run_limit && stop_counter > run_limit
+      break 
+    end
+  end
+  puts "=" * 40
+  # puts "Total Tracks: #{tracks.size}" # This number is confusing 
+  puts "Total Unique Media Files: #{md5_tracks.keys.size}"
+  tracks_with_dups = md5_tracks.delete_if {|key, value| value.size < 2} 
+  puts "Tracks With Duplicates: #{tracks_with_dups.keys.size}"
+  puts "=" * 40
+  return tracks_with_dups
+end
+
+# Takes an array of PossibleDuplicateTracks with the same MD5 signature and
+# figures out which one with the duplicate simply by choosing the one(s) with the 
+# longer filename.
+def determine_duplicate( tracks )
+  # Get the track with the shortest unix_path
+  tracks = tracks.sort_by {|track| File.basename( track.unix_path ).size } # shortest name first
+  original = tracks.shift
+  return original, tracks
+end
+
+def run(pattern=nil, run_limit=nil)
+  # Load the iTunes instance
+  title = "itunes duplicate track removal machine"
+  width = 80
+  puts "-" * width
+  puts
+  puts "itunes duplicate track removal machine".center(width)
+  puts
+  puts "by daniel choi".center(width)
+  puts "betahouse".center(width)
+  puts "cambridge, ma, usa".center(width)
+  puts 
+  puts "contact: dhchoi@gmail.com".center(width)
+  puts
+  puts "-" * width
+  
+  puts
+  sleep 1
+  puts <<END
+Running program. It may take several minutes to analyze all the tracks in your
+iTunes library.
+END
+  sleep 2
+  puts 
+  puts <<END
+To avoid any potential hangs and crashes, don't use iTunes while the program is
+running.
+END
+  sleep 2
+  puts
+  puts <<END
+When this program is done analyzing your tracks, it will show you the duplicates
+it has detected and ask for your confirmation before removing any files. So feel
+free to go get some coffee and come back.
+END
+  puts
+  sleep 1
+  puts "To make the program run faster, we can skip iTunes files that are very large."
+  puts "How many megabytes do you want to set as the maximum? "
+  print "(Default: 100mb. Type 'none' for no limit): "
+  response = gets
+  if response =~ /none/
+    mb_limit = nil
+  elsif response.to_i == 0 
+    mb_limit = 100
+  else 
+    mb_limit = response.to_i
+  end
+  if mb_limit
+    puts "Setting maximum file size to analyze to #{mb_limit}MB."
+  else
+    puts "Setting maximum file size to analyze to unlimited."
+  end
+  sleep 3
+  source = Itunes.sources.first 
+  puts "Using iTunes source: #{source.name}"
+  puts "Using iTunes playlist: #{source.playlists.first.name}"
+
+  duplicate_tracks = []
+  fileTracks = source.libraryPlaylists[0].fileTracks
+  if pattern # filter tracks
+    fileTracks = fileTracks.select {|track| track.name.to_s =~ pattern}
+  end
+  puts "Calculating MD5 digests for all tracks with files..." 
+  sleep 2
+  puts
+  tracks_with_dups = find_duplicates( fileTracks, run_limit, mb_limit )
+  puts "Finding duplicates..."
+  puts
+  sleep 2
+  
+  # Now do something with the duplicate tracks
+  puts "=" * 40
+  SkippedFiles.compact! 
+  SkippedFiles.reject! {|x| x.strip == ''}
+  unless SkippedFiles.empty?
+    puts "#{SkippedFiles.length} files were skipped, either because they could not be found or because they were too large (over #{mb_limit}MB) to analyze quickly. Also, files may be skipped because their path contains non-standard characters. Please look for duplicates in these files manually."
+    puts
+    puts "Please handle any duplicates of these special files manually."
+    puts
+    puts "See these files? (y/n) (default: n) "
+    if gets.strip.downcase == "y"
+      puts "Skipped Files:"
+      SkippedFiles.each do |file|
+        puts file
+      end
+      puts "=" * 40
+      puts "Continue? (y/n)"
+      if gets =~ /^n/
+        puts "Aborted."
+        exit
+      end
+
+    end
+  end
+
+  puts "FOUND DUPLICATES:"
+  sleep 1
+  if tracks_with_dups.empty?
+    puts "No true duplicates found."
+    puts "Exiting."
+    display_note
+    exit 
+  end
+  tracks_with_dups.each do |k,tracks|
+    puts "#{k} :"
+    original, dups = determine_duplicate( tracks )
+    puts "  #{original.database_id} : #{original.unix_path} <-- original"
+    dups.each do |track|
+      puts "  #{track.database_id} : #{track.unix_path} <-- duplicate"
+    end
+    duplicate_tracks << dups 
+  end
+  duplicate_tracks.flatten!
+
+  num_to_remove = duplicate_tracks.length
+  print "Go ahead and delete the #{num_to_remove} files marked 'duplicate'? (y/n) (default: y) "
+
+  unless gets.strip.downcase == "n"
+    puts "OK, here goes..."
+    # DELETE EM
+    duplicate_tracks.each {|track| track.delete!}
+    sleep 2
+    puts "Done. #{num_to_remove} duplicates removed."
+    display_note
+  else
+    puts "Canceled."
+  end
+end
+
+def display_note
+  puts
+  puts <<-END
+# NOTE: This program is designed to detect and remove only exact duplicates.
+# Some apparent duplicates may be left intact in the iTunes library after the
+# program runs. These are not true duplicates because their MD5 hashes do not
+# match for one reason or another. It could be that one version of the track is
+# protected and the other is not; that one version is compressed more than the
+# other, or that one version uses different protection encryption, or that the
+# versions simply have a few bits set differently for whatever reason. Please
+# check the files manually to be sure. 
+  END
+  puts
+end

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification 
+name: itunes-rm-dups
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: universal-darwin-9
+authors: 
+- Daniel Choi
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-05-21 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: This ruby gem allows a OS X Leopard user to remove duplicate tracks from her iTunes library and file system from the command line. Duplicates are detected robustly, using MD5 digests of track files instead of iTunes metadata matching, which can produce false positives and false negatives.  The program is a ruby script that use RubyCocoa and the OS X Cocoa Scripting Bridge.  To run the program, type itunes-rm-dups on the command line and follow the prompts.
+email: dhchoi@gmail.com
+executables: 
+- itunes-rm-dups
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- lib/itunes_dup_handler.rb
+- README
+has_rdoc: true
+homepage: http://cesareborgia.com/software/itunes-rm-dups/
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: Remove duplicate tracks from an iTunes library and file system from the command line.
+test_files: []
+