fingerprint 1.4.0 → 3.0.1

data/lib/fingerprint/command/analyze.rb

@@ -0,0 +1,74 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+require 'fileutils'
+
+require_relative '../checksums'
+require_relative '../scanner'
+require_relative '../record'
+
+module Fingerprint
+	module Command
+		class Analyze < Samovar::Command
+			self.description = "Generates a fingerprint for the specified paths and saves it."
+			
+			options do
+				option "-n/--name <name>", "The fingerprint file name.", default: INDEX_FINGERPRINT
+				
+				option "-f/--force", "Force all operations to complete despite warnings."
+				option "-x/--extended", "Include extended information about files and directories."
+				option "-s/--checksums <SHA2.256>", "Specify what checksum algorithms to use: #{CHECKSUMS.keys.join(', ')}.", default: DEFAULT_CHECKSUMS, type: Checksums
+				
+				option "--progress", "Print structured progress to standard error."
+				option "--verbose", "Verbose fingerprint output, e.g. excluded paths."
+			end
+			
+			many :paths, "Paths relative to the root to use for verification, or pwd if not specified.", default: ["./"]
+			
+			def call
+				output_file = @options[:name]
+
+				if File.exist?(output_file) and !@options[:force]
+					abort "Output file #{output_file} already exists. Aborting."
+				end
+
+				options = @options.dup
+				options[:excludes] = [File.expand_path(options[:name], Dir.pwd)]
+
+				finished = false
+				begin
+					File.open(output_file, "w") do |io|
+						options[:output] = io
+
+						Scanner.scan_paths(@paths, options)
+					end
+					finished = true
+				ensure
+					FileUtils.rm(output_file) unless finished
+				end	
+			end
+		end
+	end
+end

data/lib/fingerprint/command/compare.rb

@@ -0,0 +1,55 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+module Fingerprint
+	module Command
+		class Compare < Samovar::Command
+			self.description = "Compare two fingerprints and report additions, removals and changes."
+			
+			options do
+				option "-x/--extended", "Include extended information about files and directories."
+				option "-a/--additions", "Report files that have been added to the copy."
+				option "--fail-on-errors", "Exit with non-zero status if errors are encountered."
+				
+				option "--progress", "Print structured progress to standard error."
+			end
+			
+			one :master, "The fingerprint which represents the original data."
+			one :copy,  "The fingerprint which represents a copy of the data."
+			
+			def call
+				options = @options.dup
+				options[:output] = @parent.output
+				
+				error_count = Checker.check_files(@master, @copy, options)
+
+				if @options[:fail_on_errors]
+					abort "Data inconsistent, #{error_count} error(s) found!" if error_count != 0
+				end
+			end
+		end
+	end
+end

data/lib/fingerprint/command/duplicates.rb

@@ -0,0 +1,97 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+module Fingerprint
+	module Command
+		class Duplicates < Samovar::Command
+			self.description = "Efficiently find duplicates in a given fingerprint."
+			
+			options do
+				option "-i/--inverse", "Invert the output, i.e. show files which are not duplicates."
+				option "-x/--extended", "Include extended information about files and directories."
+				
+				option "--verbose", "Verbose output, e.g. what is happening."
+			end
+			
+			one :master, "The source fingerprint which represents the primarily file list."
+			many :copies, "Zero or more fingerprints which might contain duplicates."
+			
+			attr :duplicates_recordset
+			
+			def call
+				@options[:output] = @parent.output
+				
+				@duplicates_recordset = RecordSet.new
+				results = RecordSetPrinter.new(duplicates_recordset, @options[:output])
+				
+				master_file_path = @master
+				File.open(master_file_path) do |master_file|
+					master_recordset = RecordSet.new
+					master_recordset.parse(master_file)
+					
+					ignore_similar = false
+					
+					copy_file_paths = @copies || []
+					if copy_file_paths.size == 0
+						copy_file_paths = [master_file_path]
+						ignore_similar = true
+					end
+					
+					copy_file_paths.each do |copy_file_path|
+						File.open(copy_file_path) do |copy_file|
+							copy_recordset = RecordSet.new
+							copy_recordset.parse(copy_file)
+							
+							copy_recordset.records.each do |record|
+								record.metadata['fingerprint'] = copy_file_path
+								# We need to see if the record exists in the master
+								
+								if @options[:verbose]
+									$stderr.puts "Checking #{record.inspect}"
+								end
+								
+								main_record = master_recordset.find_by_key(record)
+								
+								# If we are scanning the same index, don't print out every file, just those that are duplicates within the single file.
+								if ignore_similar && main_record && (main_record.path == record.path)
+									main_record = nil
+								end
+								
+								if main_record
+									record.metadata['original.path'] = main_record.path
+									record.metadata['original.fingerprint'] = master_file_path
+									results << record if !@options[:inverse]
+								else
+									results << record if @options[:inverse]
+								end
+							end
+						end
+					end
+				end
+			end
+		end
+	end
+end

data/lib/fingerprint/command/scan.rb

@@ -0,0 +1,61 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+require_relative '../checksums'
+require_relative '../scanner'
+require_relative '../record'
+
+module Fingerprint
+	module Command
+		class Scan < Samovar::Command
+			self.description = "Generate a fingerprint from the given paths."
+			
+			options do
+				option "-p/--path <path>", "Analyze the given path relative to root.", default: "./"
+				
+				option "-x/--extended", "Include extended information about files and directories."
+				option "-s/--checksums <SHA2.256>", "Specify what checksum algorithms to use: #{CHECKSUMS.keys.join(', ')}.", default: DEFAULT_CHECKSUMS, type: Checksums
+				
+				option "--progress", "Print structured progress to standard error."
+				option "--verbose", "Verbose fingerprint output, e.g. excluded paths."
+			end
+			
+			many :paths, "Paths to scan."
+			
+			def call
+				@paths = [Dir.pwd] unless @paths
+				
+				options = @options.dup
+				
+				# This configuration ensures that the output is printed to $stdout.
+				options[:output] = @parent.output
+				options[:recordset] = nil
+				
+				Scanner.scan_paths(@paths, options)
+			end
+		end
+	end
+end

data/lib/fingerprint/command/verify.rb

@@ -0,0 +1,86 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+require_relative '../checker'
+require_relative '../record'
+
+module Fingerprint
+	module Command
+		class Verify < Samovar::Command
+			self.description = "Check an existing fingerprint against the filesystem."
+			
+			options do
+				option "-n/--name <name>", "The fingerprint file name.", default: INDEX_FINGERPRINT
+				
+				option "-f/--force", "Force all operations to complete despite warnings."
+				option "-x/--extended", "Include extended information about files and directories."
+				
+				option "-s/--checksums <SHA2.256>", "Specify what checksum algorithms to use (#{Fingerprint::CHECKSUMS.keys.join(', ')}).", default: Fingerprint::DEFAULT_CHECKSUMS
+				
+				option "--progress", "Print structured progress to standard error."
+				option "--verbose", "Verbose fingerprint output, e.g. excluded paths."
+				
+				option "--fail-on-errors", "Exit with non-zero status if errors are encountered."
+			end
+			
+			many :paths, "Paths relative to the root to use for verification, or ./ if not specified.", default: ["./"]
+			
+			attr :error_count
+			
+			def call
+				input_file = @options[:name]
+
+				unless File.exist? input_file
+					abort "Can't find index #{input_file}. Aborting."
+				end
+
+				options = @options.dup
+				options[:output] = @parent.output
+
+				master = RecordSet.new
+
+				File.open(input_file, "r") do |io|
+					master.parse(io)
+				end
+
+				if master.configuration
+					options.merge!(master.configuration.options)
+				end
+
+				scanner = Scanner.new(@paths, options)
+				
+				# We use a sparse record set here, so we can't check for additions.
+				copy = SparseRecordSet.new(scanner)
+
+				@error_count = Checker.verify(master, copy, **options)
+				
+				if @options[:fail_on_errors]
+					abort "Data inconsistent, #{error_count} error(s) found!" if error_count != 0
+				end
+			end
+		end
+	end
+end

data/lib/fingerprint/find.rb

@@ -0,0 +1,40 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+require 'find'
+require 'build/files/path'
+require 'build/files/system'
+
+module Fingerprint
+	module Find
+		def self.find(root)
+			# Ensure root is a directory:
+			root += File::SEPARATOR unless root.end_with?(File::SEPARATOR)
+			
+			::Find.find(root) do |path|
+				yield Build::Files::Path.new(path, root)
+			end
+		end
+		
+		def self.prune
+			::Find.prune
+		end
+	end
+end

data/lib/fingerprint/record.rb

@@ -1,4 +1,4 @@
-# Copyright (c) 2011 Samuel G. D. Williams. <http://www.oriontransfer.co.nz>
+# Copyright, 2011, by Samuel G. D. Williams. <http://www.codeotaku.com>
 # 
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -25,6 +25,7 @@ module Fingerprint
 	MODES = {
 		:configuration => 'C',
 		:file => 'F',
+		:link => 'L',
 		:directory => 'D',
 		:summary => 'S',
 		:warning => 'W',
@@ -84,6 +85,16 @@ module Fingerprint
 	end
 	
 	class RecordSet
+		def self.load_file(path)
+			File.open(path, "r") do |io|
+				self.load(io)
+			end
+		end
+		
+		def self.load(io)
+			self.new.tap{|record_set| record_set.parse(io)}
+		end
+		
 		def initialize
 			@records = []
 			@paths = {}
@@ -115,6 +126,14 @@ module Fingerprint
 			end
 		end
 
+		def include?(path)
+			@paths.include?(path)
+		end
+		
+		def empty?
+			@paths.empty?
+		end
+
 		def lookup(path)
 			return @paths[path]
 		end