fingerprint 1.3.1 → 3.1.0

data/lib/fingerprint/command.rb

@@ -0,0 +1,90 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+require_relative 'scanner'
+
+require_relative 'command/scan'
+require_relative 'command/analyze'
+require_relative 'command/verify'
+require_relative 'command/compare'
+require_relative 'command/duplicates'
+
+module Fingerprint
+	module Command
+		def self.call(*args)
+			Top.call(*args)
+		end
+		
+		class Top < Samovar::Command
+			self.description = "A file checksum analysis and verification tool."
+			
+			options do
+				option '--root <path>', "Work in the given root directory."
+				
+				option '-o/--output <path>', "Output the transcript to a specific file rather than stdout."
+				
+				option '-h/--help', "Print out help information."
+				option '-v/--version', "Print out the application version."
+			end
+			
+			def chdir(&block)
+				if root = @options[:root]
+					Dir.chdir(root, &block)
+				else
+					yield
+				end
+			end
+			
+			def output
+				if path = @options[:output]
+					File.open(path, "w")
+				else
+					$stdout
+				end
+			end
+			
+			nested :command, {
+				'scan' => Scan,
+				'analyze' => Analyze,
+				'verify' => Verify,
+				'compare' => Compare,
+				'duplicates' => Duplicates
+			}, default: 'analyze'
+			
+			def call
+				if @options[:version]
+					puts "fingerprint v#{VERSION}"
+				elsif @options[:help]
+					self.print_usage
+				else
+					chdir do
+						@command.call
+					end
+				end
+			end
+		end
+	end
+end

data/lib/fingerprint/command/analyze.rb

@@ -0,0 +1,74 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+require 'fileutils'
+
+require_relative '../checksums'
+require_relative '../scanner'
+require_relative '../record'
+
+module Fingerprint
+	module Command
+		class Analyze < Samovar::Command
+			self.description = "Generates a fingerprint for the specified paths and saves it."
+			
+			options do
+				option "-n/--name <name>", "The fingerprint file name.", default: INDEX_FINGERPRINT
+				
+				option "-f/--force", "Force all operations to complete despite warnings."
+				option "-x/--extended", "Include extended information about files and directories."
+				option "-s/--checksums <SHA2.256>", "Specify what checksum algorithms to use: #{CHECKSUMS.keys.join(', ')}.", default: DEFAULT_CHECKSUMS, type: Checksums
+				
+				option "--progress", "Print structured progress to standard error."
+				option "--verbose", "Verbose fingerprint output, e.g. excluded paths."
+			end
+			
+			many :paths, "Paths relative to the root to use for verification, or pwd if not specified.", default: ["./"]
+			
+			def call
+				output_file = @options[:name]
+
+				if File.exist?(output_file) and !@options[:force]
+					abort "Output file #{output_file} already exists. Aborting."
+				end
+
+				options = @options.dup
+				options[:excludes] = [File.expand_path(options[:name], Dir.pwd)]
+
+				finished = false
+				begin
+					File.open(output_file, "w") do |io|
+						options[:output] = io
+
+						Scanner.scan_paths(@paths, **options)
+					end
+					finished = true
+				ensure
+					FileUtils.rm(output_file) unless finished
+				end	
+			end
+		end
+	end
+end

data/lib/fingerprint/command/compare.rb

@@ -0,0 +1,55 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+module Fingerprint
+	module Command
+		class Compare < Samovar::Command
+			self.description = "Compare two fingerprints and report additions, removals and changes."
+			
+			options do
+				option "-x/--extended", "Include extended information about files and directories."
+				option "-a/--additions", "Report files that have been added to the copy."
+				option "--fail-on-errors", "Exit with non-zero status if errors are encountered."
+				
+				option "--progress", "Print structured progress to standard error."
+			end
+			
+			one :master, "The fingerprint which represents the original data."
+			one :copy,  "The fingerprint which represents a copy of the data."
+			
+			def call
+				options = @options.dup
+				options[:output] = @parent.output
+				
+				error_count = Checker.check_files(@master, @copy, **options)
+
+				if @options[:fail_on_errors]
+					abort "Data inconsistent, #{error_count} error(s) found!" if error_count != 0
+				end
+			end
+		end
+	end
+end

data/lib/fingerprint/command/duplicates.rb

@@ -0,0 +1,98 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+module Fingerprint
+	module Command
+		class Duplicates < Samovar::Command
+			self.description = "Efficiently find duplicates in a given fingerprint."
+			
+			options do
+				option "-i/--inverse", "Invert the output, i.e. show files which are not duplicates."
+				option "-x/--extended", "Include extended information about files and directories."
+				
+				option "--verbose", "Verbose output, e.g. what is happening."
+			end
+			
+			one :master, "The source fingerprint which represents the primarily file list."
+			many :copies, "Zero or more fingerprints which might contain duplicates.", default: []
+			
+			attr :duplicates_recordset
+			
+			def call
+				@options[:output] = @parent.output
+				
+				@duplicates_recordset = RecordSet.new
+				results = RecordSetPrinter.new(duplicates_recordset, @options[:output])
+				
+				master_file_path = @master
+				File.open(master_file_path) do |master_file|
+					master_recordset = RecordSet.new
+					master_recordset.parse(master_file)
+					
+					ignore_similar = false
+					
+					copy_file_paths = @copies
+					
+					if copy_file_paths.size == 0
+						copy_file_paths = [master_file_path]
+						ignore_similar = true
+					end
+					
+					copy_file_paths.each do |copy_file_path|
+						File.open(copy_file_path) do |copy_file|
+							copy_recordset = RecordSet.new
+							copy_recordset.parse(copy_file)
+							
+							copy_recordset.records.each do |record|
+								record.metadata['fingerprint'] = copy_file_path
+								# We need to see if the record exists in the master
+								
+								if @options[:verbose]
+									$stderr.puts "Checking #{record.inspect}"
+								end
+								
+								main_record = master_recordset.find_by_key(record)
+								
+								# If we are scanning the same index, don't print out every file, just those that are duplicates within the single file.
+								if ignore_similar && main_record && (main_record.path == record.path)
+									main_record = nil
+								end
+								
+								if main_record
+									record.metadata['original.path'] = main_record.path
+									record.metadata['original.fingerprint'] = master_file_path
+									results << record if !@options[:inverse]
+								else
+									results << record if @options[:inverse]
+								end
+							end
+						end
+					end
+				end
+			end
+		end
+	end
+end

data/lib/fingerprint/command/scan.rb

@@ -0,0 +1,61 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+require_relative '../checksums'
+require_relative '../scanner'
+require_relative '../record'
+
+module Fingerprint
+	module Command
+		class Scan < Samovar::Command
+			self.description = "Generate a fingerprint from the given paths."
+			
+			options do
+				option "-p/--path <path>", "Analyze the given path relative to root.", default: "./"
+				
+				option "-x/--extended", "Include extended information about files and directories."
+				option "-s/--checksums <SHA2.256>", "Specify what checksum algorithms to use: #{CHECKSUMS.keys.join(', ')}.", default: DEFAULT_CHECKSUMS, type: Checksums
+				
+				option "--progress", "Print structured progress to standard error."
+				option "--verbose", "Verbose fingerprint output, e.g. excluded paths."
+			end
+			
+			many :paths, "Paths to scan."
+			
+			def call
+				@paths = [Dir.pwd] unless @paths
+				
+				options = @options.dup
+				
+				# This configuration ensures that the output is printed to $stdout.
+				options[:output] = @parent.output
+				options[:recordset] = nil
+				
+				Scanner.scan_paths(@paths, **options)
+			end
+		end
+	end
+end

data/lib/fingerprint/command/verify.rb

@@ -0,0 +1,86 @@
+# Copyright, 2016, by Samuel G. D. Williams. <http://www.codeotaku.com>
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# This script takes a given path, and renames it with the given format. 
+# It then ensures that there is a symlink called "latest" that points 
+# to the renamed directory.
+
+require 'samovar'
+
+require_relative '../checker'
+require_relative '../record'
+
+module Fingerprint
+	module Command
+		class Verify < Samovar::Command
+			self.description = "Check an existing fingerprint against the filesystem."
+			
+			options do
+				option "-n/--name <name>", "The fingerprint file name.", default: INDEX_FINGERPRINT
+				
+				option "-f/--force", "Force all operations to complete despite warnings."
+				option "-x/--extended", "Include extended information about files and directories."
+				
+				option "-s/--checksums <SHA2.256>", "Specify what checksum algorithms to use (#{Fingerprint::CHECKSUMS.keys.join(', ')}).", default: Fingerprint::DEFAULT_CHECKSUMS
+				
+				option "--progress", "Print structured progress to standard error."
+				option "--verbose", "Verbose fingerprint output, e.g. excluded paths."
+				
+				option "--fail-on-errors", "Exit with non-zero status if errors are encountered."
+			end
+			
+			many :paths, "Paths relative to the root to use for verification, or ./ if not specified.", default: ["./"]
+			
+			attr :error_count
+			
+			def call
+				input_file = @options[:name]
+
+				unless File.exist? input_file
+					abort "Can't find index #{input_file}. Aborting."
+				end
+
+				options = @options.dup
+				options[:output] = @parent.output
+
+				master = RecordSet.new
+
+				File.open(input_file, "r") do |io|
+					master.parse(io)
+				end
+
+				if master.configuration
+					options.merge!(master.configuration.options)
+				end
+
+				scanner = Scanner.new(@paths, **options)
+				
+				# We use a sparse record set here, so we can't check for additions.
+				copy = SparseRecordSet.new(scanner)
+
+				@error_count = Checker.verify(master, copy, **options)
+				
+				if @options[:fail_on_errors]
+					abort "Data inconsistent, #{error_count} error(s) found!" if error_count != 0
+				end
+			end
+		end
+	end
+end