ckfiles 0.9.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 17f9e5e46d09082ee214c6a6960723fb488ece8a
+  data.tar.gz: 26f5b9ca6eb40eac984baabf730b92666587f3f0
+SHA512:
+  metadata.gz: fdc62bcd94914e0edb30ea507e3ca47bcdf09b39a5ab536551a165b6d212fe418443ed1734c730beb5366182044a7e372a11519c9ede3e24fea9291aa96c1eee
+  data.tar.gz: 7e7fb47bbc7e42ab80f6d55255bb93e3ab4ff72a23e3da59fd85c62e8ffa8b68150ff46a530427bd3beca3dfbb6940499814595ba525a23921c12c9fdac3f774

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Mac Ma
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,38 @@
+ckfiles
+==========================
+General file integrity checker, can check recursively. Support SFV, MD5, ZIP, CBZ, 7Z, GZ, BZ2, LHA, LZH, ARJ, CHM, XZ, RAR, CBR and media file containing CRC32 sum in filename with extension with AVI, MKV, MP4, OGM, ASF, RM, RAM, WEBM. Require 7zip and Unrar for checking archive file. Supports Linux / Mac OS X / Windows.  
+  
+Example filename:  Big Bunny [c001beef].avi
+
+#### Installation: ####
+Linux:
+1. sudo apt-get install ruby1.9.1-full ruby1.9.1-dev p7zip-full p7zip-rar
+2. sudo gem install ckfiles
+
+Mac OS X:
+1. Install [MacPorts](http://www.macports.org)
+2. sudo port install ruby20 rb-rubygems p7zip
+3. sudo gem install ckfiles
+
+Windows: (NTFS Only)
+1. Install Ruby v2.* from [RubyInstaller](http://rubyinstaller.org)
+2. Install Development Kit for v2.0 from same site as above
+2. gem install ckfiles
+3. Download and install 7z from [7-Zip](http://www.7-zip.org), copy 7z.exe and 7z.dll to bin path (e.g. C:\Ruby200\bin )
+
+All OS:
+Download and install [WinRAR/unrar](http://rarlab.com), copy unrar(.exe) to bin path
+
+#### Caveat/Notes: ####
+This program uses filesystem's extended attributes, if the filesystem doesn't support the extended attribute or isn't enabled, then you can only use the read only function. This also applies to any network drive.
+
+#### Usage: ####
+ckfiles \[arguments\] \[directory\] \<directory2\> \<directory3\>
+
+#### Arguments: ####
+	--recursive  -r  Recursively check all files
+	--noop       -n  Not moving corrupted files
+	--read-only  -x  Read Only, do not write attribute (imply -n)
+	--expiry     -e  Expiry time since last check, can be in seconds, or m/h/d/w/mo/y
+	--outfile    -o  Output error to a file
+	--help       -h  Display current help message

data/bin/ckfiles

@@ -0,0 +1,319 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+=begin
+#
+# General file integrity checker, can check recursively. Support SFV, MD5, ZIP, CBZ, 7Z, GZ, BZ2, LHA, LZH, ARJ, CHM, XZ, RAR, CBR and media file containing CRC32 sum in filename with extension  with AVI, MKV, MP4, OGM, ASF, RM, RAM, WEBM. Require 7zip and Unrar for checking archive file. Supports Linux / Mac OS X / Windows.
+#
+
+
+# attribute format
+v1
+  ckfiles.result   attribute version, unix epoxy time
+example
+  ckfiles.result   1, 1385724368
+=end
+
+# making sure program runs in UTF-8 encoding in any environment
+Encoding.default_external = Encoding::UTF_8
+Encoding.default_internal = Encoding::UTF_8
+
+
+# make print behave as print
+STDOUT.sync = true
+
+# executable program sanity check
+if `7z -h` =~ /Igor Pavlov/
+	$HAVE_BIN_7Z = true
+else
+	$HAVE_BIN_7Z = false
+	puts "***** NOTICE: 7z not installed *****"
+end
+
+if `unrar -v` =~ /Alexander Roshal/
+	$HAVE_BIN_UNRAR = true
+else
+	$HAVE_BIN_UNRAR = false
+	puts "***** NOTICE: unrar not installed *****"
+end
+
+
+require 'ffi-xattr'
+require 'fileutils'
+require 'ckfiles/libs'
+require 'digest/md5'
+require 'getoptlong'
+
+if RUBY_PLATFORM =~ /mingw/
+	# Windows support
+	# using Windows API to directly list files and directory
+	# otherwise unicode filenames will be be supported, it will all be in ?????? characters (hex falue 3F)
+
+	# ensure command prompt runs in unicode
+	`chcp 65001`
+
+	co = `chcp`.scan(/[0-9]+/).join.to_i
+	if co != 65001
+		puts 'Fail to run in unicode mode!'
+		exit
+	end
+
+	require 'win32ole'
+	$fso = WIN32OLE.new('Scripting.FileSystemObject')
+end
+
+
+# start up environment
+$noop = false
+$read_only = false
+$expiry = 3600*24*90 # expiry time default to 90 days
+$recursive = false
+$outfile = false
+
+def tpass( f )
+	if $read_only == false
+		begin
+			fd = Xattr.new( f )
+			
+			param = 'ckfiles.result'
+			param = 'user.' + param if RUBY_PLATFORM =~ /linux/
+			fd.set(param, "1, #{Time.now.to_i.to_s}")
+
+			puts "pass - #{ f }"
+		rescue => errmsg
+			puts "pass - set attr failed: #{ f }"
+		end
+	else
+		puts "pass - #{ f }"
+	end
+end
+
+def tfail( f )
+	s = "fail - #{ f }"
+	puts s
+	File.open($outfile, 'ab') { |io| io << s + "\r\n" } if $outfile # write failed result to file
+	
+	if $noop == false and $read_only == false
+		newdir = File.dirname(f) + '/Corrupted'
+		Dir.mkdir( newdir ) if ! File.directory?( newdir )
+		File.rename( f , "#{ newdir }/#{ File.basename(f) }" )
+	end
+end
+
+# is file already passed?
+def File.passed?( file, expiry=$expiry )
+	fd = Xattr.new( file )
+	param = 'ckfiles.result'
+	param = 'user.' + param if RUBY_PLATFORM =~ /linux/
+	begin
+		iresult = fd.get(param).split(', ')
+		iver = iresult[0]
+		itime = iresult[1].to_i
+		
+		# file already passed, skip
+		if ( Time.now.to_i - itime ) < expiry
+			return true
+		end
+	rescue
+		# attribute not found, so continue
+	end
+	return false
+end
+
+
+def show_help
+	puts \
+%Q{Usage:
+ckfiles [arguments] [dir] <dir2> <dir3> ...
+
+Arguments:
+--recursive  -r  Recursively check all files
+--noop       -n  Not moving corrupted files
+--read-only  -x  Read Only, do not write attribute (imply -n)
+--expiry     -e  Expiry time since last check, can be in seconds, or m/h/d/w/mo/y
+--outfile    -o  Output error to a file
+--help       -h  Display current help message}
+end
+
+
+#<main>
+opts = GetoptLong.new(
+	[ '--recursive', '-r',  GetoptLong::NO_ARGUMENT ],
+	[ '--noop',      '-n',  GetoptLong::NO_ARGUMENT ],
+	[ '--read-only', '-x',	GetoptLong::NO_ARGUMENT ],
+	[ '--expiry',	 '-e',	GetoptLong::REQUIRED_ARGUMENT ],
+	[ '--outfile',	 '-o',	GetoptLong::REQUIRED_ARGUMENT ],
+	[ '--help',      '-h',  GetoptLong::NO_ARGUMENT ],
+)
+
+opts.each { |opt,arg|
+	case opt
+		when '--help'
+			show_help
+			exit
+		when '--recursive'
+			$recursive = true
+		when '--noop'
+			$noop = true
+		when '--read-only'
+			$noop = true
+			$read_only = true
+		when '--expiry'
+			$expiry = arg.timelength_to_i
+		when '--outfile'
+			$outfile = arg
+	end
+}
+
+
+dirs = []
+if ARGV.length > 0
+	for dir in ARGV
+		if File.directory?( dir )
+			dirs << File.expand_path( dir )
+		else
+			puts "dir don't exists #{dir}"
+		end
+	end
+else
+	show_help
+	exit
+end
+
+
+def chkDir(dir)
+	files = []
+
+	if RUBY_PLATFORM =~ /mingw/
+		# Windows API call for listing files and folders
+
+		gf = $fso.GetFolder(dir)
+		for file in gf.Files
+			files << file.path
+		end
+
+		for folder in gf.SubFolders
+			files << folder.path
+		end
+	else
+		# POSIX supported file/directory listing
+
+		files = Dir.entries(dir)
+		files.delete_if { |f| f[0] == '.' and f[-1] == '.' }
+		files.collect! { |f| dir + '/' + f }
+	end
+
+	for f in files
+		# skip restricted file/directory
+		next if Dir.restricted?( f )
+
+		if File.file?( f )
+			# skip unwanted files or non-regular file
+			next if File.basename( f ) =~ /^\._.+/
+			next if File.basename( f ) =~ /^\..+$/
+			next if File.exists?( f + '.lftp-pget-status' )
+
+			# skip already passed file
+			next if File.passed?( f )
+
+
+			if f =~ /\.(zip|cbz|7z|gz|bz2|lha|lzh|arj|chm|xz)$/i
+				# skip if 7z executable isn't found
+				next unless $HAVE_BIN_7Z
+
+				# to go around "invalid byte sequence in UTF-8" error
+				result = `7z t -pAAA "#{f}" 2>&1`.encode('UTF-8', 'binary', invalid: :replace, undef: :replace, replace: '')
+				
+				if result =~ /Everything is Ok/
+					tpass( f )
+				else
+					tfail( f )
+				end
+							
+			elsif f =~ /\.(rar|cbr)$/i
+				# skip if unrar executable isn't found
+				next unless $HAVE_BIN_UNRAR
+
+				if f =~ /\.part[0-9]+\.rar$/i
+					# skip file if the part rar file isn't the first one
+					next if (f =~ /\.part[0]+1\.rar$/i) == nil
+				end
+				
+				if `unrar t -p- "#{f}" 2>&1` =~ /All OK/
+					tpass( f )
+				else
+					tfail( f )
+				end
+				
+			elsif f =~ /(\[|\()([0-9a-fA-F]{8})(\]|\))\.(avi|mkv|mp4|ogm|asf|rm|ram|webm)$/i
+				# check media file containing crc32 sum at end of filename
+
+				fcrc = File.crc32(f)
+				
+				if f.downcase.include?(fcrc)
+					tpass( f )
+				else
+					tfail( f )
+				end
+				
+			elsif f =~ /\.sfv$/i
+				# check files contained in sfv
+
+				ck_files = load_cksum_list( 'sfv', f )
+				
+				for file, sum in ck_files
+					file = File.dirname( f ) + '/' + file
+					
+					if ! File.file?( file )
+						puts "SFV: missing #{file}"
+						next
+					end
+					
+					next if File.passed?( file )
+					
+					fcrc = File.crc32(file)
+					
+					if sum.downcase == fcrc.downcase
+						tpass( file )
+					else
+						tfail( file )
+					end
+				end
+			elsif f =~ /\.md5$/i
+				ck_files = load_cksum_list( 'md5', f )
+				
+				for file, sum in ck_files
+					next unless file and sum
+
+					file = File.dirname( f ) + '/' + file
+					
+					if ! File.file?( file )
+						puts "MD5: missing #{file}"
+						next
+					end
+					
+					next if File.passed?( file )
+					
+					fmd5 = File.md5( file )
+					
+					if sum.downcase == fmd5.downcase
+						tpass( file )
+					else
+						tfail( file )
+					end
+				end
+			end
+			
+		elsif File.directory?( f ) and $recursive
+			chkDir(f)
+		end
+	end
+end
+
+
+for dir in dirs
+	chkDir(dir)
+end
+
+puts 'all done'
+#</main>

data/ckfiles.gemspec

@@ -0,0 +1,21 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path('../lib/ckfiles/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = 'ckfiles'
+  s.version     = Ckfiles::VERSION
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.description = "General file integrity checker, can check recursively. Support SFV, MD5, ZIP, CBZ, 7Z, GZ, BZ2, LHA, LZH, ARJ, CHM, XZ, RAR, CBR and media file containing CRC32 sum in filename with extension  with AVI, MKV, MP4, OGM, ASF, RM, RAM, WEBM. Require 7zip and Unrar for checking archive file. Supports Linux / Mac OS X / Windows."
+  s.summary     = s.description
+  s.authors     = ["Mac Ma"]
+  s.homepage    = 'https://github.org/comomac/ckfiles'
+  s.license     = 'MIT'
+
+  s.add_dependency 'ffi-xattr', '>= 0.0.5'
+
+  s.require_paths = ["lib"]
+  s.files = `git ls-files`.split($\)
+  s.executables = s.files.grep(%r{^bin/}).map { |f| File.basename(f) }
+end

data/lib/ckfiles/libs.rb

@@ -0,0 +1,129 @@
+require 'zlib'
+
+# check if directed is restricted
+def Dir.restricted?( dir, excludes=[] )
+	rdirs = [
+		/\.Trash/, /\.Trashes/, /\.fseventsd/, /\.Spotlight-V100/, /\.DocumentRevisions-V100/,
+		/\.git/,
+		/\/\./,
+		/\.\$EXTEND/,
+		/_SYNCAPP/,
+		/Corrupted/,
+		/System Volume Information/, /RECYCLER/,
+		/backup/i,
+		/\/Library\//,
+		/\.sparsebundle/,
+		/\.tmpdir/, /\.tmp7z/,
+		/\.AppleDouble/
+	]
+
+	for word in rdirs - excludes
+	    return true if dir.match( word )
+    end
+    return false
+end
+
+def load_cksum_list( sumtype, file )
+	# returns { file1=>sum, file2=>sum, file3=>sum ... fileN=>sum }
+	
+	if sumtype == 'sfv'
+		file_ext = 'sfv'
+	elsif sumtype == 'md5'
+		file_ext = 'md5'
+	else
+		raise 'not supported cksum type'
+	end
+	
+	sfs = File.new( file ).readlines
+	sfs.collect! { |y| y.chomp }
+	begin
+		sfs = sfs.collect { |y| y.gsub(/\\/,'/') }
+	rescue => errmsg
+		puts "ERROR: #{errmsg}  FILE:  #{file}"
+	end
+	sfs.flatten!
+	sfs.compact!
+	sfs.delete_if { |y| y == '' }
+	sfs.delete_if { |y| y[0..0] == '#' }
+	sfs.delete_if { |y| y[0..0] == ';' }
+	sfs.delete_if { |y| y[0..4] == 'MD5 (' } #this format is a pain so drop
+	
+	n = {}	
+	if sumtype == 'sfv'
+		for s in sfs
+			n[s[0..-10]] = s[-8..-1]
+		end
+	elsif sumtype == 'md5'
+		for s in sfs
+			n[s[34..-1]] = s[0..31]
+		end
+	end
+	n.each { |f,sum| puts "#{sum} #{f}" } if $debug
+
+	if n.length > 0
+		n
+	else
+		{}
+	end	
+end
+
+# return crc32 sum for file
+def File.crc32(f)
+	ptr_f = File.new(f,'rb')
+	
+	r = 0
+	while ! ptr_f.eof?
+		r = Zlib.crc32( ptr_f.read(1024**2) , r )
+	end
+	ptr_f.close
+	
+	r.to_s(16).rjust(8).gsub(' ','0').downcase
+end
+
+# return md5sum for file
+def File.md5(f)
+	Digest::MD5.file(f).hexdigest
+end
+
+class String
+	# translate length of time to integer format
+	# e.g. 3m -> 180 (seconds)
+	def timelength_to_i
+		units = {
+			'm'=>60,
+			'h'=>60*60,
+			'd'=>60*60*24,
+			'w'=>60*60*24*7,
+			'mo'=>60*60*24*30,
+			'y'=>60*60*24*365
+		}
+		
+		u = 1
+		n = self.to_f
+		for unit, num in units
+			if self =~ /#{unit}$/i
+				u = num
+				break
+			end
+		end
+		
+		return (u*n).to_i
+	end
+
+	def escape_glob
+		# escape any character that will affect Dir.glob
+		self.gsub(/([\[\]\{\}\*\?\\])/, '\\\\\1')
+	end
+
+	def ascii
+		# return only ascii characters in array
+		regex = /[A-Za-z0-9\ \%\;\'\@\~\-\(\)\[\]\&\_\{\}\+\.\/\!\,\#]/
+		self.scan(regex)
+	end
+	
+	def not_ascii
+		# return only none ascii characters in array
+		regex = /[^A-Za-z0-9\ \%\;\'\@\~\-\(\)\[\]\&\_\{\}\+\.\/\!\,\#]/
+		self.scan(regex)
+	end
+end

data/lib/ckfiles/version.rb

@@ -0,0 +1,3 @@
+class Ckfiles
+	VERSION = "0.9.3"
+end

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: ckfiles
+version: !ruby/object:Gem::Version
+  version: 0.9.3
+platform: ruby
+authors:
+- Mac Ma
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-11-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi-xattr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.5
+description: General file integrity checker, can check recursively. Support SFV, MD5,
+  ZIP, CBZ, 7Z, GZ, BZ2, LHA, LZH, ARJ, CHM, XZ, RAR, CBR and media file containing
+  CRC32 sum in filename with extension  with AVI, MKV, MP4, OGM, ASF, RM, RAM, WEBM.
+  Require 7zip and Unrar for checking archive file. Supports Linux / Mac OS X / Windows.
+email: 
+executables:
+- ckfiles
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/ckfiles
+- ckfiles.gemspec
+- lib/ckfiles/libs.rb
+- lib/ckfiles/version.rb
+homepage: https://github.org/comomac/ckfiles
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: General file integrity checker, can check recursively. Support SFV, MD5,
+  ZIP, CBZ, 7Z, GZ, BZ2, LHA, LZH, ARJ, CHM, XZ, RAR, CBR and media file containing
+  CRC32 sum in filename with extension  with AVI, MKV, MP4, OGM, ASF, RM, RAM, WEBM.
+  Require 7zip and Unrar for checking archive file. Supports Linux / Mac OS X / Windows.
+test_files: []