storazzo 0.3.8 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7af080e9d4d71c7ceb558b4b4cbd1f13345fa3c4dbea5d6060da8fb9d5b21cc0
-  data.tar.gz: fe4fd90e12756dbe535e96f0c7c42834ce44c452839890d381a0e43a78ce926e
+  metadata.gz: 9060103a49d0c2ec75f2f5d58bedd56f4fe0ac05d7c547874200a331480fa157
+  data.tar.gz: 1475ec303321f84850e63419b4d845cf282b57a36c384ab4f003000e79912cdb
 SHA512:
-  metadata.gz: 7f753c4e14018803a731de9cf992dcfcf4f4cac601fb0b978de6d16c83c8ab0f06430e0cdd2a203b025a714f168c08f0e6107606b9b354fcf792e0041be8d727
-  data.tar.gz: 072da4e8a2b24fedd83e1cd04b41f695a38a49e623cef5bd1643a11dfc8813604d32c7950976b6b5500839d2a2580293623ab7b32070d9e162db6a753f2d8d02
+  metadata.gz: b1ea60a508eae4c245c58214f8d90805ec21a9e06b0e604206ea4dd6b52e17e1547c58d15082e46e7b577f0430ad9211cedfb24cce9ab6674273eaaf6e69ce9e
+  data.tar.gz: '09fca42358436467874d6a433210a596b41fc8331a7e4a5ddb4aef1db113108a604a66dde0804dba4f2e886a99636404f6c457647dcc792b39301f3b4aaca61d'

data/Gemfile

@@ -4,3 +4,4 @@
 # to troubleshoot/debug things
 #gem 'rubocop'
 gem 'rake'
+gem 'pry'

data/VERSION

@@ -1 +1 @@
-0.3.8
+0.4.1

data/bin/stats-with-md5

@@ -0,0 +1,428 @@
+#!/usr/bin/env ruby
+
+require 'digest/md5'
+require 'digest'
+require 'socket'
+require 'optparse'       # http://ruby.about.com/od/advancedruby/a/optionparser.htm
+require 'date' # for DateTime
+require 'tempfile'
+
+if RUBY_VERSION.split('.')[0] == 1
+  puts "Refusing to launch a script form Ruby 1. Sorry Ric, its 2020 damn it!"
+  exit 2020
+end
+
+$PROG_VER = '0.5' # on 0.4 it was copied from palladius/sakura. Now should be maintained here and moved to 'stats-with-md5'
+$DEBUG    = false
+$reset_autowrite_file_reincarnation = 0
+DEFAULT_MAX_FILES = 'Infinite'
+
+=begin
+
+  ############################################################
+  @author:    Riccardo Carlesso
+  @email:     riccardo.carlesso@gmail.com
+  @maturity:  development
+  @language:  Ruby
+  @tags:      development, rcarlesso, test
+  @works_on:  Linux (little tested since v0.3), Mac (100% developed here)
+ ############################################################
+
+=end
+
+
+
+def deb(s);   puts "#DEB #{s}" if $DEBUG; end
+# colors 16
+def gray(s)    "\033[1;30m#{s}\033[0m" ; end
+def green(s)   "\033[1;32m#{s}\033[0m" ; end
+def red(s)     "\033[1;31m#{s}\033[0m" ; end
+def yellow(s)  "\033[1;33m#{s}\033[0m" ; end
+def blue(s)  "\033[1;34m#{s}\033[0m" ; end
+def purple(s)  "\033[1;35m#{s}\033[0m" ; end
+def azure(s)   "\033[1;36m#{s}\033[0m" ; end
+def white(s)   "\033[1;37m#{s}\033[0m" ; end
+
+# colors 64k
+def orange(s)   "\033[38;5;208m#{s}\033[0m" ; end
+
+# Program constants, automagically picked up by RicLib
+# More configuration could be written in:
+#    $GIC/etc/ricsvn/<FILENAME>.yml
+# That would go into the variable '$prog_conf_d'
+$myconf = {
+    :app_name            => "AppName should be sth like #{$0}",
+    :description         => "
+        This program gets stats from Ruby File.Stats library
+		plus computes MD5 or CRC32 of the file.
+		And bakes it in a way that can be easily parsed. 
+    ".strip.gsub(/^\s+/, "").gsub(/\s+$/, ""),
+}
+
+def usage(comment=nil)
+  puts white($optparse.banner)
+  puts($optparse.summarize)
+  puts("Description: " + gray($myconf[:description]))
+  puts red(comment) if comment
+  exit 13
+end
+
+def reset_autowrite_file()
+	$autowrite_file = "# #{File.basename $0} Autowrite v1.0 #{Time.now} reincarnation=#{$reset_autowrite_file_reincarnation}\n"
+	$reset_autowrite_file_reincarnation += 1 # ($reset_autowrite_file_reincarnation + 1) # rescue 1
+end 
+
+# include it in main if you want a custome one
+def init()    # see lib_autoinit in lib/util.rb
+  $opts = {}
+  # setting defaults
+  $opts[:verbose] = false
+  $opts[:dryrun] = false
+  $opts[:debug] = false
+  $opts[:autowrite] = false
+  $opts[:color] = true
+  $opts[:max_files] = nil # infinite
+
+  reset_autowrite_file()
+
+  $optparse = OptionParser.new do |opts|
+    opts.banner = "#{$0} v.#{$PROG_VER}\n Usage: #{File.basename $0} [options] file1 file2 ..."
+    opts.on( '-a', '--autowrite', 'automatically writes results to Root Folder (DFLT=false)' )  {  $opts[:autowrite] = true  }
+    opts.on( '-c', '--no-color', 'disables color (DFLT=false, that is color enabled)' )  {  $opts[:color] = false  }
+    opts.on( '-d', '--debug', 'enables debug (DFLT=false)' )  {  $opts[:debug] = true ; $DEBUG = true }
+    opts.on( '-h', '--help', 'Display this screen' )          {  usage }
+    opts.on( '-m', '--max-files NUMBER', "sets max files per Dir to X (DFLT=#{DEFAULT_MAX_FILES})" ) {|nfiles| $opts[:max_files] = nfiles.to_i }
+    opts.on( '-n', '--dryrun', "Don't really execute code" ) { $opts[:dryrun] = true }
+    opts.on( '-l', '--logfile FILE', 'Write log to FILE' )    {|file| $opts[:logfile] = file }
+    opts.on( '-v', '--verbose', 'Output more information' )   { $opts[:verbose] = true}
+  end
+  $optparse.parse!
+end
+
+def getSafeCreationTime(file)
+	File.ctime(file)
+end
+
+def compute_stats_and_md5(file)
+	ret={}
+	ret[:name] = file
+
+	begin
+		stats = File.stat(file)
+		ret[:stats_object] = stats # TODO deprecate
+		deb("Stats methods: #{stats.methods.sort.join(', ')}")
+		#deb(stats.ftype)
+		#puts(stats.methods)
+		deb(stats.ctime)
+		#puts(stats.birthtime rescue (stats.ctime))
+
+		# On Mac:
+		#<File::Stat                     
+		# dev=0x100000f,                  
+		# ino=1247768,                    
+		# mode=0100644 (file rw-r--r--),  
+		# nlink=1,
+		# uid=164825 (ricc),
+		# gid=89939 (primarygroup),
+		# rdev=0x0 (0, 0),
+		# size=564,
+		# blksize=4096,
+		# blocks=8,
+		# atime=2022-03-05 22:36:48.373362127 +0100 (1646516208),
+		# mtime=2022-03-05 22:36:48.176789949 +0100 (1646516208),
+		# ctime=2022-03-05 22:36:48.176789949 +0100 (1646516208)>
+
+		# On LInux:
+		#<File::Stat                                
+		#  dev=0xfe01,                                
+		#  ino=6293055,                               
+		#  mode=0100644 (file rw-r--r--),             
+		#  nlink=1,                                   
+		#  uid=164825 (ricc),                         
+		#  gid=89939 (primarygroup),                  
+		#  rdev=0x0 (0, 0),                           
+		#  size=7,                                    
+		#  blksize=4096,                              
+		#  blocks=8,                                  
+		#  atime=2022-06-27 08:49:38.586706861 +0200 (1656312578),
+		#  mtime=2022-03-23 14:28:45 +0100 (1648042125),
+		#  ctime=2022-05-30 10:12:10.381629305 +0200 (1653898330)>
+		ret[:size] = stats.size
+		ret[:mode] = stats.mode
+		# datetimes
+		ret[:stat_safebirthtime] = getSafeCreationTime(file) # in Mac uses birthtime,but on Linux wont work
+#			defined?(stats.birthtime) ? # rescue stats.mtime # eg,  2022-03-05 21:47:51 +0100 on Mac (not implemented on my Linuix)#
+#			stats.birthtime : # works on Mac 
+#			stats.ctime
+		ret[:stat_mtime] = stats.mtime # eg,  2022-03-05 21:47:51 +0100 Returns the modification time of stat.
+		ret[:stat_ctime] = stats.ctime # eg,  2022-03-05 21:47:51 +0100 Returns the change time for stat (that is, the time directory information about the file was changed, not the file itself). Note that on Windows (NTFS), returns creation time (birth time).
+		ret[:stat_atime] = stats.atime # eg,  2022-03-05 21:47:51 +0100 Last Access
+		ret[:stat_gid] = stats.gid     # Group Id
+		ret[:stat_uid] = stats.uid     # UUID
+		ret[:stat_ftype] = stats.ftype     # 
+		
+		ret[:md5] = '______________NONE______________'
+		if stats.ftype != "directory"
+			file_content = File.read(file)
+			ret[:md5] = Digest::MD5.hexdigest(file_content) # rescue 'none                             ' #=> string with hexadecimal digits
+		end
+	rescue Errno::EISDIR => e 
+		#puts "It's a dir, nothing I can do here except skipping the stuff"
+		ret[:md5] = "_________I'm a dir sorry________" 	
+	rescue Exception => e 
+		ret[:error] = e
+	end
+	ret 
+end
+
+$print_stats_and_md5_version = "1.1a_220624_F" # files
+$print_stats_and_md5_counter = 0
+$print_stats_and_md5_for_gcs_version = "1.1alpha_220628_G" #GCS bucket
+
+def stats_and_md5_number_of_files_processed()
+	return $print_stats_and_md5_counter 
+end
+
+def smells_like_gcs?(file)
+	file =~ /gs:\/\//
+end
+
+# You give me gs://bucket123/path/to/dir
+def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts={})
+#	opts_color = opts.fetch :color, true#
+#	opts_verbose = opts.fetch :verbose, false 
+#	opts_ping_frequency = opts.fetch :ping_frequency, 50
+	opts_max_files = opts.fetch :max_files, nil # BigDecimal('Infinity')
+	opts_autowrite = opts.fetch :autowrite, false
+
+	raise "Wrong GCS Path: #{mybucket_with_subdir}" unless smells_like_gcs?(mybucket_with_subdir)
+	#puts "0. mybucket_with_subdir: #{mybucket_with_subdir}" # gs://mybucket123/path/to/dir
+	path_split = mybucket_with_subdir.split('/')
+	mybucket = path_split[0,3].join('/')      # gs://mybucket123/
+	gcs_subpath = path_split[3,42].join('/')  # path/to/dir
+	cleanedup_bucket = path_split[2]          # mybucket123
+	#puts "1. mybucket: #{mybucket}"
+	#puts "2. gcs_subpath: #{gcs_subpath}"
+	#puts "3. cleanedup_bucket: #{cleanedup_bucket}"
+	
+	puts("[print_stats_and_md5_for_gcs] version=#{$print_stats_and_md5_for_gcs_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on='#{Time.now}' bucket=#{mybucket} subpath=#{gcs_subpath}")
+	begin 
+		require "google/cloud/storage"
+		project_id = `gcloud config get project 2>/dev/null`
+		deb "project_id: #{project_id}"
+		storage = Google::Cloud::Storage.new(project_id: project_id)
+		bucket = storage.bucket(cleanedup_bucket)
+		# grabs them ALL if nil, optherwise its an integer.
+		files = opts_max_files.nil? ? 
+			bucket.files :                          # nil -> all of them (I've tried sth more idiomatic like -1 or nil or Infinite. they all failed.)
+			bucket.files.first(opts_max_files)      # not nil -> first(N)
+		#puts(files)
+		deb("All Sorted Methods: #{files.first.methods.sort}")
+		files.each do |gcs_file|
+			md5 = Base64.decode64(gcs_file.md5).unpack('H*')[0]  # md5 in base64 -> then de-binarizied: Base64.urlsafe_decode64(md5).unpack('H*')[0]
+			size = gcs_file.size # crc rescue :boh
+			time_created = gcs_file.created_at
+			mode = 'XXXXXX' # for compatbility with files
+			file_type = gcs_file.name.to_s =~ /\/$/ ? 'd' : 'f' # if ends with a slash its a DIR :P
+			#puts("[OLD_GCS_v11] #{md5} #{size}\t#{time_created} [#{gcs_file.content_type}]\t#{gcs_file.name}")
+			print_colored_polymoprhic('gcs_v1.2', md5, mode, file_type, size, gcs_file.content_type, time_created, gcs_file.name, opts)
+			deb gcs_file.to_yaml
+		end
+		#puts("Hash Methods: #{files.first.methods.select{|x| x.match /hash/}}")
+	rescue LoadError # require Exception 
+		puts 'Error with library #{$!}, maybe type: gem install google-cloud-storage'
+	rescue Exception # generic Exception 
+		puts "print_stats_and_md5_for_gcs(): Generic Error: #{$!}"
+		exit 1
+	end
+
+	autowrite_to_dir_or_gcs(:gcs, mybucket_with_subdir) if opts_autowrite
+end 
+
+# Generic polymorpghic coloring, can be GCS. Would have been nice to pass a Dict so easier to extend, but Ruby doesnt support doesblt Dict with magic args. Plus might be a GOOD
+# thing its hard to change since it breaks Storazzo.
+# TODO(ricc): make sure the dates are the same. Currently:
+# GCS Date:  2021-12-20T12:26:13+00:00  DateTime
+# File Date: 2022-05-30 11:55:42 +0200  Time
+def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_type, creation_time, filename, opts={})
+	opts_color = opts.fetch :color, true
+	opts_verbose = opts.fetch :verbose, false 
+	opts_ping_frequency = opts.fetch :ping_frequency, 50
+	opts_autowrite = opts.fetch :autowrite, false
+
+	# Increment counter across all!
+	$print_stats_and_md5_counter += 1 # global counter! I should increment at the END of fucntion.. but you never know if i exit wrongly so i do at beginning. so within this function the real #files is N-1 (i.e., if N is 6, we're processing file %5)
+	# in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
+	#$stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
+	#puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
+	$stderr.puts "-- print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if (($print_stats_and_md5_counter) % opts_ping_frequency == 1 )
+
+	maybecolored_md5 = opts_color ? red(md5) : md5
+	maybecolored_filename = opts_color ? azure(filename) : filename
+	enlarged_size = sprintf("%7o", size.to_s ) # or SIZE 
+	maybecolored_size = opts_color ? yellow(enlarged_size) : enlarged_size
+	maybecolored_file_type = file_type
+	colored_content_type = opts_color ? white(content_type) : content_type
+	if opts_color 
+		maybecolored_file_type = case file_type
+		when 'f' then green(file_type)
+		when 'd' then blue(file_type)
+		when 's' then azure(file_type)
+		else red(file_type)
+		end
+	end	
+	standardized_creation_time = creation_time.is_a?(DateTime) ? 
+		creation_time : 
+		DateTime.parse(creation_time.to_s) # if you prefer to use Time since its already supported by Storazzo (but its ugly since it has SPACES! so hard to parse) use tt = Time.parse(d.to_s)
+		# as per https://stackoverflow.com/questions/279769/convert-to-from-datetime-and-time-in-ruby
+	#puts "#{maybecolored_md5} #{mode} #{maybecolored_file_type} #{maybecolored_size}\t#{creation_time}(DEB #{creation_time.class})(DEB2 #{DateTime.parse(creation_time.to_s)}) [#{colored_content_type}] #{maybecolored_filename}"
+	# this mightr be colored
+	str = "[#{entity_type}] #{maybecolored_md5} #{mode} #{maybecolored_file_type} #{standardized_creation_time} #{maybecolored_size} [#{colored_content_type}] #{maybecolored_filename}\n"
+	# this will NEVER be colored. WARNING, now you need to maintain TWO of these beasts :/
+	non_colored_str = "[#{entity_type}] #{md5} #{mode} #{file_type} #{standardized_creation_time} #{enlarged_size} [#{content_type}] #{filename}\n"
+	if(opts_autowrite)
+		# need to guarantee this is NOT colored. The easiest way to do it is TWICVE as expensive but... whatevs.
+		# TODO(ricc): fire me for this lazimness!
+		$autowrite_file += (non_colored_str)
+	end
+	print(str)
+	return str
+end
+
+def print_stats_and_md5(file, opts={})
+	return print_stats_and_md5_for_gcs(file, opts) if smells_like_gcs?(file)
+
+	opts_color = opts.fetch :color, true
+	opts_verbose = opts.fetch :verbose, false 
+	opts_ping_frequency = opts.fetch :ping_frequency, 50
+
+	
+	# $print_stats_and_md5_counter += 1 # global counter! I should increment at the END of fucntion.. but you never know if i exit wrongly so i do at beginning. so within this function the real #files is N-1 (i.e., if N is 6, we're processing file %5)
+
+	# # in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
+	# #$stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
+	# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
+	# $stderr.puts "-- print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if (($print_stats_and_md5_counter) % opts_ping_frequency == 1 )
+
+	#puts "print_stats_and_md5: #{file}" if opts_verbose
+	stats = compute_stats_and_md5 file 
+	#maybecolored_md5 = opts_color ? red(stats[:md5]) : stats[:md5]
+	#maybecolored_filename = opts_color ? azure(stats[:name]) : stats[:name]
+	#maybecolored_size = opts_color ? white(stats[:size]) : stats[:size]
+	mode  = sprintf("%06o", stats[:mode] ) rescue :ERROR # .to_s.right(4)        #=> "100644"
+	file_type = stats[:stat_ftype][0] rescue '?'
+	file_type = 's' if File.symlink?(file)
+	content_type = `file --mime-type -b '#{file}' 2>/dev/null`.chomp # .split(': ',2)[1]
+
+	#colored_string = "[COL] #{red stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{white stats[:name]}"
+	#boring_string =  "[B/W] #{    stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{      stats[:name]}"
+	#puts(opts_color ? colored_string : boring_string)
+	#puts "[OLDv1.1deprecated] #{maybecolored_md5} #{mode} #{file_type} #{maybecolored_size} #{stats[:stat_safebirthtime]} #{maybecolored_filename}"
+	print_colored_polymoprhic('file_v1.2', stats[:md5], mode, file_type,  stats[:size], content_type, stats[:stat_safebirthtime], stats[:name], opts)
+end
+
+def autowrite_to_dir_or_gcs(fs_type, path, opts={})
+    autowrite_version = "1.1_28jun22"
+	file_to_save = "stats-with-md5.log"
+	path_to_save = path + "/" + file_to_save
+	puts(red "TODO(ricc): write results [size=#{$autowrite_file.size}] in this Directory: #{path_to_save}") if fs_type == :dir
+	puts(red "TODO(ricc): write results [size=#{$autowrite_file.size}] in this GCS Bucket or Dir: #{path_to_save}") if fs_type == :gcs
+
+	##########################################################################
+	# do the thing: creates file locally and moves to GCS or Dir.
+	##########################################################################
+	$autowrite_file += "# [autowrite_to_dir_or_gcs v#{autowrite_version}]: about to write it now. #{Time.now}"
+	tmpfile = Tempfile.new(path_to_save)
+	tmpfile.write($autowrite_file)
+	cmd = :no_cmd_yet
+	
+	if fs_type == :gcs
+		cmd = "gsutil mv '#{tmpfile.path}' #{path}/#{file_to_save}"
+	end
+	if fs_type == :dir
+		cmd = "mv '#{tmpfile.path}' #{path}/#{file_to_save}"
+	end
+
+	puts("TMP File is this big: #{azure tmpfile.size}")
+	puts("About to execute this: #{white cmd}")
+	ret = `#{cmd}`
+	if $?.exitstatus != 0
+		puts "Error to execute: #{red cmd}. Exit: #{red $?.exitstatus}"
+		exit 53
+	end
+	puts "Command returned: '#{$?.exitstatus}'" #  - #{$?.class}"
+	# wow: fork { exec("ls") }
+
+	# removes the tmpfile :)
+	tmpfile.close 
+	tmpfile.unlink
+
+	#####################################
+	# finished, lets now clean up the file...
+	#####################################
+	reset_autowrite_file()
+	return :ok
+end
+
+def print_stats_and_md5_for_directory(directory_to_explore_recursively, opts={})
+	puts "# [print_stats_and_md5_for_directory] DIR to explore (make sure you glob also): #{directory_to_explore_recursively }"
+	puts "# DEB Options: #{opts}"
+
+	opts_autowrite = opts.fetch :autowrite, false
+
+	Dir.glob("#{(directory_to_explore_recursively)}/**/*") do |globbed_filename|
+		# Do work on files & directories ending in .rb
+		#puts "[deb] #{globbed_filename}" 
+		print_stats_and_md5(globbed_filename, :color => $opts[:color], :verbose => $opts[:verbose], :autowrite => $opts[:autowrite])
+	end
+
+	autowrite_to_dir_or_gcs(:dir, directory_to_explore_recursively) if opts_autowrite
+end 
+
+def real_program
+  t0 = Time.now
+  deb "+ Options are:  #{gray $opts}"
+  deb "+ Depured args: #{azure ARGV}"
+  # Your code goes here...
+  #puts "Description: '''#{white $myconf[:description] }'''"
+  #puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version}  host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ARGV.size > 0
+
+  common_opts = {
+	:max_files => $opts[:max_files], 
+	:color => $opts[:color], 
+	:verbose => $opts[:verbose],
+	:autowrite => $opts[:autowrite],
+  }
+
+  if ARGV.size == 1
+	directory_to_explore_recursively = ARGV[0]
+    if smells_like_gcs?(directory_to_explore_recursively)
+		print_stats_and_md5_for_gcs(directory_to_explore_recursively, common_opts)
+	else # normal file..
+		print_stats_and_md5_for_directory(directory_to_explore_recursively, common_opts)
+	end
+  elsif ARGV.size > 1
+	deb "2. I expect a lot of single files or directories:"
+    for arg in ARGV
+		if File.directory?(arg)
+			print_stats_and_md5_for_directory(arg, common_opts )
+		else
+	   		print_stats_and_md5(arg, common_opts)
+		end
+    end
+  else
+	puts "No args given. Exiting"
+	exit 41
+  end
+  tf = Time.now
+  puts "# [#{File.basename $0}] Time taken for processing #{stats_and_md5_number_of_files_processed} files: #{tf-t0}"
+end
+
+def main(filename)
+  deb "I'm called by #{white filename}"
+  deb "To remove this shit, just set $DEBUG=false :)"
+  init        # Enable this to have command line parsing capabilities!
+  real_program
+end
+
+main(__FILE__)

data/lib/storazzo/media/gcs_bucket.rb

@@ -3,10 +3,12 @@ module Storazzo::Media
     class Storazzo::RicDisk::GcsBucket
 
     
-        def self.list_all
+        def self.list_all(config=nil)
             # get lisrts from Config singletone
             #puts " self.list_all: loading config "
-            config = Storazzo::RicDiskConfig.instance # # ).get_config
+            config ||= Storazzo::RicDiskConfig.instance # # ).get_config
+            
+            config.load # in case I need to load it for the first time
             #puts config['Config']['AdditionalMountDirs']
             #puts "TODO see config: #{config}"
             #[42, 43]

data/lib/storazzo/ric_disk.rb

@@ -21,7 +21,7 @@ module Storazzo
     RicdiskVersion = '2.1'
     RicdiskHistory = [
       '2022-07-29 2.1 Added timestamp',
-      '2022-07-28 2.0 Added tags, siz, unique_hash, cmputation_hostname, wr, ...',
+      '2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...',
   ]
     DefaultGemfileTestDiskFolder = Storazzo.root + "/var/test/disks/" # was: @@default_gemfile_test_disks_folder
     # Immutable
@@ -173,7 +173,7 @@ module Storazzo
         deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
         return @ricdisk_file
       end
-      warn "RICC_WARNING This requires cmputation I wanna do it almost once"
+      deb "[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once"
       ConfigFiles.each do |papable_config_filename|
         #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
         #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
@@ -222,13 +222,13 @@ module Storazzo
     end
 
     # maybe move to a RiccFile class? Maybe even INHERIT from FILE?
-    def self.obsolescence_seconds file_path
+    def obsolescence_seconds(file_path)
       creation_time = File.stat(file_path).ctime
       deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{(Time.now - creation_time)} seconds ago")
       (Time.now - creation_time).to_i
     end
     # maybe move to a RiccFile class? Maybe even INHERIT from FILE?
-    def self.obsolescence_days(file_path)
+    def obsolescence_days(file_path)
       return obsolescence_seconds(file_path) / 86400
     end
 
@@ -296,8 +296,32 @@ module Storazzo
     end
 
     def compute_stats_files(opts={})
-      puts azure("TODO implement natively. Now I'm being lazy")
-      Storazzo::RicDisk.calculate_stats_files(path, opts)
+      puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy")
+      #Storazzo::RicDisk.calculate_stats_files(path, opts)
+      opts_upload_to_gcs = opts.fetch :upload_to_gcs, true
+      dir = path
+      
+      full_file_path = "#{dir}/#{$stats_file}"
+      #return "This refactor is for another day"
+  
+      puts("compute_stats_files(#{white dir}): #{white full_file_path}")
+      puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
+      Dir.chdir(dir)
+      if File.exists?(full_file_path) and ($opts[:force] == false)
+        puts "File '#{$stats_file}' exists already." #  - now should see if its too old, like more than 1 week old"
+        # TODO check for file time...
+        print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp }. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
+        if obsolescence_days(full_file_path) > 7 
+          puts("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
+        end
+        upload_to_gcs(full_file_path) if opts_upload_to_gcs
+      else
+        puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]" 
+        command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
+        puts("[#{`pwd`.chomp}] Executing: #{azure command}")
+        ret = backquote_execute(command)
+        puts "Done. #{ret.split("\n").count} files processed."
+      end
     end
   
 
@@ -307,7 +331,7 @@ module Storazzo
       opts_upload_to_gcs = opts.fetch :upload_to_gcs, true
       
       full_file_path = "#{dir}/#{$stats_file}"
-      return "This refacgtor is for another day"
+      return "This refactor is for another day"
   
       puts("calculate_stats_files(#{white dir}): #{white full_file_path}")
       puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
@@ -322,7 +346,7 @@ module Storazzo
         upload_to_gcs(full_file_path) if opts_upload_to_gcs
       else
         puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]" 
-        command = "find . -print0 | xargs -0 stats-with-md5.rb --no-color | tee '#{full_file_path}'"
+        command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
         puts("[#{`pwd`.chomp}] Executing: #{azure command}")
         ret = backquote_execute command
         puts "Done. #{ret.split("\n").count} files processed."
@@ -432,25 +456,27 @@ end  #/Module
 
 
   
-#     def self.backquote_execute(cmd)
-#       # executed a command wrapped by dryrun though
-#       return "DRYRUN backquote_execute(#{cmd})" if $opts[:dryrun]
-#       `#{cmd}`
-#     end
+    def backquote_execute(cmd, opts={})
+      dryrun = opts.fetch :dryrun, false
+      # executed a command wrapped by dryrun though
+      return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
+      `#{cmd}`
+    end
     
-#     def self.upload_to_gcs(file, opts={})
-#       deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
-#       mount_name = file.split('/')[-2]
-#       filename = "#{mount_name}-#{File.basename file}"
-#       hostname = Socket.gethostname[/^[^.]+/]
-#       command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{ hostname }-#{filename}"
-#       deb("Command: #{command}")
-#       ret = backquote_execute(command)
-#       # if $opts[:debug] do
-#       #   puts "+ Current list of files:"
-#       #   ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
-#       #   puts ret
-#       # end
-#       ret
-#     end
+    def upload_to_gcs(file, opts={})
+      deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
+      mount_name = file.split('/')[-2]
+      filename = "#{mount_name}-#{File.basename file}"
+      hostname = Socket.gethostname[/^[^.]+/]
+      command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{ hostname }-#{filename}"
+      deb("Command: #{command}")
+      puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
+      ret = backquote_execute(command, :dryrun => true)
+      # if $opts[:debug] do
+      #   puts "+ Current list of files:"
+      #   ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
+      #   puts ret
+      # end
+      ret
+    end
   

data/lib/storazzo/ric_disk_config.rb

@@ -116,6 +116,18 @@ public
             "POLY_#{self.class}_(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
         end
 
+        def to_verbose_s
+            h = {}
+            h[:description] = "This is a Verbose Hash describing a RicDiskConfig or its child RicDiskSampleConfig to understand why it keeps failing.."
+            h[:to_s] = self.to_s 
+            h[:class] = self.class 
+            h[:file] = __FILE__ 
+            h[:id] = self.object_id
+            h[:get_bucket_paths] = self.get_bucket_paths()
+            h[:get_local_folders] = self.get_local_folders()
+            return h
+        end
+
         def get_config(opts={})
             return load(opts) if @config.nil?
             @config
@@ -160,27 +172,26 @@ public
             #         RicDisk.write_config_yaml_to_disk(dir)
             #         RicDisk.calculate_stats_files(dir) # dir is inutile
             #     } # TODO refactor in option sbrodola_afterwards=true. :)
-            # else
-                raise "Wrong input: #{files_list} " unless files_list.is_a?(Array)
-   
-                puts "iterate_through_file_list_for_disks(): I consider files_list as a list of directories to parse :)" if verbose
-
-                #dirs = RicDisk.find_active_dirs()
-                files_list.each do |dir| 
-                    dir = File.expand_path(dir)
-                    if File.directory?(dir)
-                    #if dirs.include?(dir)
-                        puts "iterate_through_file_list_for_disks() Legit dir: #{green dir}" if verbose
-                        rd = RicDisk.new(dir)
-                        rd.write_config_yaml_to_disk(dir)
-                        #RicDisk.write_config_yaml_to_disk(dir)
-                        #RicDisk.calculate_stats_files (CLASS) => will become OBJECT compute_stats_files
-                        rd.compute_stats_files() # dir is inutile # TODO
-                    else
-                        deb red("Doesnt seem a legit dir to me: #{dir}")
-                    #     deb "Figghiu ri buttana: doesnt exist #{red dir}" 
-                    end
+        # else
+            raise "Wrong input, I need an array here: #{files_list} " unless files_list.is_a?(Array)
+            puts "iterate_through_file_list_for_disks(): I consider files_list as a list of directories to parse :)" if verbose
+
+            #dirs = RicDisk.find_active_dirs()
+            files_list.each do |dir| 
+                dir = File.expand_path(dir)
+                if File.directory?(dir)
+                #if dirs.include?(dir)
+                    puts "iterate_through_file_list_for_disks() Legit dir: #{green dir}" if verbose
+                    rd = RicDisk.new(dir)
+                    puts "RicDisk: #{rd}"
+                    rd.write_config_yaml_to_disk(dir)
+                    #RicDisk.write_config_yaml_to_disk(dir)
+                    #RicDisk.calculate_stats_files (CLASS) => will become OBJECT compute_stats_files
+                    rd.compute_stats_files() # dir is inutile # TODO
+                else
+                    raise("Doesnt seem a dir to me, quitting: #{dir}")
                 end
+            end
             #end
         end #/iterate_through_file_list_for_disks