storazzo 0.5.7 → 0.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (42) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +7 -5
  3. data/Makefile +8 -6
  4. data/Rakefile +15 -13
  5. data/VERSION +1 -1
  6. data/bin/hello-storazzo +1 -0
  7. data/bin/ricdisk-magic +33 -30
  8. data/bin/stats-with-md5 +182 -157
  9. data/bin/storazzo +112 -114
  10. data/bin/storazzo-symlink.rb +1 -0
  11. data/lib/storazzo/colors.rb +77 -36
  12. data/lib/storazzo/common.rb +72 -68
  13. data/lib/storazzo/debug.rb +2 -0
  14. data/lib/storazzo/hashify.rb +7 -5
  15. data/lib/storazzo/main.rb +59 -49
  16. data/lib/storazzo/media/abstract_ric_disk.rb +188 -179
  17. data/lib/storazzo/media/gcs_bucket.rb +76 -58
  18. data/lib/storazzo/media/local_folder.rb +56 -42
  19. data/lib/storazzo/media/mount_point.rb +20 -12
  20. data/lib/storazzo/ric_disk.rb +386 -383
  21. data/lib/storazzo/ric_disk_config.rb +227 -209
  22. data/lib/storazzo/ric_disk_sample_config.rb +26 -24
  23. data/lib/storazzo/ric_disk_statsfile.rb +19 -17
  24. data/lib/storazzo/ric_disk_ugly.rb +1 -0
  25. data/lib/storazzo/version.rb +3 -3
  26. data/lib/storazzo.rb +6 -7
  27. data/storazzo.gemspec +24 -16
  28. data/test/benchmark/for_future_use.rb +4 -2
  29. data/test/benchmark/test_hashing_functions-speed.rb +17 -0
  30. data/test/bin/new-idea.rb +17 -0
  31. data/test/bin/storazzo.rb +22 -25
  32. data/test/media/test_abstract_ric_disk.rb +5 -3
  33. data/test/media/test_gcs_bucket.rb +24 -23
  34. data/test/media/test_local_folder.rb +24 -23
  35. data/test/media/test_mount_point.rb +6 -5
  36. data/test/test_ric_disk.rb +6 -4
  37. data/test/test_ric_disk_config.rb +12 -11
  38. data/test/test_ric_disk_stats_file.rb +5 -3
  39. data/test/test_storazzo.rb +6 -4
  40. data/var/test/disks/disk02-full/Rakefile +7 -5
  41. data/var/test/disks/ricdisk_stats_v11.rds +11 -0
  42. metadata +13 -21
data/bin/stats-with-md5 CHANGED
@@ -8,7 +8,7 @@ require 'date' # for DateTime
8
8
  require 'tempfile'
9
9
 
10
10
  if RUBY_VERSION.split('.')[0] == 1
11
- puts "Refusing to launch a script form Ruby 1. Sorry Ric, its 2020 damn it!"
11
+ puts 'Refusing to launch a script form Ruby 1. Sorry Ric, its 2020 damn it!'
12
12
  exit 2020
13
13
  end
14
14
 
@@ -16,62 +16,89 @@ $PROG_VER = '0.5' # on 0.4 it was copied from palladius/sakura. Now should be ma
16
16
  $DEBUG = false
17
17
  $reset_autowrite_file_reincarnation = 0
18
18
  DEFAULT_MAX_FILES = 'Infinite'
19
+ $default_ping_frequency = 50
20
+
21
+ #
22
+ # ############################################################
23
+ # @author: Riccardo Carlesso
24
+ # @email: riccardo.carlesso@gmail.com
25
+ # @maturity: development
26
+ # @language: Ruby
27
+ # @tags: development, rcarlesso, test
28
+ # @works_on: Linux (little tested since v0.3), Mac (100% developed here)
29
+ # ############################################################
30
+ #
31
+
32
+ def deb(s)
33
+ puts "#DEB #{s}" if $DEBUG
34
+ end
19
35
 
20
- =begin
36
+ # colors 16
37
+ def gray(s)
38
+ "\033[1;30m#{s}\033[0m"
39
+ end
21
40
 
22
- ############################################################
23
- @author: Riccardo Carlesso
24
- @email: riccardo.carlesso@gmail.com
25
- @maturity: development
26
- @language: Ruby
27
- @tags: development, rcarlesso, test
28
- @works_on: Linux (little tested since v0.3), Mac (100% developed here)
29
- ############################################################
41
+ def green(s)
42
+ "\033[1;32m#{s}\033[0m"
43
+ end
30
44
 
31
- =end
45
+ def red(s)
46
+ "\033[1;31m#{s}\033[0m"
47
+ end
32
48
 
33
- def deb(s); puts "#DEB #{s}" if $DEBUG; end
34
- # colors 16
35
- def gray(s) "\033[1;30m#{s}\033[0m"; end
36
- def green(s) "\033[1;32m#{s}\033[0m"; end
37
- def red(s) "\033[1;31m#{s}\033[0m"; end
38
- def yellow(s) "\033[1;33m#{s}\033[0m"; end
39
- def blue(s) "\033[1;34m#{s}\033[0m"; end
40
- def purple(s) "\033[1;35m#{s}\033[0m"; end
41
- def azure(s) "\033[1;36m#{s}\033[0m"; end
42
- def white(s) "\033[1;37m#{s}\033[0m"; end
49
+ def yellow(s)
50
+ "\033[1;33m#{s}\033[0m"
51
+ end
52
+
53
+ def blue(s)
54
+ "\033[1;34m#{s}\033[0m"
55
+ end
56
+
57
+ def purple(s)
58
+ "\033[1;35m#{s}\033[0m"
59
+ end
60
+
61
+ def azure(s)
62
+ "\033[1;36m#{s}\033[0m"
63
+ end
64
+
65
+ def white(s)
66
+ "\033[1;37m#{s}\033[0m"
67
+ end
43
68
 
44
69
  # colors 64k
45
- def orange(s) "\033[38;5;208m#{s}\033[0m"; end
70
+ def orange(s)
71
+ "\033[38;5;208m#{s}\033[0m"
72
+ end
46
73
 
47
74
  # Program constants, automagically picked up by RicLib
48
75
  # More configuration could be written in:
49
76
  # $GIC/etc/ricsvn/<FILENAME>.yml
50
77
  # That would go into the variable '$prog_conf_d'
51
78
  $myconf = {
52
- :app_name => "AppName should be sth like #{$0}",
53
- :description => "
79
+ app_name: "AppName should be sth like #{$0}",
80
+ description: "
54
81
  This program gets stats from Ruby File.Stats library
55
82
  plus computes MD5 or CRC32 of the file.
56
83
  And bakes it in a way that can be easily parsed.
57
- ".strip.gsub(/^\s+/, "").gsub(/\s+$/, ""),
84
+ ".strip.gsub(/^\s+/, '').gsub(/\s+$/, '')
58
85
  }
59
86
 
60
87
  def usage(comment = nil)
61
88
  puts white($optparse.banner)
62
89
  puts($optparse.summarize)
63
- puts("Description: " + gray($myconf[:description]))
90
+ puts('Description: ' + gray($myconf[:description]))
64
91
  puts red(comment) if comment
65
92
  exit 13
66
93
  end
67
94
 
68
- def reset_autowrite_file()
95
+ def reset_autowrite_file
69
96
  $autowrite_file = "# #{File.basename $0} Autowrite v1.0 #{Time.now} reincarnation=#{$reset_autowrite_file_reincarnation}\n"
70
97
  $reset_autowrite_file_reincarnation += 1 # ($reset_autowrite_file_reincarnation + 1) # rescue 1
71
98
  end
72
99
 
73
100
  # include it in main if you want a custome one
74
- def init() # see lib_autoinit in lib/util.rb
101
+ def init # see lib_autoinit in lib/util.rb
75
102
  $opts = {}
76
103
  # setting defaults
77
104
  $opts[:verbose] = false
@@ -81,19 +108,22 @@ def init() # see lib_autoinit in lib/util.rb
81
108
  $opts[:color] = true
82
109
  $opts[:max_files] = nil # infinite
83
110
 
84
- reset_autowrite_file()
111
+ reset_autowrite_file
85
112
 
86
113
  $optparse = OptionParser.new do |opts|
87
114
  opts.banner = "#{$0} v.#{$PROG_VER}\n Usage: #{File.basename $0} [options] file1 file2 ..."
88
- opts.on('-a', '--autowrite', 'automatically writes results to Root Folder (DFLT=false)') {
115
+ opts.on('-a', '--autowrite', 'automatically writes results to Root Folder (DFLT=false)') do
89
116
  $opts[:autowrite] = true
90
- }
117
+ end
91
118
  opts.on('-c', '--no-color', 'disables color (DFLT=false, that is color enabled)') { $opts[:color] = false }
92
- opts.on('-d', '--debug', 'enables debug (DFLT=false)') { $opts[:debug] = true; $DEBUG = true }
93
- opts.on('-h', '--help', 'Display this screen') { usage }
94
- opts.on('-m', '--max-files NUMBER', "sets max files per Dir to X (DFLT=#{DEFAULT_MAX_FILES})") { |nfiles|
119
+ opts.on('-d', '--debug', 'enables debug (DFLT=false)') do
120
+ $opts[:debug] = true
121
+ $DEBUG = true
122
+ end
123
+ opts.on('-h', '--help', 'Display this screen') { usage }
124
+ opts.on('-m', '--max-files NUMBER', "sets max files per Dir to X (DFLT=#{DEFAULT_MAX_FILES})") do |nfiles|
95
125
  $opts[:max_files] = nfiles.to_i
96
- }
126
+ end
97
127
  opts.on('-n', '--dryrun', "Don't really execute code") { $opts[:dryrun] = true }
98
128
  opts.on('-l', '--logfile FILE', 'Write log to FILE') { |file| $opts[:logfile] = file }
99
129
  opts.on('-v', '--verbose', 'Output more information') { $opts[:verbose] = true }
@@ -109,60 +139,61 @@ def compute_stats_and_md5(file)
109
139
  ret = {}
110
140
  ret[:name] = file
111
141
 
112
- begin
113
- stats = File.stat(file)
114
- ret[:stats_object] = stats # TODO deprecate
115
- #deb("Stats methods: #{stats.methods.sort.join(', ')}")
116
- #deb(stats.ctime)
117
- #puts(stats.birthtime rescue (stats.ctime))
118
- # On Mac/Linux: see test/dumps/***.yaml
119
- ret[:size] = stats.size
120
- ret[:mode] = stats.mode
121
- # datetimes
122
- ret[:stat_safebirthtime] = getSafeCreationTime(file) # in Mac uses birthtime,but on Linux wont work
123
- # defined?(stats.birthtime) ? # rescue stats.mtime # eg, 2022-03-05 21:47:51 +0100 on Mac (not implemented on my Linuix)#
124
- # stats.birthtime : # works on Mac
125
- # stats.ctime
126
- ret[:stat_mtime] = stats.mtime # eg, 2022-03-05 21:47:51 +0100 Returns the modification time of stat.
127
- ret[:stat_ctime] = stats.ctime # eg, 2022-03-05 21:47:51 +0100 Returns the change time for stat (that is, the time directory information about the file was changed, not the file itself). Note that on Windows (NTFS), returns creation time (birth time).
128
- ret[:stat_atime] = stats.atime # eg, 2022-03-05 21:47:51 +0100 Last Access
129
- ret[:stat_gid] = stats.gid # Group Id
130
- ret[:stat_uid] = stats.uid # UUID
131
- ret[:stat_ftype] = stats.ftype #
132
-
133
- ret[:md5] = '______________NONE______________'
134
- if stats.ftype != "directory"
135
- file_content = File.read(file)
136
- ret[:md5] = Digest::MD5.hexdigest(file_content) # rescue 'none ' #=> string with hexadecimal digits
137
- end
138
- rescue Errno::EISDIR => e
139
- #puts "It's a dir, nothing I can do here except skipping the stuff"
140
- ret[:md5] = "_________I'm a dir sorry________"
141
- rescue Exception => e
142
- ret[:error] = e
143
- end
144
- ret
142
+ begin
143
+ stats = File.stat(file)
144
+ ret[:stats_object] = stats # TODO: deprecate
145
+ # deb("Stats methods: #{stats.methods.sort.join(', ')}")
146
+ # deb(stats.ctime)
147
+ # puts(stats.birthtime rescue (stats.ctime))
148
+ # On Mac/Linux: see test/dumps/***.yaml
149
+ ret[:size] = stats.size
150
+ ret[:mode] = stats.mode
151
+ # datetimes
152
+ ret[:stat_safebirthtime] = getSafeCreationTime(file) # in Mac uses birthtime,but on Linux wont work
153
+ # defined?(stats.birthtime) ? # rescue stats.mtime # eg, 2022-03-05 21:47:51 +0100 on Mac (not implemented on my Linuix)#
154
+ # stats.birthtime : # works on Mac
155
+ # stats.ctime
156
+ ret[:stat_mtime] = stats.mtime # eg, 2022-03-05 21:47:51 +0100 Returns the modification time of stat.
157
+ ret[:stat_ctime] = stats.ctime # eg, 2022-03-05 21:47:51 +0100 Returns the change time for stat (that is, the time directory information about the file was changed, not the file itself). Note that on Windows (NTFS), returns creation time (birth time).
158
+ ret[:stat_atime] = stats.atime # eg, 2022-03-05 21:47:51 +0100 Last Access
159
+ ret[:stat_gid] = stats.gid # Group Id
160
+ ret[:stat_uid] = stats.uid # UUID
161
+ ret[:stat_ftype] = stats.ftype #
162
+
163
+ ret[:md5] = '______________NONE______________'
164
+ if stats.ftype != 'directory'
165
+ file_content = File.read(file)
166
+ ret[:md5] = Digest::MD5.hexdigest(file_content) # rescue 'none ' #=> string with hexadecimal digits
167
+ end
168
+ rescue Errno::EISDIR => e
169
+ # puts "It's a dir, nothing I can do here except skipping the stuff"
170
+ ret[:md5] = "_________I'm a dir sorry________"
171
+ rescue Exception => e
172
+ ret[:error] = e
173
+ end
174
+ ret
145
175
  end
146
176
 
147
- $print_stats_and_md5_version = "1.1b_220805_F" # files
177
+ $print_stats_and_md5_version = '1.1b_220805_F' # files
148
178
  $print_stats_and_md5_counter = 0
149
- $print_stats_and_md5_for_gcs_version = "1.1alpha_220628_G" # GCS bucket
179
+ $print_stats_and_md5_for_gcs_version = '1.1alpha_220628_G' # GCS bucket
150
180
 
151
- def stats_and_md5_number_of_files_processed()
152
- return $print_stats_and_md5_counter
181
+ def stats_and_md5_number_of_files_processed
182
+ $print_stats_and_md5_counter
153
183
  end
154
184
 
155
185
  def smells_like_gcs?(file)
156
- file =~ /gs:\/\//
186
+ file =~ %r{gs://}
157
187
  end
158
188
 
159
189
  # You give me gs://bucket123/path/to/dir
160
190
  def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts = {})
161
191
  # opts_color = opts.fetch :color, true#
162
192
  # opts_verbose = opts.fetch :verbose, false
163
- # opts_ping_frequency = opts.fetch :ping_frequency, 50
193
+ # opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
164
194
  opts_max_files = opts.fetch :max_files, nil # BigDecimal('Infinity')
165
195
  opts_autowrite = opts.fetch :autowrite, false
196
+ opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
166
197
 
167
198
  raise "Wrong GCS Path: #{mybucket_with_subdir}" unless smells_like_gcs?(mybucket_with_subdir)
168
199
 
@@ -177,23 +208,25 @@ def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts = {})
177
208
 
178
209
  puts("[print_stats_and_md5_for_gcs] version=#{$print_stats_and_md5_for_gcs_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on='#{Time.now}' bucket=#{mybucket} subpath=#{gcs_subpath}")
179
210
  begin
180
- require "google/cloud/storage"
211
+ require 'google/cloud/storage'
181
212
  project_id = `gcloud config get project 2>/dev/null`
182
213
  deb "project_id: #{project_id}"
183
214
  storage = Google::Cloud::Storage.new(project_id: project_id)
184
215
  bucket = storage.bucket(cleanedup_bucket)
185
216
  # grabs them ALL if nil, optherwise its an integer.
186
- files = opts_max_files.nil? ?
187
- bucket.files : # nil -> all of them (I've tried sth more idiomatic like -1 or nil or Infinite. they all failed.)
188
- bucket.files.first(opts_max_files) # not nil -> first(N)
217
+ files = if opts_max_files.nil?
218
+ bucket.files
219
+ else
220
+ bucket.files.first(opts_max_files)
221
+ end # not nil -> first(N)
189
222
  # puts(files)
190
223
  deb("All Sorted Methods: #{files.first.methods.sort}")
191
224
  files.each do |gcs_file|
192
- md5 = Base64.decode64(gcs_file.md5).unpack('H*')[0] # md5 in base64 -> then de-binarizied: Base64.urlsafe_decode64(md5).unpack('H*')[0]
225
+ md5 = Base64.decode64(gcs_file.md5).unpack1('H*') # md5 in base64 -> then de-binarizied: Base64.urlsafe_decode64(md5).unpack('H*')[0]
193
226
  size = gcs_file.size # crc rescue :boh
194
227
  time_created = gcs_file.created_at
195
228
  mode = 'XXXXXX' # for compatbility with files
196
- file_type = gcs_file.name.to_s =~ /\/$/ ? 'd' : 'f' # if ends with a slash its a DIR :P
229
+ file_type = gcs_file.name.to_s =~ %r{/$} ? 'd' : 'f' # if ends with a slash its a DIR :P
197
230
  # puts("[OLD_GCS_v11] #{md5} #{size}\t#{time_created} [#{gcs_file.content_type}]\t#{gcs_file.name}")
198
231
  print_colored_polymoprhic('gcs_v1.2', md5, mode, file_type, size, gcs_file.content_type, time_created,
199
232
  gcs_file.name, opts)
@@ -217,7 +250,7 @@ end
217
250
  def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_type, creation_time, filename, opts = {})
218
251
  opts_color = opts.fetch :color, true
219
252
  opts_verbose = opts.fetch :verbose, false
220
- opts_ping_frequency = opts.fetch :ping_frequency, 50
253
+ opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
221
254
  opts_autowrite = opts.fetch :autowrite, false
222
255
 
223
256
  # Increment counter across all!
@@ -225,11 +258,11 @@ def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_t
225
258
  # in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
226
259
  # $stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
227
260
  # puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
228
- $stderr.puts "-- print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if (($print_stats_and_md5_counter) % opts_ping_frequency == 1)
261
+ warn "-- #{Time.now} print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if $print_stats_and_md5_counter % opts_ping_frequency == 1
229
262
 
230
263
  maybecolored_md5 = opts_color ? red(md5) : md5
231
264
  maybecolored_filename = opts_color ? azure(filename) : filename
232
- enlarged_size = sprintf("%7o", size.to_s) # or SIZE
265
+ enlarged_size = format('%7o', size.to_s) # or SIZE
233
266
  maybecolored_size = opts_color ? yellow(enlarged_size) : enlarged_size
234
267
  maybecolored_file_type = file_type
235
268
  colored_content_type = opts_color ? white(content_type) : content_type
@@ -241,22 +274,24 @@ def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_t
241
274
  else red(file_type)
242
275
  end
243
276
  end
244
- standardized_creation_time = creation_time.is_a?(DateTime) ?
245
- creation_time :
246
- DateTime.parse(creation_time.to_s) # if you prefer to use Time since its already supported by Storazzo (but its ugly since it has SPACES! so hard to parse) use tt = Time.parse(d.to_s)
277
+ standardized_creation_time = if creation_time.is_a?(DateTime)
278
+ creation_time
279
+ else
280
+ DateTime.parse(creation_time.to_s)
281
+ end # if you prefer to use Time since its already supported by Storazzo (but its ugly since it has SPACES! so hard to parse) use tt = Time.parse(d.to_s)
247
282
  # as per https://stackoverflow.com/questions/279769/convert-to-from-datetime-and-time-in-ruby
248
283
  # puts "#{maybecolored_md5} #{mode} #{maybecolored_file_type} #{maybecolored_size}\t#{creation_time}(DEB #{creation_time.class})(DEB2 #{DateTime.parse(creation_time.to_s)}) [#{colored_content_type}] #{maybecolored_filename}"
249
284
  # this mightr be colored
250
285
  str = "[#{entity_type}] #{maybecolored_md5} #{mode} #{maybecolored_file_type} #{standardized_creation_time} #{maybecolored_size} [#{colored_content_type}] #{maybecolored_filename}\n"
251
286
  # this will NEVER be colored. WARNING, now you need to maintain TWO of these beasts :/
252
287
  non_colored_str = "[#{entity_type}] #{md5} #{mode} #{file_type} #{standardized_creation_time} #{enlarged_size} [#{content_type}] #{filename}\n"
253
- if (opts_autowrite)
288
+ if opts_autowrite
254
289
  # need to guarantee this is NOT colored. The easiest way to do it is TWICVE as expensive but... whatevs.
255
290
  # TODO(ricc): fire me for this lazimness!
256
- $autowrite_file += (non_colored_str)
291
+ $autowrite_file += non_colored_str
257
292
  end
258
293
  print(str)
259
- return str
294
+ str
260
295
  end
261
296
 
262
297
  def print_stats_and_md5(file, opts = {})
@@ -264,39 +299,34 @@ def print_stats_and_md5(file, opts = {})
264
299
 
265
300
  opts_color = opts.fetch :color, true
266
301
  opts_verbose = opts.fetch :verbose, false
267
- opts_ping_frequency = opts.fetch :ping_frequency, 50
268
-
269
- # $print_stats_and_md5_counter += 1 # global counter! I should increment at the END of fucntion.. but you never know if i exit wrongly so i do at beginning. so within this function the real #files is N-1 (i.e., if N is 6, we're processing file %5)
270
-
271
- # # in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
272
- # #$stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
273
- # puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
274
- # $stderr.puts "-- print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if (($print_stats_and_md5_counter) % opts_ping_frequency == 1 )
302
+ opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
275
303
 
276
- # puts "print_stats_and_md5: #{file}" if opts_verbose
304
+ puts "print_stats_and_md5: #{file}" if opts_verbose
277
305
  stats = compute_stats_and_md5 file
278
- # maybecolored_md5 = opts_color ? red(stats[:md5]) : stats[:md5]
279
- # maybecolored_filename = opts_color ? azure(stats[:name]) : stats[:name]
280
- # maybecolored_size = opts_color ? white(stats[:size]) : stats[:size]
281
- mode = sprintf("%06o", stats[:mode]) rescue :ERROR # .to_s.right(4) #=> "100644"
282
- file_type = stats[:stat_ftype][0] rescue '?'
306
+
307
+ mode = begin
308
+ format('%06o', stats[:mode])
309
+ rescue StandardError
310
+ :ERROR
311
+ end # .to_s.right(4) #=> "100644"
312
+ file_type = begin
313
+ stats[:stat_ftype][0]
314
+ rescue StandardError
315
+ '?'
316
+ end
283
317
  file_type = 's' if File.symlink?(file)
284
318
  content_type = `file --mime-type -b '#{file}' 2>/dev/null`.chomp # .split(': ',2)[1]
285
319
 
286
- # colored_string = "[COL] #{red stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{white stats[:name]}"
287
- # boring_string = "[B/W] #{ stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{ stats[:name]}"
288
- # puts(opts_color ? colored_string : boring_string)
289
- # puts "[OLDv1.1deprecated] #{maybecolored_md5} #{mode} #{file_type} #{maybecolored_size} #{stats[:stat_safebirthtime]} #{maybecolored_filename}"
290
320
  print_colored_polymoprhic('file_v1.2', stats[:md5], mode, file_type, stats[:size], content_type,
291
321
  stats[:stat_safebirthtime], stats[:name], opts)
292
322
  end
293
323
 
294
- def autowrite_to_dir_or_gcs(fs_type, path, opts = {})
295
- autowrite_version = "1.1_28jun22"
296
- file_to_save = "stats-with-md5.log"
297
- path_to_save = path + "/" + file_to_save
298
- puts(red "TODO(ricc): write results [size=#{$autowrite_file.size}] in this Directory: #{path_to_save}") if fs_type == :dir
299
- puts(red "TODO(ricc): write results [size=#{$autowrite_file.size}] in this GCS Bucket or Dir: #{path_to_save}") if fs_type == :gcs
324
+ def autowrite_to_dir_or_gcs(fs_type, path, _opts = {})
325
+ autowrite_version = '1.1_28jun22'
326
+ file_to_save = 'stats-with-md5.log'
327
+ path_to_save = path + '/' + file_to_save
328
+ puts(red("TODO(ricc): write results [size=#{$autowrite_file.size}] in this Directory: #{path_to_save}")) if fs_type == :dir
329
+ puts(red("TODO(ricc): write results [size=#{$autowrite_file.size}] in this GCS Bucket or Dir: #{path_to_save}")) if fs_type == :gcs
300
330
 
301
331
  ##########################################################################
302
332
  # do the thing: creates file locally and moves to GCS or Dir.
@@ -306,12 +336,8 @@ def autowrite_to_dir_or_gcs(fs_type, path, opts = {})
306
336
  tmpfile.write($autowrite_file)
307
337
  cmd = :no_cmd_yet
308
338
 
309
- if fs_type == :gcs
310
- cmd = "gsutil mv '#{tmpfile.path}' #{path}/#{file_to_save}"
311
- end
312
- if fs_type == :dir
313
- cmd = "mv '#{tmpfile.path}' #{path}/#{file_to_save}"
314
- end
339
+ cmd = "gsutil mv '#{tmpfile.path}' #{path}/#{file_to_save}" if fs_type == :gcs
340
+ cmd = "mv '#{tmpfile.path}' #{path}/#{file_to_save}" if fs_type == :dir
315
341
 
316
342
  puts("TMP File is this big: #{azure tmpfile.size}")
317
343
  puts("About to execute this: #{white cmd}")
@@ -330,28 +356,27 @@ def autowrite_to_dir_or_gcs(fs_type, path, opts = {})
330
356
  #####################################
331
357
  # finished, lets now clean up the file...
332
358
  #####################################
333
- reset_autowrite_file()
334
- return :ok
359
+ reset_autowrite_file
360
+ :ok
335
361
  end
336
362
 
363
+ def print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, opts = {})
364
+ puts "# [print_stats_and_md5_for_directory_from_cli] v#{$print_stats_and_md5_version}] DIR to explore (make sure you glob also): #{directory_to_explore_recursively}"
365
+ opts_autowrite = opts.fetch :autowrite, false
366
+ opts_verbose = opts.fetch(:verbose, $opts[:verbose])
367
+ opts_debug = opts.fetch :debug, $opts[:debug]
368
+ opts_color = opts.fetch :color, $opts[:color]
369
+ if opts_debug
370
+ puts "# DEB Options1 opts (FUNC) BAD: s #{opts}" # probably useless since this is invoked by CLI and has ful fledged options.
371
+ puts "# DEB Options2 $opts (ARGV) GOOD: #{$opts}" # This is what we ewant in this CLI. TODO(ricc): if you move this function to be used as function invert the priority...
372
+ puts("# DEB version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") # if ARGV.size > 0
373
+ end
337
374
 
338
- def print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, opts={})
339
- puts "# [print_stats_and_md5_for_directory_from_cli] v#{$print_stats_and_md5_version}] DIR to explore (make sure you glob also): #{directory_to_explore_recursively }"
340
- opts_autowrite = opts.fetch :autowrite, false
341
- opts_verbose = opts.fetch( :verbose, $opts[:verbose])
342
- opts_debug = opts.fetch :debug, $opts[:debug]
343
- opts_color = opts.fetch :color, $opts[:color]
344
- if opts_debug
345
- puts "# DEB Options1 opts (FUNC) BAD: s #{opts}" # probably useless since this is invoked by CLI and has ful fledged options.
346
- puts "# DEB Options2 $opts (ARGV) GOOD: #{$opts}" # This is what we ewant in this CLI. TODO(ricc): if you move this function to be used as function invert the priority...
347
- puts("# DEB version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") # if ARGV.size > 0
348
- end
349
-
350
- Dir.glob("#{(directory_to_explore_recursively)}/**/*") do |globbed_filename|
351
- # Do work on files & directories ending in .rb
352
- #puts "[deb] #{globbed_filename}"
353
- print_stats_and_md5(globbed_filename, :color => opts_color, :verbose => opts_verbose, :autowrite => opts_autowrite)
354
- end
375
+ Dir.glob("#{directory_to_explore_recursively}/**/*") do |globbed_filename|
376
+ # Do work on files & directories ending in .rb
377
+ # puts "[deb] #{globbed_filename}"
378
+ print_stats_and_md5(globbed_filename, color: opts_color, verbose: opts_verbose, autowrite: opts_autowrite)
379
+ end
355
380
 
356
381
  autowrite_to_dir_or_gcs(:dir, directory_to_explore_recursively) if opts_autowrite
357
382
  end
@@ -365,41 +390,41 @@ def real_program
365
390
  # puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ARGV.size > 0
366
391
 
367
392
  common_opts = {
368
- :max_files => $opts[:max_files],
369
- :color => $opts[:color],
370
- :verbose => $opts[:verbose],
371
- :autowrite => $opts[:autowrite],
393
+ max_files: $opts[:max_files],
394
+ color: $opts[:color],
395
+ verbose: $opts[:verbose],
396
+ autowrite: $opts[:autowrite]
372
397
  }
373
398
 
374
399
  if ARGV.size == 1
375
400
  directory_to_explore_recursively = ARGV[0]
376
401
  if smells_like_gcs?(directory_to_explore_recursively)
377
- print_stats_and_md5_for_gcs(directory_to_explore_recursively, common_opts)
378
- else # normal file..
379
- print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, common_opts)
380
- end
402
+ print_stats_and_md5_for_gcs(directory_to_explore_recursively, common_opts)
403
+ else # normal file..
404
+ print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, common_opts)
405
+ end
381
406
  elsif ARGV.size > 1
382
- deb "2. I expect a lot of single files or directories:"
407
+ deb '2. I expect a lot of single files or directories:'
383
408
  for arg in ARGV
384
- if File.directory?(arg)
385
- print_stats_and_md5_for_directory_from_cli(arg, common_opts )
386
- else
387
- print_stats_and_md5(arg, common_opts)
388
- end
409
+ if File.directory?(arg)
410
+ print_stats_and_md5_for_directory_from_cli(arg, common_opts)
411
+ else
412
+ print_stats_and_md5(arg, common_opts)
413
+ end
389
414
  end
390
415
  else
391
- puts "No args given. Exiting"
416
+ puts 'No args given. Exiting'
392
417
  exit 41
393
418
  end
394
419
  tf = Time.now
395
- #rounding to 3 decimals
396
- approx_delta = (tf-t0).round(3)
397
- puts "# [#{File.basename $0}] Time taken for processing #{stats_and_md5_number_of_files_processed} files: #{approx_delta} seconds"
420
+ # rounding to 3 decimals
421
+ approx_delta = (tf - t0).round(3)
422
+ puts "# [#{File.basename $0}] Time taken for processing #{stats_and_md5_number_of_files_processed} files: #{approx_delta} seconds"
398
423
  end
399
424
 
400
425
  def main(filename)
401
426
  deb "I'm called by #{white filename}"
402
- deb "To remove this shit, just set $DEBUG=false :)"
427
+ deb 'To remove this shit, just set $DEBUG=false :)'
403
428
  init # Enable this to have command line parsing capabilities!
404
429
  real_program
405
430
  end