storazzo 0.5.7 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +7 -5
- data/Makefile +8 -6
- data/Rakefile +15 -13
- data/VERSION +1 -1
- data/bin/hello-storazzo +1 -0
- data/bin/ricdisk-magic +33 -30
- data/bin/stats-with-md5 +182 -157
- data/bin/storazzo +112 -114
- data/bin/storazzo-symlink.rb +1 -0
- data/lib/storazzo/colors.rb +77 -36
- data/lib/storazzo/common.rb +72 -68
- data/lib/storazzo/debug.rb +2 -0
- data/lib/storazzo/hashify.rb +7 -5
- data/lib/storazzo/main.rb +59 -49
- data/lib/storazzo/media/abstract_ric_disk.rb +188 -179
- data/lib/storazzo/media/gcs_bucket.rb +76 -58
- data/lib/storazzo/media/local_folder.rb +56 -42
- data/lib/storazzo/media/mount_point.rb +20 -12
- data/lib/storazzo/ric_disk.rb +386 -383
- data/lib/storazzo/ric_disk_config.rb +227 -209
- data/lib/storazzo/ric_disk_sample_config.rb +26 -24
- data/lib/storazzo/ric_disk_statsfile.rb +19 -17
- data/lib/storazzo/ric_disk_ugly.rb +1 -0
- data/lib/storazzo/version.rb +3 -3
- data/lib/storazzo.rb +6 -7
- data/storazzo.gemspec +24 -16
- data/test/benchmark/for_future_use.rb +4 -2
- data/test/benchmark/test_hashing_functions-speed.rb +17 -0
- data/test/bin/new-idea.rb +17 -0
- data/test/bin/storazzo.rb +22 -25
- data/test/media/test_abstract_ric_disk.rb +5 -3
- data/test/media/test_gcs_bucket.rb +24 -23
- data/test/media/test_local_folder.rb +24 -23
- data/test/media/test_mount_point.rb +6 -5
- data/test/test_ric_disk.rb +6 -4
- data/test/test_ric_disk_config.rb +12 -11
- data/test/test_ric_disk_stats_file.rb +5 -3
- data/test/test_storazzo.rb +6 -4
- data/var/test/disks/disk02-full/Rakefile +7 -5
- data/var/test/disks/ricdisk_stats_v11.rds +11 -0
- metadata +13 -21
data/bin/stats-with-md5
CHANGED
@@ -8,7 +8,7 @@ require 'date' # for DateTime
|
|
8
8
|
require 'tempfile'
|
9
9
|
|
10
10
|
if RUBY_VERSION.split('.')[0] == 1
|
11
|
-
puts
|
11
|
+
puts 'Refusing to launch a script form Ruby 1. Sorry Ric, its 2020 damn it!'
|
12
12
|
exit 2020
|
13
13
|
end
|
14
14
|
|
@@ -16,62 +16,89 @@ $PROG_VER = '0.5' # on 0.4 it was copied from palladius/sakura. Now should be ma
|
|
16
16
|
$DEBUG = false
|
17
17
|
$reset_autowrite_file_reincarnation = 0
|
18
18
|
DEFAULT_MAX_FILES = 'Infinite'
|
19
|
+
$default_ping_frequency = 50
|
20
|
+
|
21
|
+
#
|
22
|
+
# ############################################################
|
23
|
+
# @author: Riccardo Carlesso
|
24
|
+
# @email: riccardo.carlesso@gmail.com
|
25
|
+
# @maturity: development
|
26
|
+
# @language: Ruby
|
27
|
+
# @tags: development, rcarlesso, test
|
28
|
+
# @works_on: Linux (little tested since v0.3), Mac (100% developed here)
|
29
|
+
# ############################################################
|
30
|
+
#
|
31
|
+
|
32
|
+
def deb(s)
|
33
|
+
puts "#DEB #{s}" if $DEBUG
|
34
|
+
end
|
19
35
|
|
20
|
-
|
36
|
+
# colors 16
|
37
|
+
def gray(s)
|
38
|
+
"\033[1;30m#{s}\033[0m"
|
39
|
+
end
|
21
40
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
@maturity: development
|
26
|
-
@language: Ruby
|
27
|
-
@tags: development, rcarlesso, test
|
28
|
-
@works_on: Linux (little tested since v0.3), Mac (100% developed here)
|
29
|
-
############################################################
|
41
|
+
def green(s)
|
42
|
+
"\033[1;32m#{s}\033[0m"
|
43
|
+
end
|
30
44
|
|
31
|
-
|
45
|
+
def red(s)
|
46
|
+
"\033[1;31m#{s}\033[0m"
|
47
|
+
end
|
32
48
|
|
33
|
-
def
|
34
|
-
#
|
35
|
-
|
36
|
-
|
37
|
-
def
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
def
|
42
|
-
|
49
|
+
def yellow(s)
|
50
|
+
"\033[1;33m#{s}\033[0m"
|
51
|
+
end
|
52
|
+
|
53
|
+
def blue(s)
|
54
|
+
"\033[1;34m#{s}\033[0m"
|
55
|
+
end
|
56
|
+
|
57
|
+
def purple(s)
|
58
|
+
"\033[1;35m#{s}\033[0m"
|
59
|
+
end
|
60
|
+
|
61
|
+
def azure(s)
|
62
|
+
"\033[1;36m#{s}\033[0m"
|
63
|
+
end
|
64
|
+
|
65
|
+
def white(s)
|
66
|
+
"\033[1;37m#{s}\033[0m"
|
67
|
+
end
|
43
68
|
|
44
69
|
# colors 64k
|
45
|
-
def orange(s)
|
70
|
+
def orange(s)
|
71
|
+
"\033[38;5;208m#{s}\033[0m"
|
72
|
+
end
|
46
73
|
|
47
74
|
# Program constants, automagically picked up by RicLib
|
48
75
|
# More configuration could be written in:
|
49
76
|
# $GIC/etc/ricsvn/<FILENAME>.yml
|
50
77
|
# That would go into the variable '$prog_conf_d'
|
51
78
|
$myconf = {
|
52
|
-
:
|
53
|
-
:
|
79
|
+
app_name: "AppName should be sth like #{$0}",
|
80
|
+
description: "
|
54
81
|
This program gets stats from Ruby File.Stats library
|
55
82
|
plus computes MD5 or CRC32 of the file.
|
56
83
|
And bakes it in a way that can be easily parsed.
|
57
|
-
".strip.gsub(/^\s+/,
|
84
|
+
".strip.gsub(/^\s+/, '').gsub(/\s+$/, '')
|
58
85
|
}
|
59
86
|
|
60
87
|
def usage(comment = nil)
|
61
88
|
puts white($optparse.banner)
|
62
89
|
puts($optparse.summarize)
|
63
|
-
puts(
|
90
|
+
puts('Description: ' + gray($myconf[:description]))
|
64
91
|
puts red(comment) if comment
|
65
92
|
exit 13
|
66
93
|
end
|
67
94
|
|
68
|
-
def reset_autowrite_file
|
95
|
+
def reset_autowrite_file
|
69
96
|
$autowrite_file = "# #{File.basename $0} Autowrite v1.0 #{Time.now} reincarnation=#{$reset_autowrite_file_reincarnation}\n"
|
70
97
|
$reset_autowrite_file_reincarnation += 1 # ($reset_autowrite_file_reincarnation + 1) # rescue 1
|
71
98
|
end
|
72
99
|
|
73
100
|
# include it in main if you want a custome one
|
74
|
-
def init
|
101
|
+
def init # see lib_autoinit in lib/util.rb
|
75
102
|
$opts = {}
|
76
103
|
# setting defaults
|
77
104
|
$opts[:verbose] = false
|
@@ -81,19 +108,22 @@ def init() # see lib_autoinit in lib/util.rb
|
|
81
108
|
$opts[:color] = true
|
82
109
|
$opts[:max_files] = nil # infinite
|
83
110
|
|
84
|
-
reset_autowrite_file
|
111
|
+
reset_autowrite_file
|
85
112
|
|
86
113
|
$optparse = OptionParser.new do |opts|
|
87
114
|
opts.banner = "#{$0} v.#{$PROG_VER}\n Usage: #{File.basename $0} [options] file1 file2 ..."
|
88
|
-
opts.on('-a', '--autowrite', 'automatically writes results to Root Folder (DFLT=false)')
|
115
|
+
opts.on('-a', '--autowrite', 'automatically writes results to Root Folder (DFLT=false)') do
|
89
116
|
$opts[:autowrite] = true
|
90
|
-
|
117
|
+
end
|
91
118
|
opts.on('-c', '--no-color', 'disables color (DFLT=false, that is color enabled)') { $opts[:color] = false }
|
92
|
-
opts.on('-d', '--debug', 'enables debug (DFLT=false)')
|
93
|
-
|
94
|
-
|
119
|
+
opts.on('-d', '--debug', 'enables debug (DFLT=false)') do
|
120
|
+
$opts[:debug] = true
|
121
|
+
$DEBUG = true
|
122
|
+
end
|
123
|
+
opts.on('-h', '--help', 'Display this screen') { usage }
|
124
|
+
opts.on('-m', '--max-files NUMBER', "sets max files per Dir to X (DFLT=#{DEFAULT_MAX_FILES})") do |nfiles|
|
95
125
|
$opts[:max_files] = nfiles.to_i
|
96
|
-
|
126
|
+
end
|
97
127
|
opts.on('-n', '--dryrun', "Don't really execute code") { $opts[:dryrun] = true }
|
98
128
|
opts.on('-l', '--logfile FILE', 'Write log to FILE') { |file| $opts[:logfile] = file }
|
99
129
|
opts.on('-v', '--verbose', 'Output more information') { $opts[:verbose] = true }
|
@@ -109,60 +139,61 @@ def compute_stats_and_md5(file)
|
|
109
139
|
ret = {}
|
110
140
|
ret[:name] = file
|
111
141
|
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
# defined?(stats.birthtime) ? # rescue stats.mtime # eg, 2022-03-05 21:47:51 +0100 on Mac (not implemented on my Linuix)#
|
124
|
-
# stats.birthtime : # works on Mac
|
125
|
-
# stats.ctime
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
142
|
+
begin
|
143
|
+
stats = File.stat(file)
|
144
|
+
ret[:stats_object] = stats # TODO: deprecate
|
145
|
+
# deb("Stats methods: #{stats.methods.sort.join(', ')}")
|
146
|
+
# deb(stats.ctime)
|
147
|
+
# puts(stats.birthtime rescue (stats.ctime))
|
148
|
+
# On Mac/Linux: see test/dumps/***.yaml
|
149
|
+
ret[:size] = stats.size
|
150
|
+
ret[:mode] = stats.mode
|
151
|
+
# datetimes
|
152
|
+
ret[:stat_safebirthtime] = getSafeCreationTime(file) # in Mac uses birthtime,but on Linux wont work
|
153
|
+
# defined?(stats.birthtime) ? # rescue stats.mtime # eg, 2022-03-05 21:47:51 +0100 on Mac (not implemented on my Linuix)#
|
154
|
+
# stats.birthtime : # works on Mac
|
155
|
+
# stats.ctime
|
156
|
+
ret[:stat_mtime] = stats.mtime # eg, 2022-03-05 21:47:51 +0100 Returns the modification time of stat.
|
157
|
+
ret[:stat_ctime] = stats.ctime # eg, 2022-03-05 21:47:51 +0100 Returns the change time for stat (that is, the time directory information about the file was changed, not the file itself). Note that on Windows (NTFS), returns creation time (birth time).
|
158
|
+
ret[:stat_atime] = stats.atime # eg, 2022-03-05 21:47:51 +0100 Last Access
|
159
|
+
ret[:stat_gid] = stats.gid # Group Id
|
160
|
+
ret[:stat_uid] = stats.uid # UUID
|
161
|
+
ret[:stat_ftype] = stats.ftype #
|
162
|
+
|
163
|
+
ret[:md5] = '______________NONE______________'
|
164
|
+
if stats.ftype != 'directory'
|
165
|
+
file_content = File.read(file)
|
166
|
+
ret[:md5] = Digest::MD5.hexdigest(file_content) # rescue 'none ' #=> string with hexadecimal digits
|
167
|
+
end
|
168
|
+
rescue Errno::EISDIR => e
|
169
|
+
# puts "It's a dir, nothing I can do here except skipping the stuff"
|
170
|
+
ret[:md5] = "_________I'm a dir sorry________"
|
171
|
+
rescue Exception => e
|
172
|
+
ret[:error] = e
|
173
|
+
end
|
174
|
+
ret
|
145
175
|
end
|
146
176
|
|
147
|
-
$print_stats_and_md5_version =
|
177
|
+
$print_stats_and_md5_version = '1.1b_220805_F' # files
|
148
178
|
$print_stats_and_md5_counter = 0
|
149
|
-
$print_stats_and_md5_for_gcs_version =
|
179
|
+
$print_stats_and_md5_for_gcs_version = '1.1alpha_220628_G' # GCS bucket
|
150
180
|
|
151
|
-
def stats_and_md5_number_of_files_processed
|
152
|
-
|
181
|
+
def stats_and_md5_number_of_files_processed
|
182
|
+
$print_stats_and_md5_counter
|
153
183
|
end
|
154
184
|
|
155
185
|
def smells_like_gcs?(file)
|
156
|
-
file =~
|
186
|
+
file =~ %r{gs://}
|
157
187
|
end
|
158
188
|
|
159
189
|
# You give me gs://bucket123/path/to/dir
|
160
190
|
def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts = {})
|
161
191
|
# opts_color = opts.fetch :color, true#
|
162
192
|
# opts_verbose = opts.fetch :verbose, false
|
163
|
-
# opts_ping_frequency = opts.fetch :ping_frequency,
|
193
|
+
# opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
|
164
194
|
opts_max_files = opts.fetch :max_files, nil # BigDecimal('Infinity')
|
165
195
|
opts_autowrite = opts.fetch :autowrite, false
|
196
|
+
opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
|
166
197
|
|
167
198
|
raise "Wrong GCS Path: #{mybucket_with_subdir}" unless smells_like_gcs?(mybucket_with_subdir)
|
168
199
|
|
@@ -177,23 +208,25 @@ def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts = {})
|
|
177
208
|
|
178
209
|
puts("[print_stats_and_md5_for_gcs] version=#{$print_stats_and_md5_for_gcs_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on='#{Time.now}' bucket=#{mybucket} subpath=#{gcs_subpath}")
|
179
210
|
begin
|
180
|
-
require
|
211
|
+
require 'google/cloud/storage'
|
181
212
|
project_id = `gcloud config get project 2>/dev/null`
|
182
213
|
deb "project_id: #{project_id}"
|
183
214
|
storage = Google::Cloud::Storage.new(project_id: project_id)
|
184
215
|
bucket = storage.bucket(cleanedup_bucket)
|
185
216
|
# grabs them ALL if nil, optherwise its an integer.
|
186
|
-
files = opts_max_files.nil?
|
187
|
-
|
188
|
-
|
217
|
+
files = if opts_max_files.nil?
|
218
|
+
bucket.files
|
219
|
+
else
|
220
|
+
bucket.files.first(opts_max_files)
|
221
|
+
end # not nil -> first(N)
|
189
222
|
# puts(files)
|
190
223
|
deb("All Sorted Methods: #{files.first.methods.sort}")
|
191
224
|
files.each do |gcs_file|
|
192
|
-
md5 = Base64.decode64(gcs_file.md5).
|
225
|
+
md5 = Base64.decode64(gcs_file.md5).unpack1('H*') # md5 in base64 -> then de-binarizied: Base64.urlsafe_decode64(md5).unpack('H*')[0]
|
193
226
|
size = gcs_file.size # crc rescue :boh
|
194
227
|
time_created = gcs_file.created_at
|
195
228
|
mode = 'XXXXXX' # for compatbility with files
|
196
|
-
file_type = gcs_file.name.to_s =~
|
229
|
+
file_type = gcs_file.name.to_s =~ %r{/$} ? 'd' : 'f' # if ends with a slash its a DIR :P
|
197
230
|
# puts("[OLD_GCS_v11] #{md5} #{size}\t#{time_created} [#{gcs_file.content_type}]\t#{gcs_file.name}")
|
198
231
|
print_colored_polymoprhic('gcs_v1.2', md5, mode, file_type, size, gcs_file.content_type, time_created,
|
199
232
|
gcs_file.name, opts)
|
@@ -217,7 +250,7 @@ end
|
|
217
250
|
def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_type, creation_time, filename, opts = {})
|
218
251
|
opts_color = opts.fetch :color, true
|
219
252
|
opts_verbose = opts.fetch :verbose, false
|
220
|
-
opts_ping_frequency = opts.fetch :ping_frequency,
|
253
|
+
opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
|
221
254
|
opts_autowrite = opts.fetch :autowrite, false
|
222
255
|
|
223
256
|
# Increment counter across all!
|
@@ -225,11 +258,11 @@ def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_t
|
|
225
258
|
# in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
|
226
259
|
# $stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
|
227
260
|
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
|
228
|
-
|
261
|
+
warn "-- #{Time.now} print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if $print_stats_and_md5_counter % opts_ping_frequency == 1
|
229
262
|
|
230
263
|
maybecolored_md5 = opts_color ? red(md5) : md5
|
231
264
|
maybecolored_filename = opts_color ? azure(filename) : filename
|
232
|
-
enlarged_size =
|
265
|
+
enlarged_size = format('%7o', size.to_s) # or SIZE
|
233
266
|
maybecolored_size = opts_color ? yellow(enlarged_size) : enlarged_size
|
234
267
|
maybecolored_file_type = file_type
|
235
268
|
colored_content_type = opts_color ? white(content_type) : content_type
|
@@ -241,22 +274,24 @@ def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_t
|
|
241
274
|
else red(file_type)
|
242
275
|
end
|
243
276
|
end
|
244
|
-
standardized_creation_time = creation_time.is_a?(DateTime)
|
245
|
-
|
246
|
-
|
277
|
+
standardized_creation_time = if creation_time.is_a?(DateTime)
|
278
|
+
creation_time
|
279
|
+
else
|
280
|
+
DateTime.parse(creation_time.to_s)
|
281
|
+
end # if you prefer to use Time since its already supported by Storazzo (but its ugly since it has SPACES! so hard to parse) use tt = Time.parse(d.to_s)
|
247
282
|
# as per https://stackoverflow.com/questions/279769/convert-to-from-datetime-and-time-in-ruby
|
248
283
|
# puts "#{maybecolored_md5} #{mode} #{maybecolored_file_type} #{maybecolored_size}\t#{creation_time}(DEB #{creation_time.class})(DEB2 #{DateTime.parse(creation_time.to_s)}) [#{colored_content_type}] #{maybecolored_filename}"
|
249
284
|
# this mightr be colored
|
250
285
|
str = "[#{entity_type}] #{maybecolored_md5} #{mode} #{maybecolored_file_type} #{standardized_creation_time} #{maybecolored_size} [#{colored_content_type}] #{maybecolored_filename}\n"
|
251
286
|
# this will NEVER be colored. WARNING, now you need to maintain TWO of these beasts :/
|
252
287
|
non_colored_str = "[#{entity_type}] #{md5} #{mode} #{file_type} #{standardized_creation_time} #{enlarged_size} [#{content_type}] #{filename}\n"
|
253
|
-
if
|
288
|
+
if opts_autowrite
|
254
289
|
# need to guarantee this is NOT colored. The easiest way to do it is TWICVE as expensive but... whatevs.
|
255
290
|
# TODO(ricc): fire me for this lazimness!
|
256
|
-
$autowrite_file +=
|
291
|
+
$autowrite_file += non_colored_str
|
257
292
|
end
|
258
293
|
print(str)
|
259
|
-
|
294
|
+
str
|
260
295
|
end
|
261
296
|
|
262
297
|
def print_stats_and_md5(file, opts = {})
|
@@ -264,39 +299,34 @@ def print_stats_and_md5(file, opts = {})
|
|
264
299
|
|
265
300
|
opts_color = opts.fetch :color, true
|
266
301
|
opts_verbose = opts.fetch :verbose, false
|
267
|
-
opts_ping_frequency = opts.fetch :ping_frequency,
|
268
|
-
|
269
|
-
# $print_stats_and_md5_counter += 1 # global counter! I should increment at the END of fucntion.. but you never know if i exit wrongly so i do at beginning. so within this function the real #files is N-1 (i.e., if N is 6, we're processing file %5)
|
270
|
-
|
271
|
-
# # in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
|
272
|
-
# #$stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
|
273
|
-
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
|
274
|
-
# $stderr.puts "-- print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if (($print_stats_and_md5_counter) % opts_ping_frequency == 1 )
|
302
|
+
opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
|
275
303
|
|
276
|
-
|
304
|
+
puts "print_stats_and_md5: #{file}" if opts_verbose
|
277
305
|
stats = compute_stats_and_md5 file
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
306
|
+
|
307
|
+
mode = begin
|
308
|
+
format('%06o', stats[:mode])
|
309
|
+
rescue StandardError
|
310
|
+
:ERROR
|
311
|
+
end # .to_s.right(4) #=> "100644"
|
312
|
+
file_type = begin
|
313
|
+
stats[:stat_ftype][0]
|
314
|
+
rescue StandardError
|
315
|
+
'?'
|
316
|
+
end
|
283
317
|
file_type = 's' if File.symlink?(file)
|
284
318
|
content_type = `file --mime-type -b '#{file}' 2>/dev/null`.chomp # .split(': ',2)[1]
|
285
319
|
|
286
|
-
# colored_string = "[COL] #{red stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{white stats[:name]}"
|
287
|
-
# boring_string = "[B/W] #{ stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{ stats[:name]}"
|
288
|
-
# puts(opts_color ? colored_string : boring_string)
|
289
|
-
# puts "[OLDv1.1deprecated] #{maybecolored_md5} #{mode} #{file_type} #{maybecolored_size} #{stats[:stat_safebirthtime]} #{maybecolored_filename}"
|
290
320
|
print_colored_polymoprhic('file_v1.2', stats[:md5], mode, file_type, stats[:size], content_type,
|
291
321
|
stats[:stat_safebirthtime], stats[:name], opts)
|
292
322
|
end
|
293
323
|
|
294
|
-
def autowrite_to_dir_or_gcs(fs_type, path,
|
295
|
-
autowrite_version =
|
296
|
-
file_to_save =
|
297
|
-
path_to_save = path +
|
298
|
-
puts(red
|
299
|
-
puts(red
|
324
|
+
def autowrite_to_dir_or_gcs(fs_type, path, _opts = {})
|
325
|
+
autowrite_version = '1.1_28jun22'
|
326
|
+
file_to_save = 'stats-with-md5.log'
|
327
|
+
path_to_save = path + '/' + file_to_save
|
328
|
+
puts(red("TODO(ricc): write results [size=#{$autowrite_file.size}] in this Directory: #{path_to_save}")) if fs_type == :dir
|
329
|
+
puts(red("TODO(ricc): write results [size=#{$autowrite_file.size}] in this GCS Bucket or Dir: #{path_to_save}")) if fs_type == :gcs
|
300
330
|
|
301
331
|
##########################################################################
|
302
332
|
# do the thing: creates file locally and moves to GCS or Dir.
|
@@ -306,12 +336,8 @@ def autowrite_to_dir_or_gcs(fs_type, path, opts = {})
|
|
306
336
|
tmpfile.write($autowrite_file)
|
307
337
|
cmd = :no_cmd_yet
|
308
338
|
|
309
|
-
if fs_type == :gcs
|
310
|
-
|
311
|
-
end
|
312
|
-
if fs_type == :dir
|
313
|
-
cmd = "mv '#{tmpfile.path}' #{path}/#{file_to_save}"
|
314
|
-
end
|
339
|
+
cmd = "gsutil mv '#{tmpfile.path}' #{path}/#{file_to_save}" if fs_type == :gcs
|
340
|
+
cmd = "mv '#{tmpfile.path}' #{path}/#{file_to_save}" if fs_type == :dir
|
315
341
|
|
316
342
|
puts("TMP File is this big: #{azure tmpfile.size}")
|
317
343
|
puts("About to execute this: #{white cmd}")
|
@@ -330,28 +356,27 @@ def autowrite_to_dir_or_gcs(fs_type, path, opts = {})
|
|
330
356
|
#####################################
|
331
357
|
# finished, lets now clean up the file...
|
332
358
|
#####################################
|
333
|
-
reset_autowrite_file
|
334
|
-
|
359
|
+
reset_autowrite_file
|
360
|
+
:ok
|
335
361
|
end
|
336
362
|
|
363
|
+
def print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, opts = {})
|
364
|
+
puts "# [print_stats_and_md5_for_directory_from_cli] v#{$print_stats_and_md5_version}] DIR to explore (make sure you glob also): #{directory_to_explore_recursively}"
|
365
|
+
opts_autowrite = opts.fetch :autowrite, false
|
366
|
+
opts_verbose = opts.fetch(:verbose, $opts[:verbose])
|
367
|
+
opts_debug = opts.fetch :debug, $opts[:debug]
|
368
|
+
opts_color = opts.fetch :color, $opts[:color]
|
369
|
+
if opts_debug
|
370
|
+
puts "# DEB Options1 opts (FUNC) BAD: s #{opts}" # probably useless since this is invoked by CLI and has ful fledged options.
|
371
|
+
puts "# DEB Options2 $opts (ARGV) GOOD: #{$opts}" # This is what we ewant in this CLI. TODO(ricc): if you move this function to be used as function invert the priority...
|
372
|
+
puts("# DEB version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") # if ARGV.size > 0
|
373
|
+
end
|
337
374
|
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
opts_color = opts.fetch :color, $opts[:color]
|
344
|
-
if opts_debug
|
345
|
-
puts "# DEB Options1 opts (FUNC) BAD: s #{opts}" # probably useless since this is invoked by CLI and has ful fledged options.
|
346
|
-
puts "# DEB Options2 $opts (ARGV) GOOD: #{$opts}" # This is what we ewant in this CLI. TODO(ricc): if you move this function to be used as function invert the priority...
|
347
|
-
puts("# DEB version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") # if ARGV.size > 0
|
348
|
-
end
|
349
|
-
|
350
|
-
Dir.glob("#{(directory_to_explore_recursively)}/**/*") do |globbed_filename|
|
351
|
-
# Do work on files & directories ending in .rb
|
352
|
-
#puts "[deb] #{globbed_filename}"
|
353
|
-
print_stats_and_md5(globbed_filename, :color => opts_color, :verbose => opts_verbose, :autowrite => opts_autowrite)
|
354
|
-
end
|
375
|
+
Dir.glob("#{directory_to_explore_recursively}/**/*") do |globbed_filename|
|
376
|
+
# Do work on files & directories ending in .rb
|
377
|
+
# puts "[deb] #{globbed_filename}"
|
378
|
+
print_stats_and_md5(globbed_filename, color: opts_color, verbose: opts_verbose, autowrite: opts_autowrite)
|
379
|
+
end
|
355
380
|
|
356
381
|
autowrite_to_dir_or_gcs(:dir, directory_to_explore_recursively) if opts_autowrite
|
357
382
|
end
|
@@ -365,41 +390,41 @@ def real_program
|
|
365
390
|
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ARGV.size > 0
|
366
391
|
|
367
392
|
common_opts = {
|
368
|
-
:
|
369
|
-
:
|
370
|
-
:
|
371
|
-
:
|
393
|
+
max_files: $opts[:max_files],
|
394
|
+
color: $opts[:color],
|
395
|
+
verbose: $opts[:verbose],
|
396
|
+
autowrite: $opts[:autowrite]
|
372
397
|
}
|
373
398
|
|
374
399
|
if ARGV.size == 1
|
375
400
|
directory_to_explore_recursively = ARGV[0]
|
376
401
|
if smells_like_gcs?(directory_to_explore_recursively)
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
402
|
+
print_stats_and_md5_for_gcs(directory_to_explore_recursively, common_opts)
|
403
|
+
else # normal file..
|
404
|
+
print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, common_opts)
|
405
|
+
end
|
381
406
|
elsif ARGV.size > 1
|
382
|
-
deb
|
407
|
+
deb '2. I expect a lot of single files or directories:'
|
383
408
|
for arg in ARGV
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
409
|
+
if File.directory?(arg)
|
410
|
+
print_stats_and_md5_for_directory_from_cli(arg, common_opts)
|
411
|
+
else
|
412
|
+
print_stats_and_md5(arg, common_opts)
|
413
|
+
end
|
389
414
|
end
|
390
415
|
else
|
391
|
-
puts
|
416
|
+
puts 'No args given. Exiting'
|
392
417
|
exit 41
|
393
418
|
end
|
394
419
|
tf = Time.now
|
395
|
-
#rounding to 3 decimals
|
396
|
-
approx_delta = (tf-t0).round(3)
|
397
|
-
puts "# [#{File.basename $0}] Time taken for processing #{stats_and_md5_number_of_files_processed} files: #{approx_delta} seconds"
|
420
|
+
# rounding to 3 decimals
|
421
|
+
approx_delta = (tf - t0).round(3)
|
422
|
+
puts "# [#{File.basename $0}] Time taken for processing #{stats_and_md5_number_of_files_processed} files: #{approx_delta} seconds"
|
398
423
|
end
|
399
424
|
|
400
425
|
def main(filename)
|
401
426
|
deb "I'm called by #{white filename}"
|
402
|
-
deb
|
427
|
+
deb 'To remove this shit, just set $DEBUG=false :)'
|
403
428
|
init # Enable this to have command line parsing capabilities!
|
404
429
|
real_program
|
405
430
|
end
|