storazzo 0.5.1 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +7 -5
- data/LICENSE +20 -1
- data/Makefile +9 -5
- data/Rakefile +22 -9
- data/VERSION +1 -1
- data/bin/hello-storazzo +1 -0
- data/bin/ricdisk-magic +33 -30
- data/bin/stats-with-md5 +146 -176
- data/bin/storazzo +158 -0
- data/bin/storazzo-symlink.rb +1 -0
- data/bin/storazzo-util +1 -0
- data/lib/storazzo/colors.rb +77 -36
- data/lib/storazzo/common.rb +72 -65
- data/lib/storazzo/debug.rb +2 -0
- data/lib/storazzo/hashify.rb +7 -5
- data/lib/storazzo/main.rb +59 -49
- data/lib/storazzo/media/abstract_ric_disk.rb +190 -161
- data/lib/storazzo/media/gcs_bucket.rb +78 -47
- data/lib/storazzo/media/local_folder.rb +56 -42
- data/lib/storazzo/media/mount_point.rb +22 -6
- data/lib/storazzo/ric_disk.rb +386 -383
- data/lib/storazzo/ric_disk_config.rb +229 -197
- data/lib/storazzo/ric_disk_sample_config.rb +26 -24
- data/lib/storazzo/ric_disk_statsfile.rb +19 -17
- data/lib/storazzo/ric_disk_ugly.rb +1 -0
- data/lib/storazzo/version.rb +3 -3
- data/lib/storazzo.rb +6 -7
- data/storazzo.gemspec +31 -19
- data/test/benchmark/for_future_use.rb +15 -0
- data/test/benchmark/test_hashing_functions-speed.rb +17 -0
- data/test/bin/new-idea.rb +17 -0
- data/test/bin/storazzo.rb +25 -0
- data/test/media/test_abstract_ric_disk.rb +6 -4
- data/test/media/test_gcs_bucket.rb +55 -21
- data/test/media/test_local_folder.rb +28 -29
- data/test/media/test_mount_point.rb +7 -5
- data/test/test_ric_disk.rb +8 -6
- data/test/test_ric_disk_config.rb +13 -12
- data/test/test_ric_disk_stats_file.rb +5 -3
- data/test/test_storazzo.rb +6 -4
- data/var/test/disks/disk02-full/Rakefile +7 -5
- data/var/test/disks/ricdisk_stats_v11.rds +11 -22
- metadata +22 -22
data/bin/stats-with-md5
CHANGED
@@ -8,7 +8,7 @@ require 'date' # for DateTime
|
|
8
8
|
require 'tempfile'
|
9
9
|
|
10
10
|
if RUBY_VERSION.split('.')[0] == 1
|
11
|
-
puts
|
11
|
+
puts 'Refusing to launch a script form Ruby 1. Sorry Ric, its 2020 damn it!'
|
12
12
|
exit 2020
|
13
13
|
end
|
14
14
|
|
@@ -16,62 +16,89 @@ $PROG_VER = '0.5' # on 0.4 it was copied from palladius/sakura. Now should be ma
|
|
16
16
|
$DEBUG = false
|
17
17
|
$reset_autowrite_file_reincarnation = 0
|
18
18
|
DEFAULT_MAX_FILES = 'Infinite'
|
19
|
+
$default_ping_frequency = 50
|
20
|
+
|
21
|
+
#
|
22
|
+
# ############################################################
|
23
|
+
# @author: Riccardo Carlesso
|
24
|
+
# @email: riccardo.carlesso@gmail.com
|
25
|
+
# @maturity: development
|
26
|
+
# @language: Ruby
|
27
|
+
# @tags: development, rcarlesso, test
|
28
|
+
# @works_on: Linux (little tested since v0.3), Mac (100% developed here)
|
29
|
+
# ############################################################
|
30
|
+
#
|
31
|
+
|
32
|
+
def deb(s)
|
33
|
+
puts "#DEB #{s}" if $DEBUG
|
34
|
+
end
|
35
|
+
|
36
|
+
# colors 16
|
37
|
+
def gray(s)
|
38
|
+
"\033[1;30m#{s}\033[0m"
|
39
|
+
end
|
19
40
|
|
20
|
-
|
41
|
+
def green(s)
|
42
|
+
"\033[1;32m#{s}\033[0m"
|
43
|
+
end
|
21
44
|
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
@maturity: development
|
26
|
-
@language: Ruby
|
27
|
-
@tags: development, rcarlesso, test
|
28
|
-
@works_on: Linux (little tested since v0.3), Mac (100% developed here)
|
29
|
-
############################################################
|
45
|
+
def red(s)
|
46
|
+
"\033[1;31m#{s}\033[0m"
|
47
|
+
end
|
30
48
|
|
31
|
-
|
49
|
+
def yellow(s)
|
50
|
+
"\033[1;33m#{s}\033[0m"
|
51
|
+
end
|
32
52
|
|
33
|
-
def
|
34
|
-
#
|
35
|
-
|
36
|
-
|
37
|
-
def
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
def azure(s)
|
42
|
-
|
53
|
+
def blue(s)
|
54
|
+
"\033[1;34m#{s}\033[0m"
|
55
|
+
end
|
56
|
+
|
57
|
+
def purple(s)
|
58
|
+
"\033[1;35m#{s}\033[0m"
|
59
|
+
end
|
60
|
+
|
61
|
+
def azure(s)
|
62
|
+
"\033[1;36m#{s}\033[0m"
|
63
|
+
end
|
64
|
+
|
65
|
+
def white(s)
|
66
|
+
"\033[1;37m#{s}\033[0m"
|
67
|
+
end
|
43
68
|
|
44
69
|
# colors 64k
|
45
|
-
def orange(s)
|
70
|
+
def orange(s)
|
71
|
+
"\033[38;5;208m#{s}\033[0m"
|
72
|
+
end
|
46
73
|
|
47
74
|
# Program constants, automagically picked up by RicLib
|
48
75
|
# More configuration could be written in:
|
49
76
|
# $GIC/etc/ricsvn/<FILENAME>.yml
|
50
77
|
# That would go into the variable '$prog_conf_d'
|
51
78
|
$myconf = {
|
52
|
-
:
|
53
|
-
:
|
79
|
+
app_name: "AppName should be sth like #{$0}",
|
80
|
+
description: "
|
54
81
|
This program gets stats from Ruby File.Stats library
|
55
82
|
plus computes MD5 or CRC32 of the file.
|
56
83
|
And bakes it in a way that can be easily parsed.
|
57
|
-
".strip.gsub(/^\s+/,
|
84
|
+
".strip.gsub(/^\s+/, '').gsub(/\s+$/, '')
|
58
85
|
}
|
59
86
|
|
60
87
|
def usage(comment = nil)
|
61
88
|
puts white($optparse.banner)
|
62
89
|
puts($optparse.summarize)
|
63
|
-
puts(
|
90
|
+
puts('Description: ' + gray($myconf[:description]))
|
64
91
|
puts red(comment) if comment
|
65
92
|
exit 13
|
66
93
|
end
|
67
94
|
|
68
|
-
def reset_autowrite_file
|
95
|
+
def reset_autowrite_file
|
69
96
|
$autowrite_file = "# #{File.basename $0} Autowrite v1.0 #{Time.now} reincarnation=#{$reset_autowrite_file_reincarnation}\n"
|
70
97
|
$reset_autowrite_file_reincarnation += 1 # ($reset_autowrite_file_reincarnation + 1) # rescue 1
|
71
98
|
end
|
72
99
|
|
73
100
|
# include it in main if you want a custome one
|
74
|
-
def init
|
101
|
+
def init # see lib_autoinit in lib/util.rb
|
75
102
|
$opts = {}
|
76
103
|
# setting defaults
|
77
104
|
$opts[:verbose] = false
|
@@ -81,19 +108,22 @@ def init() # see lib_autoinit in lib/util.rb
|
|
81
108
|
$opts[:color] = true
|
82
109
|
$opts[:max_files] = nil # infinite
|
83
110
|
|
84
|
-
reset_autowrite_file
|
111
|
+
reset_autowrite_file
|
85
112
|
|
86
113
|
$optparse = OptionParser.new do |opts|
|
87
114
|
opts.banner = "#{$0} v.#{$PROG_VER}\n Usage: #{File.basename $0} [options] file1 file2 ..."
|
88
|
-
opts.on('-a', '--autowrite', 'automatically writes results to Root Folder (DFLT=false)')
|
115
|
+
opts.on('-a', '--autowrite', 'automatically writes results to Root Folder (DFLT=false)') do
|
89
116
|
$opts[:autowrite] = true
|
90
|
-
|
117
|
+
end
|
91
118
|
opts.on('-c', '--no-color', 'disables color (DFLT=false, that is color enabled)') { $opts[:color] = false }
|
92
|
-
opts.on('-d', '--debug', 'enables debug (DFLT=false)')
|
93
|
-
|
94
|
-
|
119
|
+
opts.on('-d', '--debug', 'enables debug (DFLT=false)') do
|
120
|
+
$opts[:debug] = true
|
121
|
+
$DEBUG = true
|
122
|
+
end
|
123
|
+
opts.on('-h', '--help', 'Display this screen') { usage }
|
124
|
+
opts.on('-m', '--max-files NUMBER', "sets max files per Dir to X (DFLT=#{DEFAULT_MAX_FILES})") do |nfiles|
|
95
125
|
$opts[:max_files] = nfiles.to_i
|
96
|
-
|
126
|
+
end
|
97
127
|
opts.on('-n', '--dryrun', "Don't really execute code") { $opts[:dryrun] = true }
|
98
128
|
opts.on('-l', '--logfile FILE', 'Write log to FILE') { |file| $opts[:logfile] = file }
|
99
129
|
opts.on('-v', '--verbose', 'Output more information') { $opts[:verbose] = true }
|
@@ -109,12 +139,11 @@ def compute_stats_and_md5(file)
|
|
109
139
|
ret = {}
|
110
140
|
ret[:name] = file
|
111
141
|
|
112
|
-
<<<<<<< HEAD
|
113
142
|
begin
|
114
143
|
stats = File.stat(file)
|
115
|
-
ret[:stats_object] = stats # TODO deprecate
|
116
|
-
deb("Stats methods: #{stats.methods.sort.join(', ')}")
|
117
|
-
deb(stats.ctime)
|
144
|
+
ret[:stats_object] = stats # TODO: deprecate
|
145
|
+
# deb("Stats methods: #{stats.methods.sort.join(', ')}")
|
146
|
+
# deb(stats.ctime)
|
118
147
|
# puts(stats.birthtime rescue (stats.ctime))
|
119
148
|
# On Mac/Linux: see test/dumps/***.yaml
|
120
149
|
ret[:size] = stats.size
|
@@ -132,7 +161,7 @@ def compute_stats_and_md5(file)
|
|
132
161
|
ret[:stat_ftype] = stats.ftype #
|
133
162
|
|
134
163
|
ret[:md5] = '______________NONE______________'
|
135
|
-
if stats.ftype !=
|
164
|
+
if stats.ftype != 'directory'
|
136
165
|
file_content = File.read(file)
|
137
166
|
ret[:md5] = Digest::MD5.hexdigest(file_content) # rescue 'none ' #=> string with hexadecimal digits
|
138
167
|
end
|
@@ -143,62 +172,28 @@ def compute_stats_and_md5(file)
|
|
143
172
|
ret[:error] = e
|
144
173
|
end
|
145
174
|
ret
|
146
|
-
=======
|
147
|
-
begin
|
148
|
-
stats = File.stat(file)
|
149
|
-
ret[:stats_object] = stats # TODO deprecate
|
150
|
-
#deb("Stats methods: #{stats.methods.sort.join(', ')}")
|
151
|
-
#deb(stats.ctime)
|
152
|
-
#puts(stats.birthtime rescue (stats.ctime))
|
153
|
-
# On Mac/Linux: see test/dumps/***.yaml
|
154
|
-
ret[:size] = stats.size
|
155
|
-
ret[:mode] = stats.mode
|
156
|
-
# datetimes
|
157
|
-
ret[:stat_safebirthtime] = getSafeCreationTime(file) # in Mac uses birthtime,but on Linux wont work
|
158
|
-
# defined?(stats.birthtime) ? # rescue stats.mtime # eg, 2022-03-05 21:47:51 +0100 on Mac (not implemented on my Linuix)#
|
159
|
-
# stats.birthtime : # works on Mac
|
160
|
-
# stats.ctime
|
161
|
-
ret[:stat_mtime] = stats.mtime # eg, 2022-03-05 21:47:51 +0100 Returns the modification time of stat.
|
162
|
-
ret[:stat_ctime] = stats.ctime # eg, 2022-03-05 21:47:51 +0100 Returns the change time for stat (that is, the time directory information about the file was changed, not the file itself). Note that on Windows (NTFS), returns creation time (birth time).
|
163
|
-
ret[:stat_atime] = stats.atime # eg, 2022-03-05 21:47:51 +0100 Last Access
|
164
|
-
ret[:stat_gid] = stats.gid # Group Id
|
165
|
-
ret[:stat_uid] = stats.uid # UUID
|
166
|
-
ret[:stat_ftype] = stats.ftype #
|
167
|
-
|
168
|
-
ret[:md5] = '______________NONE______________'
|
169
|
-
if stats.ftype != "directory"
|
170
|
-
file_content = File.read(file)
|
171
|
-
ret[:md5] = Digest::MD5.hexdigest(file_content) # rescue 'none ' #=> string with hexadecimal digits
|
172
|
-
end
|
173
|
-
rescue Errno::EISDIR => e
|
174
|
-
#puts "It's a dir, nothing I can do here except skipping the stuff"
|
175
|
-
ret[:md5] = "_________I'm a dir sorry________"
|
176
|
-
rescue Exception => e
|
177
|
-
ret[:error] = e
|
178
|
-
end
|
179
|
-
ret
|
180
|
-
>>>>>>> 97a64f1 (perfected CLI)
|
181
175
|
end
|
182
176
|
|
183
|
-
$print_stats_and_md5_version =
|
177
|
+
$print_stats_and_md5_version = '1.1b_220805_F' # files
|
184
178
|
$print_stats_and_md5_counter = 0
|
185
|
-
$print_stats_and_md5_for_gcs_version =
|
179
|
+
$print_stats_and_md5_for_gcs_version = '1.1alpha_220628_G' # GCS bucket
|
186
180
|
|
187
|
-
def stats_and_md5_number_of_files_processed
|
188
|
-
|
181
|
+
def stats_and_md5_number_of_files_processed
|
182
|
+
$print_stats_and_md5_counter
|
189
183
|
end
|
190
184
|
|
191
185
|
def smells_like_gcs?(file)
|
192
|
-
file =~
|
186
|
+
file =~ %r{gs://}
|
193
187
|
end
|
194
188
|
|
195
189
|
# You give me gs://bucket123/path/to/dir
|
196
190
|
def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts = {})
|
197
191
|
# opts_color = opts.fetch :color, true#
|
198
192
|
# opts_verbose = opts.fetch :verbose, false
|
199
|
-
# opts_ping_frequency = opts.fetch :ping_frequency,
|
193
|
+
# opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
|
200
194
|
opts_max_files = opts.fetch :max_files, nil # BigDecimal('Infinity')
|
201
195
|
opts_autowrite = opts.fetch :autowrite, false
|
196
|
+
opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
|
202
197
|
|
203
198
|
raise "Wrong GCS Path: #{mybucket_with_subdir}" unless smells_like_gcs?(mybucket_with_subdir)
|
204
199
|
|
@@ -213,23 +208,25 @@ def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts = {})
|
|
213
208
|
|
214
209
|
puts("[print_stats_and_md5_for_gcs] version=#{$print_stats_and_md5_for_gcs_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on='#{Time.now}' bucket=#{mybucket} subpath=#{gcs_subpath}")
|
215
210
|
begin
|
216
|
-
require
|
211
|
+
require 'google/cloud/storage'
|
217
212
|
project_id = `gcloud config get project 2>/dev/null`
|
218
213
|
deb "project_id: #{project_id}"
|
219
214
|
storage = Google::Cloud::Storage.new(project_id: project_id)
|
220
215
|
bucket = storage.bucket(cleanedup_bucket)
|
221
216
|
# grabs them ALL if nil, optherwise its an integer.
|
222
|
-
files = opts_max_files.nil?
|
223
|
-
|
224
|
-
|
217
|
+
files = if opts_max_files.nil?
|
218
|
+
bucket.files
|
219
|
+
else
|
220
|
+
bucket.files.first(opts_max_files)
|
221
|
+
end # not nil -> first(N)
|
225
222
|
# puts(files)
|
226
223
|
deb("All Sorted Methods: #{files.first.methods.sort}")
|
227
224
|
files.each do |gcs_file|
|
228
|
-
md5 = Base64.decode64(gcs_file.md5).
|
225
|
+
md5 = Base64.decode64(gcs_file.md5).unpack1('H*') # md5 in base64 -> then de-binarizied: Base64.urlsafe_decode64(md5).unpack('H*')[0]
|
229
226
|
size = gcs_file.size # crc rescue :boh
|
230
227
|
time_created = gcs_file.created_at
|
231
228
|
mode = 'XXXXXX' # for compatbility with files
|
232
|
-
file_type = gcs_file.name.to_s =~
|
229
|
+
file_type = gcs_file.name.to_s =~ %r{/$} ? 'd' : 'f' # if ends with a slash its a DIR :P
|
233
230
|
# puts("[OLD_GCS_v11] #{md5} #{size}\t#{time_created} [#{gcs_file.content_type}]\t#{gcs_file.name}")
|
234
231
|
print_colored_polymoprhic('gcs_v1.2', md5, mode, file_type, size, gcs_file.content_type, time_created,
|
235
232
|
gcs_file.name, opts)
|
@@ -253,7 +250,7 @@ end
|
|
253
250
|
def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_type, creation_time, filename, opts = {})
|
254
251
|
opts_color = opts.fetch :color, true
|
255
252
|
opts_verbose = opts.fetch :verbose, false
|
256
|
-
opts_ping_frequency = opts.fetch :ping_frequency,
|
253
|
+
opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
|
257
254
|
opts_autowrite = opts.fetch :autowrite, false
|
258
255
|
|
259
256
|
# Increment counter across all!
|
@@ -261,11 +258,11 @@ def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_t
|
|
261
258
|
# in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
|
262
259
|
# $stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
|
263
260
|
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
|
264
|
-
|
261
|
+
warn "-- #{Time.now} print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if $print_stats_and_md5_counter % opts_ping_frequency == 1
|
265
262
|
|
266
263
|
maybecolored_md5 = opts_color ? red(md5) : md5
|
267
264
|
maybecolored_filename = opts_color ? azure(filename) : filename
|
268
|
-
enlarged_size =
|
265
|
+
enlarged_size = format('%7o', size.to_s) # or SIZE
|
269
266
|
maybecolored_size = opts_color ? yellow(enlarged_size) : enlarged_size
|
270
267
|
maybecolored_file_type = file_type
|
271
268
|
colored_content_type = opts_color ? white(content_type) : content_type
|
@@ -277,22 +274,24 @@ def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_t
|
|
277
274
|
else red(file_type)
|
278
275
|
end
|
279
276
|
end
|
280
|
-
standardized_creation_time = creation_time.is_a?(DateTime)
|
281
|
-
|
282
|
-
|
277
|
+
standardized_creation_time = if creation_time.is_a?(DateTime)
|
278
|
+
creation_time
|
279
|
+
else
|
280
|
+
DateTime.parse(creation_time.to_s)
|
281
|
+
end # if you prefer to use Time since its already supported by Storazzo (but its ugly since it has SPACES! so hard to parse) use tt = Time.parse(d.to_s)
|
283
282
|
# as per https://stackoverflow.com/questions/279769/convert-to-from-datetime-and-time-in-ruby
|
284
283
|
# puts "#{maybecolored_md5} #{mode} #{maybecolored_file_type} #{maybecolored_size}\t#{creation_time}(DEB #{creation_time.class})(DEB2 #{DateTime.parse(creation_time.to_s)}) [#{colored_content_type}] #{maybecolored_filename}"
|
285
284
|
# this mightr be colored
|
286
285
|
str = "[#{entity_type}] #{maybecolored_md5} #{mode} #{maybecolored_file_type} #{standardized_creation_time} #{maybecolored_size} [#{colored_content_type}] #{maybecolored_filename}\n"
|
287
286
|
# this will NEVER be colored. WARNING, now you need to maintain TWO of these beasts :/
|
288
287
|
non_colored_str = "[#{entity_type}] #{md5} #{mode} #{file_type} #{standardized_creation_time} #{enlarged_size} [#{content_type}] #{filename}\n"
|
289
|
-
if
|
288
|
+
if opts_autowrite
|
290
289
|
# need to guarantee this is NOT colored. The easiest way to do it is TWICVE as expensive but... whatevs.
|
291
290
|
# TODO(ricc): fire me for this lazimness!
|
292
|
-
$autowrite_file +=
|
291
|
+
$autowrite_file += non_colored_str
|
293
292
|
end
|
294
293
|
print(str)
|
295
|
-
|
294
|
+
str
|
296
295
|
end
|
297
296
|
|
298
297
|
def print_stats_and_md5(file, opts = {})
|
@@ -300,39 +299,34 @@ def print_stats_and_md5(file, opts = {})
|
|
300
299
|
|
301
300
|
opts_color = opts.fetch :color, true
|
302
301
|
opts_verbose = opts.fetch :verbose, false
|
303
|
-
opts_ping_frequency = opts.fetch :ping_frequency,
|
304
|
-
|
305
|
-
# $print_stats_and_md5_counter += 1 # global counter! I should increment at the END of fucntion.. but you never know if i exit wrongly so i do at beginning. so within this function the real #files is N-1 (i.e., if N is 6, we're processing file %5)
|
306
|
-
|
307
|
-
# # in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
|
308
|
-
# #$stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
|
309
|
-
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
|
310
|
-
# $stderr.puts "-- print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if (($print_stats_and_md5_counter) % opts_ping_frequency == 1 )
|
302
|
+
opts_ping_frequency = opts.fetch :ping_frequency, $default_ping_frequency
|
311
303
|
|
312
|
-
|
304
|
+
puts "print_stats_and_md5: #{file}" if opts_verbose
|
313
305
|
stats = compute_stats_and_md5 file
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
306
|
+
|
307
|
+
mode = begin
|
308
|
+
format('%06o', stats[:mode])
|
309
|
+
rescue StandardError
|
310
|
+
:ERROR
|
311
|
+
end # .to_s.right(4) #=> "100644"
|
312
|
+
file_type = begin
|
313
|
+
stats[:stat_ftype][0]
|
314
|
+
rescue StandardError
|
315
|
+
'?'
|
316
|
+
end
|
319
317
|
file_type = 's' if File.symlink?(file)
|
320
318
|
content_type = `file --mime-type -b '#{file}' 2>/dev/null`.chomp # .split(': ',2)[1]
|
321
319
|
|
322
|
-
# colored_string = "[COL] #{red stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{white stats[:name]}"
|
323
|
-
# boring_string = "[B/W] #{ stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{ stats[:name]}"
|
324
|
-
# puts(opts_color ? colored_string : boring_string)
|
325
|
-
# puts "[OLDv1.1deprecated] #{maybecolored_md5} #{mode} #{file_type} #{maybecolored_size} #{stats[:stat_safebirthtime]} #{maybecolored_filename}"
|
326
320
|
print_colored_polymoprhic('file_v1.2', stats[:md5], mode, file_type, stats[:size], content_type,
|
327
321
|
stats[:stat_safebirthtime], stats[:name], opts)
|
328
322
|
end
|
329
323
|
|
330
|
-
def autowrite_to_dir_or_gcs(fs_type, path,
|
331
|
-
autowrite_version =
|
332
|
-
file_to_save =
|
333
|
-
path_to_save = path +
|
334
|
-
puts(red
|
335
|
-
puts(red
|
324
|
+
def autowrite_to_dir_or_gcs(fs_type, path, _opts = {})
|
325
|
+
autowrite_version = '1.1_28jun22'
|
326
|
+
file_to_save = 'stats-with-md5.log'
|
327
|
+
path_to_save = path + '/' + file_to_save
|
328
|
+
puts(red("TODO(ricc): write results [size=#{$autowrite_file.size}] in this Directory: #{path_to_save}")) if fs_type == :dir
|
329
|
+
puts(red("TODO(ricc): write results [size=#{$autowrite_file.size}] in this GCS Bucket or Dir: #{path_to_save}")) if fs_type == :gcs
|
336
330
|
|
337
331
|
##########################################################################
|
338
332
|
# do the thing: creates file locally and moves to GCS or Dir.
|
@@ -342,12 +336,8 @@ def autowrite_to_dir_or_gcs(fs_type, path, opts = {})
|
|
342
336
|
tmpfile.write($autowrite_file)
|
343
337
|
cmd = :no_cmd_yet
|
344
338
|
|
345
|
-
if fs_type == :gcs
|
346
|
-
|
347
|
-
end
|
348
|
-
if fs_type == :dir
|
349
|
-
cmd = "mv '#{tmpfile.path}' #{path}/#{file_to_save}"
|
350
|
-
end
|
339
|
+
cmd = "gsutil mv '#{tmpfile.path}' #{path}/#{file_to_save}" if fs_type == :gcs
|
340
|
+
cmd = "mv '#{tmpfile.path}' #{path}/#{file_to_save}" if fs_type == :dir
|
351
341
|
|
352
342
|
puts("TMP File is this big: #{azure tmpfile.size}")
|
353
343
|
puts("About to execute this: #{white cmd}")
|
@@ -366,28 +356,27 @@ def autowrite_to_dir_or_gcs(fs_type, path, opts = {})
|
|
366
356
|
#####################################
|
367
357
|
# finished, lets now clean up the file...
|
368
358
|
#####################################
|
369
|
-
reset_autowrite_file
|
370
|
-
|
359
|
+
reset_autowrite_file
|
360
|
+
:ok
|
371
361
|
end
|
372
362
|
|
363
|
+
def print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, opts = {})
|
364
|
+
puts "# [print_stats_and_md5_for_directory_from_cli] v#{$print_stats_and_md5_version}] DIR to explore (make sure you glob also): #{directory_to_explore_recursively}"
|
365
|
+
opts_autowrite = opts.fetch :autowrite, false
|
366
|
+
opts_verbose = opts.fetch(:verbose, $opts[:verbose])
|
367
|
+
opts_debug = opts.fetch :debug, $opts[:debug]
|
368
|
+
opts_color = opts.fetch :color, $opts[:color]
|
369
|
+
if opts_debug
|
370
|
+
puts "# DEB Options1 opts (FUNC) BAD: s #{opts}" # probably useless since this is invoked by CLI and has ful fledged options.
|
371
|
+
puts "# DEB Options2 $opts (ARGV) GOOD: #{$opts}" # This is what we ewant in this CLI. TODO(ricc): if you move this function to be used as function invert the priority...
|
372
|
+
puts("# DEB version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") # if ARGV.size > 0
|
373
|
+
end
|
373
374
|
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
opts_color = opts.fetch :color, $opts[:color]
|
380
|
-
if opts_debug
|
381
|
-
puts "# DEB Options1 opts (FUNC) BAD: s #{opts}" # probably useless since this is invoked by CLI and has ful fledged options.
|
382
|
-
puts "# DEB Options2 $opts (ARGV) GOOD: #{$opts}" # This is what we ewant in this CLI. TODO(ricc): if you move this function to be used as function invert the priority...
|
383
|
-
puts("# DEB version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") # if ARGV.size > 0
|
384
|
-
end
|
385
|
-
|
386
|
-
Dir.glob("#{(directory_to_explore_recursively)}/**/*") do |globbed_filename|
|
387
|
-
# Do work on files & directories ending in .rb
|
388
|
-
#puts "[deb] #{globbed_filename}"
|
389
|
-
print_stats_and_md5(globbed_filename, :color => opts_color, :verbose => opts_verbose, :autowrite => opts_autowrite)
|
390
|
-
end
|
375
|
+
Dir.glob("#{directory_to_explore_recursively}/**/*") do |globbed_filename|
|
376
|
+
# Do work on files & directories ending in .rb
|
377
|
+
# puts "[deb] #{globbed_filename}"
|
378
|
+
print_stats_and_md5(globbed_filename, color: opts_color, verbose: opts_verbose, autowrite: opts_autowrite)
|
379
|
+
end
|
391
380
|
|
392
381
|
autowrite_to_dir_or_gcs(:dir, directory_to_explore_recursively) if opts_autowrite
|
393
382
|
end
|
@@ -401,60 +390,41 @@ def real_program
|
|
401
390
|
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ARGV.size > 0
|
402
391
|
|
403
392
|
common_opts = {
|
404
|
-
:
|
405
|
-
:
|
406
|
-
:
|
407
|
-
:
|
393
|
+
max_files: $opts[:max_files],
|
394
|
+
color: $opts[:color],
|
395
|
+
verbose: $opts[:verbose],
|
396
|
+
autowrite: $opts[:autowrite]
|
408
397
|
}
|
409
398
|
|
410
399
|
if ARGV.size == 1
|
411
400
|
directory_to_explore_recursively = ARGV[0]
|
412
401
|
if smells_like_gcs?(directory_to_explore_recursively)
|
413
|
-
<<<<<<< HEAD
|
414
402
|
print_stats_and_md5_for_gcs(directory_to_explore_recursively, common_opts)
|
415
403
|
else # normal file..
|
416
|
-
|
404
|
+
print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, common_opts)
|
417
405
|
end
|
418
|
-
=======
|
419
|
-
print_stats_and_md5_for_gcs(directory_to_explore_recursively, common_opts)
|
420
|
-
else # normal file..
|
421
|
-
print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, common_opts)
|
422
|
-
end
|
423
|
-
>>>>>>> 97a64f1 (perfected CLI)
|
424
406
|
elsif ARGV.size > 1
|
425
|
-
deb
|
407
|
+
deb '2. I expect a lot of single files or directories:'
|
426
408
|
for arg in ARGV
|
427
|
-
<<<<<<< HEAD
|
428
409
|
if File.directory?(arg)
|
429
|
-
|
410
|
+
print_stats_and_md5_for_directory_from_cli(arg, common_opts)
|
430
411
|
else
|
431
412
|
print_stats_and_md5(arg, common_opts)
|
432
413
|
end
|
433
|
-
=======
|
434
|
-
if File.directory?(arg)
|
435
|
-
print_stats_and_md5_for_directory_from_cli(arg, common_opts )
|
436
|
-
else
|
437
|
-
print_stats_and_md5(arg, common_opts)
|
438
|
-
end
|
439
|
-
>>>>>>> 97a64f1 (perfected CLI)
|
440
414
|
end
|
441
415
|
else
|
442
|
-
puts
|
416
|
+
puts 'No args given. Exiting'
|
443
417
|
exit 41
|
444
418
|
end
|
445
419
|
tf = Time.now
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
#rounding to 3 decimals
|
450
|
-
approx_delta = (tf-t0).round(3)
|
451
|
-
puts "# [#{File.basename $0}] Time taken for processing #{stats_and_md5_number_of_files_processed} files: #{approx_delta} seconds"
|
452
|
-
>>>>>>> 97a64f1 (perfected CLI)
|
420
|
+
# rounding to 3 decimals
|
421
|
+
approx_delta = (tf - t0).round(3)
|
422
|
+
puts "# [#{File.basename $0}] Time taken for processing #{stats_and_md5_number_of_files_processed} files: #{approx_delta} seconds"
|
453
423
|
end
|
454
424
|
|
455
425
|
def main(filename)
|
456
426
|
deb "I'm called by #{white filename}"
|
457
|
-
deb
|
427
|
+
deb 'To remove this shit, just set $DEBUG=false :)'
|
458
428
|
init # Enable this to have command line parsing capabilities!
|
459
429
|
real_program
|
460
430
|
end
|