storazzo 0.4.10 → 0.5.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +5 -4
- data/Rakefile +8 -11
- data/VERSION +1 -1
- data/bin/hello-storazzo +0 -1
- data/bin/ricdisk-magic +50 -56
- data/bin/stats-with-md5 +251 -210
- data/bin/storazzo +32 -0
- data/bin/storazzo-util +1 -0
- data/lib/storazzo/colors.rb +43 -45
- data/lib/storazzo/common.rb +52 -46
- data/lib/storazzo/debug.rb +5 -8
- data/lib/storazzo/hashify.rb +44 -43
- data/lib/storazzo/main.rb +40 -41
- data/lib/storazzo/media/abstract_ric_disk.rb +162 -151
- data/lib/storazzo/media/gcs_bucket.rb +52 -51
- data/lib/storazzo/media/local_folder.rb +43 -44
- data/lib/storazzo/media/mount_point.rb +8 -9
- data/lib/storazzo/ric_disk.rb +153 -170
- data/lib/storazzo/ric_disk_config.rb +224 -219
- data/lib/storazzo/ric_disk_sample_config.rb +12 -12
- data/lib/storazzo/ric_disk_statsfile.rb +17 -15
- data/lib/storazzo/ric_disk_ugly.rb +35 -38
- data/lib/storazzo/version.rb +7 -7
- data/lib/storazzo.rb +36 -36
- data/storazzo.gemspec +26 -21
- data/test/media/test_abstract_ric_disk.rb +12 -16
- data/test/media/test_gcs_bucket.rb +44 -47
- data/test/media/test_local_folder.rb +134 -136
- data/test/media/test_mount_point.rb +18 -19
- data/test/test_ric_disk.rb +12 -14
- data/test/test_ric_disk_config.rb +20 -24
- data/test/test_ric_disk_stats_file.rb +13 -14
- data/test/test_storazzo.rb +26 -26
- data/var/test/disks/disk02-full/Rakefile +4 -4
- metadata +7 -3
data/bin/stats-with-md5
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
require 'digest/md5'
|
4
4
|
require 'digest'
|
5
5
|
require 'socket'
|
6
|
-
require 'optparse'
|
6
|
+
require 'optparse' # http://ruby.about.com/od/advancedruby/a/optionparser.htm
|
7
7
|
require 'date' # for DateTime
|
8
8
|
require 'tempfile'
|
9
9
|
|
@@ -30,36 +30,34 @@ DEFAULT_MAX_FILES = 'Infinite'
|
|
30
30
|
|
31
31
|
=end
|
32
32
|
|
33
|
-
|
34
|
-
|
35
|
-
def deb(s); puts "#DEB #{s}" if $DEBUG; end
|
33
|
+
def deb(s); puts "#DEB #{s}" if $DEBUG; end
|
36
34
|
# colors 16
|
37
|
-
def gray(s) "\033[1;30m#{s}\033[0m"
|
38
|
-
def green(s) "\033[1;32m#{s}\033[0m"
|
39
|
-
def red(s) "\033[1;31m#{s}\033[0m"
|
40
|
-
def yellow(s) "\033[1;33m#{s}\033[0m"
|
41
|
-
def blue(s)
|
42
|
-
def purple(s) "\033[1;35m#{s}\033[0m"
|
43
|
-
def azure(s) "\033[1;36m#{s}\033[0m"
|
44
|
-
def white(s) "\033[1;37m#{s}\033[0m"
|
35
|
+
def gray(s) "\033[1;30m#{s}\033[0m"; end
|
36
|
+
def green(s) "\033[1;32m#{s}\033[0m"; end
|
37
|
+
def red(s) "\033[1;31m#{s}\033[0m"; end
|
38
|
+
def yellow(s) "\033[1;33m#{s}\033[0m"; end
|
39
|
+
def blue(s) "\033[1;34m#{s}\033[0m"; end
|
40
|
+
def purple(s) "\033[1;35m#{s}\033[0m"; end
|
41
|
+
def azure(s) "\033[1;36m#{s}\033[0m"; end
|
42
|
+
def white(s) "\033[1;37m#{s}\033[0m"; end
|
45
43
|
|
46
44
|
# colors 64k
|
47
|
-
def orange(s)
|
45
|
+
def orange(s) "\033[38;5;208m#{s}\033[0m"; end
|
48
46
|
|
49
47
|
# Program constants, automagically picked up by RicLib
|
50
48
|
# More configuration could be written in:
|
51
49
|
# $GIC/etc/ricsvn/<FILENAME>.yml
|
52
50
|
# That would go into the variable '$prog_conf_d'
|
53
51
|
$myconf = {
|
54
|
-
|
55
|
-
|
52
|
+
:app_name => "AppName should be sth like #{$0}",
|
53
|
+
:description => "
|
56
54
|
This program gets stats from Ruby File.Stats library
|
57
55
|
plus computes MD5 or CRC32 of the file.
|
58
|
-
And bakes it in a way that can be easily parsed.
|
56
|
+
And bakes it in a way that can be easily parsed.
|
59
57
|
".strip.gsub(/^\s+/, "").gsub(/\s+$/, ""),
|
60
58
|
}
|
61
59
|
|
62
|
-
def usage(comment=nil)
|
60
|
+
def usage(comment = nil)
|
63
61
|
puts white($optparse.banner)
|
64
62
|
puts($optparse.summarize)
|
65
63
|
puts("Description: " + gray($myconf[:description]))
|
@@ -68,12 +66,12 @@ def usage(comment=nil)
|
|
68
66
|
end
|
69
67
|
|
70
68
|
def reset_autowrite_file()
|
71
|
-
|
72
|
-
|
73
|
-
end
|
69
|
+
$autowrite_file = "# #{File.basename $0} Autowrite v1.0 #{Time.now} reincarnation=#{$reset_autowrite_file_reincarnation}\n"
|
70
|
+
$reset_autowrite_file_reincarnation += 1 # ($reset_autowrite_file_reincarnation + 1) # rescue 1
|
71
|
+
end
|
74
72
|
|
75
73
|
# include it in main if you want a custome one
|
76
|
-
def init()
|
74
|
+
def init() # see lib_autoinit in lib/util.rb
|
77
75
|
$opts = {}
|
78
76
|
# setting defaults
|
79
77
|
$opts[:verbose] = false
|
@@ -87,26 +85,65 @@ def init() # see lib_autoinit in lib/util.rb
|
|
87
85
|
|
88
86
|
$optparse = OptionParser.new do |opts|
|
89
87
|
opts.banner = "#{$0} v.#{$PROG_VER}\n Usage: #{File.basename $0} [options] file1 file2 ..."
|
90
|
-
opts.on(
|
91
|
-
|
92
|
-
|
93
|
-
opts.on(
|
94
|
-
opts.on(
|
95
|
-
opts.on(
|
96
|
-
opts.on(
|
97
|
-
|
88
|
+
opts.on('-a', '--autowrite', 'automatically writes results to Root Folder (DFLT=false)') {
|
89
|
+
$opts[:autowrite] = true
|
90
|
+
}
|
91
|
+
opts.on('-c', '--no-color', 'disables color (DFLT=false, that is color enabled)') { $opts[:color] = false }
|
92
|
+
opts.on('-d', '--debug', 'enables debug (DFLT=false)') { $opts[:debug] = true; $DEBUG = true }
|
93
|
+
opts.on('-h', '--help', 'Display this screen') { usage }
|
94
|
+
opts.on('-m', '--max-files NUMBER', "sets max files per Dir to X (DFLT=#{DEFAULT_MAX_FILES})") { |nfiles|
|
95
|
+
$opts[:max_files] = nfiles.to_i
|
96
|
+
}
|
97
|
+
opts.on('-n', '--dryrun', "Don't really execute code") { $opts[:dryrun] = true }
|
98
|
+
opts.on('-l', '--logfile FILE', 'Write log to FILE') { |file| $opts[:logfile] = file }
|
99
|
+
opts.on('-v', '--verbose', 'Output more information') { $opts[:verbose] = true }
|
98
100
|
end
|
99
101
|
$optparse.parse!
|
100
102
|
end
|
101
103
|
|
102
104
|
def getSafeCreationTime(file)
|
103
|
-
|
105
|
+
File.ctime(file)
|
104
106
|
end
|
105
107
|
|
106
108
|
def compute_stats_and_md5(file)
|
107
|
-
|
108
|
-
|
109
|
-
|
109
|
+
ret = {}
|
110
|
+
ret[:name] = file
|
111
|
+
|
112
|
+
<<<<<<< HEAD
|
113
|
+
begin
|
114
|
+
stats = File.stat(file)
|
115
|
+
ret[:stats_object] = stats # TODO deprecate
|
116
|
+
deb("Stats methods: #{stats.methods.sort.join(', ')}")
|
117
|
+
deb(stats.ctime)
|
118
|
+
# puts(stats.birthtime rescue (stats.ctime))
|
119
|
+
# On Mac/Linux: see test/dumps/***.yaml
|
120
|
+
ret[:size] = stats.size
|
121
|
+
ret[:mode] = stats.mode
|
122
|
+
# datetimes
|
123
|
+
ret[:stat_safebirthtime] = getSafeCreationTime(file) # in Mac uses birthtime,but on Linux wont work
|
124
|
+
# defined?(stats.birthtime) ? # rescue stats.mtime # eg, 2022-03-05 21:47:51 +0100 on Mac (not implemented on my Linuix)#
|
125
|
+
# stats.birthtime : # works on Mac
|
126
|
+
# stats.ctime
|
127
|
+
ret[:stat_mtime] = stats.mtime # eg, 2022-03-05 21:47:51 +0100 Returns the modification time of stat.
|
128
|
+
ret[:stat_ctime] = stats.ctime # eg, 2022-03-05 21:47:51 +0100 Returns the change time for stat (that is, the time directory information about the file was changed, not the file itself). Note that on Windows (NTFS), returns creation time (birth time).
|
129
|
+
ret[:stat_atime] = stats.atime # eg, 2022-03-05 21:47:51 +0100 Last Access
|
130
|
+
ret[:stat_gid] = stats.gid # Group Id
|
131
|
+
ret[:stat_uid] = stats.uid # UUID
|
132
|
+
ret[:stat_ftype] = stats.ftype #
|
133
|
+
|
134
|
+
ret[:md5] = '______________NONE______________'
|
135
|
+
if stats.ftype != "directory"
|
136
|
+
file_content = File.read(file)
|
137
|
+
ret[:md5] = Digest::MD5.hexdigest(file_content) # rescue 'none ' #=> string with hexadecimal digits
|
138
|
+
end
|
139
|
+
rescue Errno::EISDIR => e
|
140
|
+
# puts "It's a dir, nothing I can do here except skipping the stuff"
|
141
|
+
ret[:md5] = "_________I'm a dir sorry________"
|
142
|
+
rescue Exception => e
|
143
|
+
ret[:error] = e
|
144
|
+
end
|
145
|
+
ret
|
146
|
+
=======
|
110
147
|
begin
|
111
148
|
stats = File.stat(file)
|
112
149
|
ret[:stats_object] = stats # TODO deprecate
|
@@ -140,196 +177,200 @@ def compute_stats_and_md5(file)
|
|
140
177
|
ret[:error] = e
|
141
178
|
end
|
142
179
|
ret
|
180
|
+
>>>>>>> 97a64f1 (perfected CLI)
|
143
181
|
end
|
144
182
|
|
145
183
|
$print_stats_and_md5_version = "1.1b_220805_F" # files
|
146
184
|
$print_stats_and_md5_counter = 0
|
147
|
-
$print_stats_and_md5_for_gcs_version = "1.1alpha_220628_G" #GCS bucket
|
185
|
+
$print_stats_and_md5_for_gcs_version = "1.1alpha_220628_G" # GCS bucket
|
148
186
|
|
149
187
|
def stats_and_md5_number_of_files_processed()
|
150
|
-
|
188
|
+
return $print_stats_and_md5_counter
|
151
189
|
end
|
152
190
|
|
153
191
|
def smells_like_gcs?(file)
|
154
|
-
|
192
|
+
file =~ /gs:\/\//
|
155
193
|
end
|
156
194
|
|
157
195
|
# You give me gs://bucket123/path/to/dir
|
158
|
-
def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts={})
|
159
|
-
# opts_color = opts.fetch :color, true#
|
160
|
-
# opts_verbose = opts.fetch :verbose, false
|
161
|
-
# opts_ping_frequency = opts.fetch :ping_frequency, 50
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
end
|
196
|
+
def print_stats_and_md5_for_gcs(mybucket_with_subdir, opts = {})
|
197
|
+
# opts_color = opts.fetch :color, true#
|
198
|
+
# opts_verbose = opts.fetch :verbose, false
|
199
|
+
# opts_ping_frequency = opts.fetch :ping_frequency, 50
|
200
|
+
opts_max_files = opts.fetch :max_files, nil # BigDecimal('Infinity')
|
201
|
+
opts_autowrite = opts.fetch :autowrite, false
|
202
|
+
|
203
|
+
raise "Wrong GCS Path: #{mybucket_with_subdir}" unless smells_like_gcs?(mybucket_with_subdir)
|
204
|
+
|
205
|
+
# puts "0. mybucket_with_subdir: #{mybucket_with_subdir}" # gs://mybucket123/path/to/dir
|
206
|
+
path_split = mybucket_with_subdir.split('/')
|
207
|
+
mybucket = path_split[0, 3].join('/') # gs://mybucket123/
|
208
|
+
gcs_subpath = path_split[3, 42].join('/') # path/to/dir
|
209
|
+
cleanedup_bucket = path_split[2] # mybucket123
|
210
|
+
# puts "1. mybucket: #{mybucket}"
|
211
|
+
# puts "2. gcs_subpath: #{gcs_subpath}"
|
212
|
+
# puts "3. cleanedup_bucket: #{cleanedup_bucket}"
|
213
|
+
|
214
|
+
puts("[print_stats_and_md5_for_gcs] version=#{$print_stats_and_md5_for_gcs_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on='#{Time.now}' bucket=#{mybucket} subpath=#{gcs_subpath}")
|
215
|
+
begin
|
216
|
+
require "google/cloud/storage"
|
217
|
+
project_id = `gcloud config get project 2>/dev/null`
|
218
|
+
deb "project_id: #{project_id}"
|
219
|
+
storage = Google::Cloud::Storage.new(project_id: project_id)
|
220
|
+
bucket = storage.bucket(cleanedup_bucket)
|
221
|
+
# grabs them ALL if nil, optherwise its an integer.
|
222
|
+
files = opts_max_files.nil? ?
|
223
|
+
bucket.files : # nil -> all of them (I've tried sth more idiomatic like -1 or nil or Infinite. they all failed.)
|
224
|
+
bucket.files.first(opts_max_files) # not nil -> first(N)
|
225
|
+
# puts(files)
|
226
|
+
deb("All Sorted Methods: #{files.first.methods.sort}")
|
227
|
+
files.each do |gcs_file|
|
228
|
+
md5 = Base64.decode64(gcs_file.md5).unpack('H*')[0] # md5 in base64 -> then de-binarizied: Base64.urlsafe_decode64(md5).unpack('H*')[0]
|
229
|
+
size = gcs_file.size # crc rescue :boh
|
230
|
+
time_created = gcs_file.created_at
|
231
|
+
mode = 'XXXXXX' # for compatbility with files
|
232
|
+
file_type = gcs_file.name.to_s =~ /\/$/ ? 'd' : 'f' # if ends with a slash its a DIR :P
|
233
|
+
# puts("[OLD_GCS_v11] #{md5} #{size}\t#{time_created} [#{gcs_file.content_type}]\t#{gcs_file.name}")
|
234
|
+
print_colored_polymoprhic('gcs_v1.2', md5, mode, file_type, size, gcs_file.content_type, time_created,
|
235
|
+
gcs_file.name, opts)
|
236
|
+
deb gcs_file.to_yaml
|
237
|
+
end
|
238
|
+
# puts("Hash Methods: #{files.first.methods.select{|x| x.match /hash/}}")
|
239
|
+
rescue LoadError # require Exception
|
240
|
+
puts 'Error with library #{$!}, maybe type: gem install google-cloud-storage'
|
241
|
+
rescue Exception # generic Exception
|
242
|
+
puts "print_stats_and_md5_for_gcs(): Generic Error: #{$!}"
|
243
|
+
exit 1
|
244
|
+
end
|
245
|
+
autowrite_to_dir_or_gcs(:gcs, mybucket_with_subdir) if opts_autowrite
|
246
|
+
end
|
207
247
|
|
208
248
|
# Generic polymorpghic coloring, can be GCS. Would have been nice to pass a Dict so easier to extend, but Ruby doesnt support doesblt Dict with magic args. Plus might be a GOOD
|
209
249
|
# thing its hard to change since it breaks Storazzo.
|
210
250
|
# TODO(ricc): make sure the dates are the same. Currently:
|
211
251
|
# GCS Date: 2021-12-20T12:26:13+00:00 DateTime
|
212
252
|
# File Date: 2022-05-30 11:55:42 +0200 Time
|
213
|
-
def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_type, creation_time, filename, opts={})
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
253
|
+
def print_colored_polymoprhic(entity_type, md5, mode, file_type, size, content_type, creation_time, filename, opts = {})
|
254
|
+
opts_color = opts.fetch :color, true
|
255
|
+
opts_verbose = opts.fetch :verbose, false
|
256
|
+
opts_ping_frequency = opts.fetch :ping_frequency, 50
|
257
|
+
opts_autowrite = opts.fetch :autowrite, false
|
258
|
+
|
259
|
+
# Increment counter across all!
|
260
|
+
$print_stats_and_md5_counter += 1 # global counter! I should increment at the END of fucntion.. but you never know if i exit wrongly so i do at beginning. so within this function the real #files is N-1 (i.e., if N is 6, we're processing file %5)
|
261
|
+
# in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
|
262
|
+
# $stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
|
263
|
+
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
|
264
|
+
$stderr.puts "-- print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if (($print_stats_and_md5_counter) % opts_ping_frequency == 1)
|
265
|
+
|
266
|
+
maybecolored_md5 = opts_color ? red(md5) : md5
|
267
|
+
maybecolored_filename = opts_color ? azure(filename) : filename
|
268
|
+
enlarged_size = sprintf("%7o", size.to_s) # or SIZE
|
269
|
+
maybecolored_size = opts_color ? yellow(enlarged_size) : enlarged_size
|
270
|
+
maybecolored_file_type = file_type
|
271
|
+
colored_content_type = opts_color ? white(content_type) : content_type
|
272
|
+
if opts_color
|
273
|
+
maybecolored_file_type = case file_type
|
274
|
+
when 'f' then green(file_type)
|
275
|
+
when 'd' then blue(file_type)
|
276
|
+
when 's' then azure(file_type)
|
277
|
+
else red(file_type)
|
278
|
+
end
|
279
|
+
end
|
280
|
+
standardized_creation_time = creation_time.is_a?(DateTime) ?
|
281
|
+
creation_time :
|
282
|
+
DateTime.parse(creation_time.to_s) # if you prefer to use Time since its already supported by Storazzo (but its ugly since it has SPACES! so hard to parse) use tt = Time.parse(d.to_s)
|
283
|
+
# as per https://stackoverflow.com/questions/279769/convert-to-from-datetime-and-time-in-ruby
|
284
|
+
# puts "#{maybecolored_md5} #{mode} #{maybecolored_file_type} #{maybecolored_size}\t#{creation_time}(DEB #{creation_time.class})(DEB2 #{DateTime.parse(creation_time.to_s)}) [#{colored_content_type}] #{maybecolored_filename}"
|
285
|
+
# this mightr be colored
|
286
|
+
str = "[#{entity_type}] #{maybecolored_md5} #{mode} #{maybecolored_file_type} #{standardized_creation_time} #{maybecolored_size} [#{colored_content_type}] #{maybecolored_filename}\n"
|
287
|
+
# this will NEVER be colored. WARNING, now you need to maintain TWO of these beasts :/
|
288
|
+
non_colored_str = "[#{entity_type}] #{md5} #{mode} #{file_type} #{standardized_creation_time} #{enlarged_size} [#{content_type}] #{filename}\n"
|
289
|
+
if (opts_autowrite)
|
290
|
+
# need to guarantee this is NOT colored. The easiest way to do it is TWICVE as expensive but... whatevs.
|
291
|
+
# TODO(ricc): fire me for this lazimness!
|
292
|
+
$autowrite_file += (non_colored_str)
|
293
|
+
end
|
294
|
+
print(str)
|
295
|
+
return str
|
256
296
|
end
|
257
297
|
|
258
|
-
def print_stats_and_md5(file, opts={})
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
298
|
+
def print_stats_and_md5(file, opts = {})
|
299
|
+
return print_stats_and_md5_for_gcs(file, opts) if smells_like_gcs?(file)
|
300
|
+
|
301
|
+
opts_color = opts.fetch :color, true
|
302
|
+
opts_verbose = opts.fetch :verbose, false
|
303
|
+
opts_ping_frequency = opts.fetch :ping_frequency, 50
|
304
|
+
|
305
|
+
# $print_stats_and_md5_counter += 1 # global counter! I should increment at the END of fucntion.. but you never know if i exit wrongly so i do at beginning. so within this function the real #files is N-1 (i.e., if N is 6, we're processing file %5)
|
306
|
+
|
307
|
+
# # in main i had to put 2 branched to invoke a single thing but if logic changes that sentence might be printed twice. Instead here, the BEGIN line could be nice to do here instead.
|
308
|
+
# #$stderr.puts("print_stats_and_md5() Fikst invocation! Consider moving the 2 things in main here :)") if ($print_stats_and_md5_counter == 1)
|
309
|
+
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ($print_stats_and_md5_counter == 1)
|
310
|
+
# $stderr.puts "-- print_stats_and_md5(v#{$print_stats_and_md5_version}): #{$print_stats_and_md5_counter - 1} files processed --" if (($print_stats_and_md5_counter) % opts_ping_frequency == 1 )
|
311
|
+
|
312
|
+
# puts "print_stats_and_md5: #{file}" if opts_verbose
|
313
|
+
stats = compute_stats_and_md5 file
|
314
|
+
# maybecolored_md5 = opts_color ? red(stats[:md5]) : stats[:md5]
|
315
|
+
# maybecolored_filename = opts_color ? azure(stats[:name]) : stats[:name]
|
316
|
+
# maybecolored_size = opts_color ? white(stats[:size]) : stats[:size]
|
317
|
+
mode = sprintf("%06o", stats[:mode]) rescue :ERROR # .to_s.right(4) #=> "100644"
|
318
|
+
file_type = stats[:stat_ftype][0] rescue '?'
|
319
|
+
file_type = 's' if File.symlink?(file)
|
320
|
+
content_type = `file --mime-type -b '#{file}' 2>/dev/null`.chomp # .split(': ',2)[1]
|
321
|
+
|
322
|
+
# colored_string = "[COL] #{red stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{white stats[:name]}"
|
323
|
+
# boring_string = "[B/W] #{ stats[:md5]} #{stats[:size]} #{stats[:stat_safebirthtime]} #{ stats[:name]}"
|
324
|
+
# puts(opts_color ? colored_string : boring_string)
|
325
|
+
# puts "[OLDv1.1deprecated] #{maybecolored_md5} #{mode} #{file_type} #{maybecolored_size} #{stats[:stat_safebirthtime]} #{maybecolored_filename}"
|
326
|
+
print_colored_polymoprhic('file_v1.2', stats[:md5], mode, file_type, stats[:size], content_type,
|
327
|
+
stats[:stat_safebirthtime], stats[:name], opts)
|
288
328
|
end
|
289
329
|
|
290
|
-
def autowrite_to_dir_or_gcs(fs_type, path, opts={})
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
330
|
+
def autowrite_to_dir_or_gcs(fs_type, path, opts = {})
|
331
|
+
autowrite_version = "1.1_28jun22"
|
332
|
+
file_to_save = "stats-with-md5.log"
|
333
|
+
path_to_save = path + "/" + file_to_save
|
334
|
+
puts(red "TODO(ricc): write results [size=#{$autowrite_file.size}] in this Directory: #{path_to_save}") if fs_type == :dir
|
335
|
+
puts(red "TODO(ricc): write results [size=#{$autowrite_file.size}] in this GCS Bucket or Dir: #{path_to_save}") if fs_type == :gcs
|
336
|
+
|
337
|
+
##########################################################################
|
338
|
+
# do the thing: creates file locally and moves to GCS or Dir.
|
339
|
+
##########################################################################
|
340
|
+
$autowrite_file += "# [autowrite_to_dir_or_gcs v#{autowrite_version}]: about to write it now. #{Time.now}"
|
341
|
+
tmpfile = Tempfile.new(path_to_save)
|
342
|
+
tmpfile.write($autowrite_file)
|
343
|
+
cmd = :no_cmd_yet
|
344
|
+
|
345
|
+
if fs_type == :gcs
|
346
|
+
cmd = "gsutil mv '#{tmpfile.path}' #{path}/#{file_to_save}"
|
347
|
+
end
|
348
|
+
if fs_type == :dir
|
349
|
+
cmd = "mv '#{tmpfile.path}' #{path}/#{file_to_save}"
|
350
|
+
end
|
311
351
|
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
352
|
+
puts("TMP File is this big: #{azure tmpfile.size}")
|
353
|
+
puts("About to execute this: #{white cmd}")
|
354
|
+
ret = `#{cmd}`
|
355
|
+
if $?.exitstatus != 0
|
356
|
+
puts "Error to execute: #{red cmd}. Exit: #{red $?.exitstatus}"
|
357
|
+
exit 53
|
358
|
+
end
|
359
|
+
puts "Command returned: '#{$?.exitstatus}'" # - #{$?.class}"
|
360
|
+
# wow: fork { exec("ls") }
|
361
|
+
|
362
|
+
# removes the tmpfile :)
|
363
|
+
tmpfile.close
|
364
|
+
tmpfile.unlink
|
365
|
+
|
366
|
+
#####################################
|
367
|
+
# finished, lets now clean up the file...
|
368
|
+
#####################################
|
369
|
+
reset_autowrite_file()
|
370
|
+
return :ok
|
331
371
|
end
|
332
372
|
|
373
|
+
|
333
374
|
def print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, opts={})
|
334
375
|
puts "# [print_stats_and_md5_for_directory_from_cli] v#{$print_stats_and_md5_version}] DIR to explore (make sure you glob also): #{directory_to_explore_recursively }"
|
335
376
|
opts_autowrite = opts.fetch :autowrite, false
|
@@ -348,33 +389,33 @@ def print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively,
|
|
348
389
|
print_stats_and_md5(globbed_filename, :color => opts_color, :verbose => opts_verbose, :autowrite => opts_autowrite)
|
349
390
|
end
|
350
391
|
|
351
|
-
|
352
|
-
end
|
392
|
+
autowrite_to_dir_or_gcs(:dir, directory_to_explore_recursively) if opts_autowrite
|
393
|
+
end
|
353
394
|
|
354
395
|
def real_program
|
355
396
|
t0 = Time.now
|
356
397
|
deb "+ Options are: #{gray $opts}"
|
357
398
|
deb "+ Depured args: #{azure ARGV}"
|
358
399
|
# Your code goes here...
|
359
|
-
#puts "Description: '''#{white $myconf[:description] }'''"
|
360
|
-
#puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ARGV.size > 0
|
400
|
+
# puts "Description: '''#{white $myconf[:description] }'''"
|
401
|
+
# puts("[print_stats_and_md5] version=#{$print_stats_and_md5_version} host=#{Socket.gethostname}(#{`uname`.chomp}) created_on=#{Time.now}") if ARGV.size > 0
|
361
402
|
|
362
403
|
common_opts = {
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
|
404
|
+
:max_files => $opts[:max_files],
|
405
|
+
:color => $opts[:color],
|
406
|
+
:verbose => $opts[:verbose],
|
407
|
+
:autowrite => $opts[:autowrite],
|
367
408
|
}
|
368
409
|
|
369
410
|
if ARGV.size == 1
|
370
|
-
|
411
|
+
directory_to_explore_recursively = ARGV[0]
|
371
412
|
if smells_like_gcs?(directory_to_explore_recursively)
|
372
413
|
print_stats_and_md5_for_gcs(directory_to_explore_recursively, common_opts)
|
373
414
|
else # normal file..
|
374
415
|
print_stats_and_md5_for_directory_from_cli(directory_to_explore_recursively, common_opts)
|
375
416
|
end
|
376
417
|
elsif ARGV.size > 1
|
377
|
-
|
418
|
+
deb "2. I expect a lot of single files or directories:"
|
378
419
|
for arg in ARGV
|
379
420
|
if File.directory?(arg)
|
380
421
|
print_stats_and_md5_for_directory_from_cli(arg, common_opts )
|
@@ -383,8 +424,8 @@ def real_program
|
|
383
424
|
end
|
384
425
|
end
|
385
426
|
else
|
386
|
-
|
387
|
-
|
427
|
+
puts "No args given. Exiting"
|
428
|
+
exit 41
|
388
429
|
end
|
389
430
|
tf = Time.now
|
390
431
|
#rounding to 3 decimals
|
data/bin/storazzo
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'storazzo'
|
4
|
+
|
5
|
+
|
6
|
+
if RUBY_VERSION.split('.')[0] == 1
|
7
|
+
puts "Refusing to launch a script form Ruby 1. Sorry Ric, its 2020 damn it!"
|
8
|
+
exit 2020
|
9
|
+
end
|
10
|
+
|
11
|
+
$PROG_VER = '0.1'
|
12
|
+
$DEBUG = ENV['DEBUG'] == 'true' # (true/false)
|
13
|
+
|
14
|
+
PROG_HISTORY = <<-BIG_LONG_MULTILINE
|
15
|
+
2022-0806 v0.1 First stesure
|
16
|
+
BIG_LONG_MULTILINE
|
17
|
+
|
18
|
+
|
19
|
+
$description = <<DESCRIPTION_HEREFILE.gsub(/\s+/, " ").strip
|
20
|
+
This util shall expose all Storazzo aPIs via CLI: show-local-dirs, ...
|
21
|
+
show GCS buckets, ...
|
22
|
+
|
23
|
+
It's also my dream to add some nice icons which are joy to the eye, like
|
24
|
+
some [ ] [x] before the list to show if its agood or bad directory and type also
|
25
|
+
DESCRIPTION_HEREFILE
|
26
|
+
# >> "SELECT * FROM users ORDER BY users.id DESC"
|
27
|
+
|
28
|
+
Storazzo::Main.say_hi("ARGV: #{ARGV}")
|
29
|
+
Storazzo::Main.say_hi("Note: if the version of this storazzo is behind local version youre probably using a old library.
|
30
|
+
I still need to learn how to call the binary with local/current lib/: bundle exec stiKazzi?")
|
31
|
+
Storazzo::Main.say_hi($description)
|
32
|
+
|
data/bin/storazzo-util
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
bin/storazzo
|