storazzo 0.4.10 → 0.5.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +5 -4
- data/Rakefile +8 -11
- data/VERSION +1 -1
- data/bin/hello-storazzo +0 -1
- data/bin/ricdisk-magic +50 -56
- data/bin/stats-with-md5 +251 -210
- data/bin/storazzo +32 -0
- data/bin/storazzo-util +1 -0
- data/lib/storazzo/colors.rb +43 -45
- data/lib/storazzo/common.rb +52 -46
- data/lib/storazzo/debug.rb +5 -8
- data/lib/storazzo/hashify.rb +44 -43
- data/lib/storazzo/main.rb +40 -41
- data/lib/storazzo/media/abstract_ric_disk.rb +162 -151
- data/lib/storazzo/media/gcs_bucket.rb +52 -51
- data/lib/storazzo/media/local_folder.rb +43 -44
- data/lib/storazzo/media/mount_point.rb +8 -9
- data/lib/storazzo/ric_disk.rb +153 -170
- data/lib/storazzo/ric_disk_config.rb +224 -219
- data/lib/storazzo/ric_disk_sample_config.rb +12 -12
- data/lib/storazzo/ric_disk_statsfile.rb +17 -15
- data/lib/storazzo/ric_disk_ugly.rb +35 -38
- data/lib/storazzo/version.rb +7 -7
- data/lib/storazzo.rb +36 -36
- data/storazzo.gemspec +26 -21
- data/test/media/test_abstract_ric_disk.rb +12 -16
- data/test/media/test_gcs_bucket.rb +44 -47
- data/test/media/test_local_folder.rb +134 -136
- data/test/media/test_mount_point.rb +18 -19
- data/test/test_ric_disk.rb +12 -14
- data/test/test_ric_disk_config.rb +20 -24
- data/test/test_ric_disk_stats_file.rb +13 -14
- data/test/test_storazzo.rb +26 -26
- data/var/test/disks/disk02-full/Rakefile +4 -4
- metadata +7 -3
data/lib/storazzo/ric_disk.rb
CHANGED
@@ -4,54 +4,52 @@
|
|
4
4
|
require 'digest'
|
5
5
|
|
6
6
|
module Storazzo
|
7
|
-
class Storazzo::RicDisk
|
8
|
-
|
7
|
+
class Storazzo::RicDisk
|
9
8
|
include Hashify
|
10
|
-
include Storazzo::Common
|
11
|
-
extend Storazzo::Common
|
9
|
+
include Storazzo::Common
|
10
|
+
extend Storazzo::Common
|
12
11
|
extend Storazzo::Colors
|
13
12
|
require 'socket'
|
14
13
|
|
15
|
-
|
16
|
-
## Instance variables
|
14
|
+
## Instance variables
|
17
15
|
|
18
16
|
# in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
|
19
17
|
# These are the files I do accept.
|
20
|
-
ConfigFiles = %W{
|
21
|
-
DefaultConfigFile = "storazzo.yaml" # .ricdisk }
|
18
|
+
ConfigFiles = %W{ricdisk.yaml .ricdisk storazzo.yaml}
|
19
|
+
DefaultConfigFile = "storazzo.yaml" # .ricdisk }
|
22
20
|
RicdiskVersion = '2.1'
|
23
21
|
RicdiskHistory = [
|
24
22
|
'2022-07-29 2.1 Added timestamp',
|
25
23
|
'2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...',
|
26
|
-
|
24
|
+
]
|
27
25
|
DefaultGemfileTestDiskFolder = Storazzo.root + "/var/test/disks/" # was: @@default_gemfile_test_disks_folder
|
28
26
|
# Immutable
|
29
|
-
DefaultMediaFolders = %w{
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
36
|
-
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
27
|
+
DefaultMediaFolders = %w{
|
28
|
+
/Volumes/
|
29
|
+
/mnt/
|
30
|
+
}.append(DefaultGemfileTestDiskFolder).append("/media/#{ENV["USER"]}/")
|
31
|
+
|
32
|
+
# # todo substitute with protobuf..
|
33
|
+
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
34
|
+
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
37
35
|
:unique_hash # new 202207
|
38
36
|
|
37
|
+
################################
|
38
|
+
## INSTANCE methods
|
39
|
+
################################
|
39
40
|
|
40
|
-
|
41
|
-
## INSTANCE methods
|
42
|
-
################################
|
43
|
-
|
44
|
-
def initialize_old_way(path, opts={})
|
41
|
+
def initialize_old_way(path, opts = {})
|
45
42
|
raise "Now I dont want a string in input, I want an OBJECT < Storazzo::Media::AbstractRicDisk"
|
46
43
|
end
|
47
44
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
45
|
+
def initialize(ric_disk_object, opts = {})
|
46
|
+
verbose = opts.fetch :verbose, true
|
47
|
+
pverbose verbose,
|
48
|
+
"This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
|
52
49
|
raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
|
50
|
+
|
53
51
|
# ok back to business, now path is a String :)
|
54
|
-
path =
|
52
|
+
path = ric_disk_object.path
|
55
53
|
deb "RicDisk initialize.. path=#{path}"
|
56
54
|
deb "RicDisk initialize.. path=#{path}"
|
57
55
|
@local_mountpoint = File.expand_path(path)
|
@@ -62,16 +60,16 @@ module Storazzo
|
|
62
60
|
@ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
|
63
61
|
@label = path.split("/").last
|
64
62
|
@name = path.split("/").last
|
65
|
-
|
66
|
-
|
63
|
+
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
64
|
+
# @wr = writeable?
|
67
65
|
@tags = ['ricdisk', 'storazzo']
|
68
|
-
@size = RicDisk._compute_size_could_take_long(path)
|
66
|
+
@size = RicDisk._compute_size_could_take_long(path)
|
69
67
|
@unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
70
68
|
@computation_hostname = Socket.gethostname
|
71
69
|
@created_at = Time.now
|
72
70
|
|
73
71
|
@ricdisk_file_empty = ricdisk_file_empty?
|
74
|
-
|
72
|
+
|
75
73
|
# @config = RicDiskConfig.instance.get_config
|
76
74
|
# #puts @config if @config
|
77
75
|
# find_info_from_mount(path)
|
@@ -89,7 +87,6 @@ module Storazzo
|
|
89
87
|
not ricdisk_file.nil?
|
90
88
|
end
|
91
89
|
|
92
|
-
|
93
90
|
def analyze_local_system()
|
94
91
|
puts "TODO This should analyzze the WHOLE system. TODO(ricc): move to another object which has to do with the system/computer."
|
95
92
|
puts "1. Interesting Mounts: #{green interesting_mount_points}"
|
@@ -98,45 +95,47 @@ module Storazzo
|
|
98
95
|
# find_info_from_df()
|
99
96
|
end
|
100
97
|
|
101
|
-
def path
|
98
|
+
def path
|
102
99
|
local_mountpoint
|
103
100
|
end
|
104
101
|
|
105
|
-
def to_s
|
102
|
+
def to_s
|
106
103
|
"RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version}, ard=#{@ard})"
|
107
104
|
end
|
108
105
|
|
109
106
|
# could take long..
|
110
|
-
# def size
|
107
|
+
# def size
|
111
108
|
# `du -s '#{path}'`.split(/\s/)[0]
|
112
109
|
# end
|
113
|
-
def self._compute_size_could_take_long(my_path)
|
110
|
+
def self._compute_size_could_take_long(my_path)
|
114
111
|
deb "Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds."
|
115
112
|
puts azure('could take long. Please take precautions like forking with timeout of 5sec')
|
116
113
|
`du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
|
117
114
|
end
|
118
115
|
|
119
|
-
def writeable?()
|
120
|
-
#memoize
|
121
|
-
return @wr unless @wr.nil?
|
116
|
+
def writeable?()
|
117
|
+
# memoize
|
118
|
+
return @wr unless @wr.nil?
|
119
|
+
|
122
120
|
# NOW: CALCULATE it
|
123
121
|
# Now I can do ONCE an EXPENSIVE calculation
|
124
122
|
puts yellow("[RicDisk.writeable] TODO(ricc): Do expensive calculation if this FS is writeable: #{path} and write/memoize it on @wr once and for all")
|
125
123
|
puts yellow("[RicDisk.writeable] I have a feeling this should be delegated to praecipuus Storazzo::Media::Object we refer to (WR is different on GCS vs Local):") # infinite loop dammit #{self.to_verbose_s}")
|
126
|
-
puts("Dir: #{
|
124
|
+
puts("Dir: #{azure path}")
|
127
125
|
puts("absolute_path: #{azure absolute_path}")
|
128
126
|
puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
|
129
127
|
bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
|
130
128
|
puts("bash_output: #{azure bash_output}")
|
131
|
-
|
129
|
+
# @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
|
132
130
|
raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
|
131
|
+
|
133
132
|
@wr = File.writable?(absolute_path) # rescue false
|
134
133
|
return @wr
|
135
|
-
|
136
|
-
#false
|
134
|
+
# :boh_todo_fix_me_and_compute
|
135
|
+
# false
|
137
136
|
end
|
138
137
|
|
139
|
-
def to_verbose_s
|
138
|
+
def to_verbose_s
|
140
139
|
h = {}
|
141
140
|
h[:to_s] = self.to_s
|
142
141
|
h[:wr] = self.wr
|
@@ -146,60 +145,58 @@ module Storazzo
|
|
146
145
|
return h
|
147
146
|
end
|
148
147
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
148
|
+
################################
|
149
|
+
## CLASS methods
|
150
|
+
################################
|
153
151
|
|
154
152
|
# All places where to find for something :)
|
155
153
|
def self.default_media_folders
|
156
154
|
DefaultMediaFolders # was DEFAULT_MEDIA_FOLDERS
|
157
155
|
end
|
158
156
|
|
159
|
-
|
160
157
|
def self.test # _localgem_disks
|
161
|
-
d = RicDisk.new(
|
158
|
+
d = RicDisk.new(DefaultGemfileTestDiskFolder)
|
162
159
|
puts (d)
|
163
160
|
puts "do something with it: #{d}"
|
164
|
-
#d.find_active_dirs()
|
161
|
+
# d.find_active_dirs()
|
165
162
|
end
|
166
163
|
|
167
164
|
def absolute_path
|
168
|
-
|
165
|
+
# @local_mountpoint + "/" + @ricdisk_file
|
169
166
|
"#{local_mountpoint}/#{ricdisk_file}"
|
170
167
|
end
|
171
|
-
|
172
|
-
def self.find_active_dirs(base_dirs=nil, also_mountpoints=true)
|
168
|
+
|
169
|
+
def self.find_active_dirs(base_dirs = nil, also_mountpoints = true)
|
173
170
|
if base_dirs.nil?
|
174
|
-
base_dirs = default_media_folders
|
171
|
+
base_dirs = default_media_folders
|
175
172
|
puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
|
176
173
|
end
|
177
174
|
active_dirs = []
|
178
|
-
base_dirs.each do |ugly_dir|
|
175
|
+
base_dirs.each do |ugly_dir|
|
179
176
|
# https://stackoverflow.com/questions/1899072/getting-a-list-of-folders-in-a-directory#:~:text=Dir.chdir(%27/destination_directory%27)%0ADir.glob(%27*%27).select%20%7B%7Cf%7C%20File.directory%3F%20f%7D
|
180
177
|
dir = File.expand_path(ugly_dir)
|
181
178
|
begin
|
182
|
-
x=[]
|
183
|
-
|
179
|
+
x = []
|
180
|
+
# puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
|
184
181
|
unless Dir.exists?(dir)
|
185
182
|
deb "Dir doesnt exist, skipping: #{dir}"
|
186
|
-
next
|
183
|
+
next
|
187
184
|
end
|
188
185
|
Dir.chdir(dir)
|
189
|
-
x = Dir.glob('*').select {|f| File.directory? f}
|
190
|
-
subdirs = x.map{|subdir|
|
191
|
-
subdirs.each{|subdir|
|
186
|
+
x = Dir.glob('*').select { |f| File.directory? f }
|
187
|
+
subdirs = x.map { |subdir| "#{dir}#{subdir}" }
|
188
|
+
subdirs.each { |subdir|
|
192
189
|
deb "Subdir: #{subdir}"
|
193
190
|
puts `ls -al "#{subdir}"` # TODO refactor in exec
|
194
191
|
active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
|
195
192
|
}
|
196
|
-
#puts(white x)
|
193
|
+
# puts(white x)
|
197
194
|
rescue Exception => e # optionally: `rescue Exception => ex`
|
198
195
|
puts "Exception: '#{e}'"
|
199
196
|
ensure # will always get executed
|
200
|
-
#deb 'Always gets executed.'
|
201
|
-
#x = []
|
202
|
-
end
|
197
|
+
# deb 'Always gets executed.'
|
198
|
+
# x = []
|
199
|
+
end
|
203
200
|
end
|
204
201
|
end
|
205
202
|
|
@@ -210,16 +207,15 @@ module Storazzo
|
|
210
207
|
end
|
211
208
|
deb "[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once"
|
212
209
|
ConfigFiles.each do |papable_config_filename|
|
213
|
-
#return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
214
|
-
#return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
210
|
+
# return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
211
|
+
# return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
215
212
|
return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
216
213
|
end
|
217
214
|
deb "File not found! Neither #{ConfigFiles} exist.."
|
218
|
-
# return nil
|
215
|
+
# return nil
|
219
216
|
return DefaultConfigFile
|
220
217
|
end
|
221
218
|
|
222
|
-
|
223
219
|
# def self.compute_ricdisk_file_by_path_once(path)
|
224
220
|
# # unless @ricdisk_file.nil?
|
225
221
|
# # deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
|
@@ -234,7 +230,6 @@ module Storazzo
|
|
234
230
|
# return nil
|
235
231
|
# end
|
236
232
|
|
237
|
-
|
238
233
|
# # new
|
239
234
|
# def self.get_ricdisk_file_obsolete(path)
|
240
235
|
# if @ricdisk_file
|
@@ -250,9 +245,8 @@ module Storazzo
|
|
250
245
|
# return nil
|
251
246
|
# end
|
252
247
|
|
253
|
-
|
254
|
-
|
255
|
-
#https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
248
|
+
def self.interesting_mount_points(opts = {})
|
249
|
+
# https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
256
250
|
`mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
|
257
251
|
end
|
258
252
|
|
@@ -262,23 +256,23 @@ module Storazzo
|
|
262
256
|
deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{(Time.now - creation_time)} seconds ago")
|
263
257
|
(Time.now - creation_time).to_i
|
264
258
|
end
|
259
|
+
|
265
260
|
# maybe move to a RiccFile class? Maybe even INHERIT from FILE?
|
266
261
|
def obsolescence_days(file_path)
|
267
262
|
return obsolescence_seconds(file_path) / 86400
|
268
263
|
end
|
269
264
|
|
270
|
-
|
271
265
|
# FORMER SBRODOLA, now write_config_yaml_to_disk
|
272
|
-
#def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
273
|
-
def write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
266
|
+
# def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
267
|
+
def write_config_yaml_to_disk(subdir, opts = {}) # sbrodola_ricdisk(subdir)
|
274
268
|
# given a path, if .ricdisk exists i do stuff with it..
|
275
269
|
disk_info = nil
|
276
270
|
unless ok_dir? # self.ok_dir?(subdir)
|
277
271
|
warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
|
278
|
-
return
|
272
|
+
return
|
279
273
|
end
|
280
274
|
ConfigFiles.each do |papable_configfile_name|
|
281
|
-
if File.exists?(
|
275
|
+
if File.exists?("#{subdir}/#{papable_configfile_name}") and File.empty?("#{subdir}/#{papable_configfile_name}")
|
282
276
|
deb("Interesting. Empty file '#{papable_configfile_name}'! Now I write YAML with it.")
|
283
277
|
disk_info = RicDisk.new(subdir, papable_configfile_name)
|
284
278
|
end
|
@@ -314,8 +308,8 @@ module Storazzo
|
|
314
308
|
puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
|
315
309
|
end
|
316
310
|
|
317
|
-
#disk_info.absolute_path
|
318
|
-
#if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
311
|
+
# disk_info.absolute_path
|
312
|
+
# if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
319
313
|
# if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
|
320
314
|
# puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
|
321
315
|
# puts(white `cat "#{disk_info.absolute_path}"`)
|
@@ -327,42 +321,45 @@ module Storazzo
|
|
327
321
|
|
328
322
|
# TODO obsolete this as i should NOT be calling it from clas, but from method.
|
329
323
|
def self.ok_dir?(subdir)
|
330
|
-
File.exists?(
|
324
|
+
File.exists?("#{subdir}/.ricdisk") or File.exists?("#{subdir}/.ricdisk.yaml")
|
331
325
|
end
|
332
326
|
|
333
|
-
def compute_stats_files(opts={})
|
334
|
-
#Storazzo::RicDisk.calculate_stats_files(path, opts)
|
327
|
+
def compute_stats_files(opts = {})
|
328
|
+
# Storazzo::RicDisk.calculate_stats_files(path, opts)
|
335
329
|
opts_upload_to_gcs = opts.fetch :upload_to_gcs, false
|
336
330
|
opts_force_rewrite = opts.fetch :force, false
|
337
331
|
opts_stats_file = opts.fetch :stats_file, "ricdisk_stats_v11.rds" # default. TODO point to proper..
|
338
332
|
dir = path
|
339
333
|
puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy. stats_file=#{opts_stats_file} dir=#{dir}")
|
340
|
-
|
334
|
+
|
341
335
|
full_file_path = "#{dir}/#{opts_stats_file}"
|
342
336
|
deb "This refactor is for another day. Actually no, TODAY "
|
343
|
-
pverbose true,
|
337
|
+
pverbose true,
|
338
|
+
"TODO(ricc): you should compute more SMARTLY the full_file_path (#{full_file_path}): if its R/O it should be elsewhere.."
|
344
339
|
puts azure("- full_file_path: #{full_file_path}")
|
345
340
|
puts azure("- writeable?: #{writeable?}")
|
346
|
-
|
341
|
+
|
347
342
|
puts("compute_stats_files(#{white dir}): #{white full_file_path}")
|
348
343
|
deb "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
|
349
344
|
raise "Directory doesnt exist: #{dir}" unless File.directory?(dir)
|
345
|
+
|
350
346
|
Dir.chdir(dir)
|
351
347
|
puts azure `ls` # im curious
|
352
348
|
if File.exists?(full_file_path)
|
353
349
|
if opts_force_rewrite
|
354
|
-
#raise "TODO implement file exists and FORCE enabled"
|
350
|
+
# raise "TODO implement file exists and FORCE enabled"
|
355
351
|
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ReWrite enabled")
|
356
352
|
else # File.exists?(full_file_path) and (opts_force)
|
357
353
|
puts "File '#{opts_stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
|
358
354
|
# TODO check for file time...
|
359
|
-
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp
|
360
|
-
if obsolescence_days(full_file_path) > 7
|
361
|
-
#puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
355
|
+
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp}. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
|
356
|
+
if obsolescence_days(full_file_path) > 7
|
357
|
+
# puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
362
358
|
puts yellow("*** ACHTUNG *** FIle is pretty old. I'll force a rewrite")
|
363
|
-
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
|
359
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
|
360
|
+
"File older than 7 days. Indeed: #{obsolescence_days(full_file_path)}")
|
364
361
|
end
|
365
|
-
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
362
|
+
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
366
363
|
end
|
367
364
|
else
|
368
365
|
deb("File doesnt exist..")
|
@@ -370,10 +367,10 @@ module Storazzo
|
|
370
367
|
end
|
371
368
|
end
|
372
369
|
|
373
|
-
def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts={})
|
370
|
+
def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts = {})
|
374
371
|
max_lines = opts.fetch :max_lines, 42 # todo move to nil or -1
|
375
|
-
#full_file_path = "#{dir}/#{stats_file}"
|
376
|
-
puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
|
372
|
+
# full_file_path = "#{dir}/#{stats_file}"
|
373
|
+
puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
|
377
374
|
if max_lines < 0 # infinite
|
378
375
|
command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
379
376
|
else
|
@@ -383,7 +380,7 @@ module Storazzo
|
|
383
380
|
# xargs -r0 myscript.sh
|
384
381
|
if mac?
|
385
382
|
puts red("Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)")
|
386
|
-
spannometric_lines = (max_lines/10)
|
383
|
+
spannometric_lines = (max_lines / 10)
|
387
384
|
command = "find . -print0 | head -n '#{spannometric_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
388
385
|
else
|
389
386
|
command = "find . -print0 | head -z -n '#{max_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
@@ -393,79 +390,65 @@ module Storazzo
|
|
393
390
|
ret = backquote_execute(command)
|
394
391
|
puts "Done. #{ret.split("\n").count} files processed."
|
395
392
|
end
|
396
|
-
|
397
|
-
|
398
393
|
|
399
|
-
def self.calculate_stats_files_DUPLICATE_STATIC(dir, opts={})
|
394
|
+
def self.calculate_stats_files_DUPLICATE_STATIC(dir, opts = {})
|
400
395
|
raise "Please use object instead. If you cant, please move the object code to STATIC and dedupe code!"
|
401
|
-
end
|
402
|
-
|
403
|
-
|
404
|
-
# if also_mountpoints
|
405
|
-
# =begin
|
406
|
-
# Example output from mount:
|
407
|
-
|
408
|
-
# devfs on /dev (devfs, local, nobrowse)
|
409
|
-
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
410
|
-
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
411
|
-
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
412
|
-
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
413
|
-
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
414
|
-
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
415
|
-
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
416
|
-
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
417
|
-
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
418
|
-
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
419
|
-
# =end
|
420
|
-
# # add directories from current mountpoints...
|
421
|
-
# mount_table_lines = interesting_mount_points()
|
422
|
-
# mount_table_lines.each{|line|
|
423
|
-
# next if line =~ /^map /
|
424
|
-
# dev, on, path, mode = line.split(/ /)
|
425
|
-
# #puts line
|
426
|
-
# #deb yellow(path)
|
427
|
-
# active_dirs << path if self.ok_dir?(path)
|
428
|
-
# }
|
429
|
-
# end
|
430
|
-
# active_dirs.uniq!
|
431
|
-
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
432
|
-
# return active_dirs
|
433
|
-
# end
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
end #/Class
|
438
|
-
end #/Module
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
def backquote_execute(cmd, opts={})
|
449
|
-
dryrun = opts.fetch :dryrun, false
|
450
|
-
# executed a command wrapped by dryrun though
|
451
|
-
return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
|
452
|
-
`#{cmd}`
|
453
396
|
end
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
end
|
471
|
-
|
397
|
+
|
398
|
+
# if also_mountpoints
|
399
|
+
# =begin
|
400
|
+
# Example output from mount:
|
401
|
+
|
402
|
+
# devfs on /dev (devfs, local, nobrowse)
|
403
|
+
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
404
|
+
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
405
|
+
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
406
|
+
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
407
|
+
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
408
|
+
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
409
|
+
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
410
|
+
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
411
|
+
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
412
|
+
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
413
|
+
# =end
|
414
|
+
# # add directories from current mountpoints...
|
415
|
+
# mount_table_lines = interesting_mount_points()
|
416
|
+
# mount_table_lines.each{|line|
|
417
|
+
# next if line =~ /^map /
|
418
|
+
# dev, on, path, mode = line.split(/ /)
|
419
|
+
# #puts line
|
420
|
+
# #deb yellow(path)
|
421
|
+
# active_dirs << path if self.ok_dir?(path)
|
422
|
+
# }
|
423
|
+
# end
|
424
|
+
# active_dirs.uniq!
|
425
|
+
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
426
|
+
# return active_dirs
|
427
|
+
# end
|
428
|
+
end # /Class
|
429
|
+
end # /Module
|
430
|
+
|
431
|
+
def backquote_execute(cmd, opts = {})
|
432
|
+
dryrun = opts.fetch :dryrun, false
|
433
|
+
# executed a command wrapped by dryrun though
|
434
|
+
return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
|
435
|
+
|
436
|
+
`#{cmd}`
|
437
|
+
end
|
438
|
+
|
439
|
+
def upload_to_gcs(file, opts = {})
|
440
|
+
deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
|
441
|
+
mount_name = file.split('/')[-2]
|
442
|
+
filename = "#{mount_name}-#{File.basename file}"
|
443
|
+
hostname = Socket.gethostname[/^[^.]+/]
|
444
|
+
command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{hostname}-#{filename}"
|
445
|
+
deb("Command: #{command}")
|
446
|
+
puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
|
447
|
+
ret = backquote_execute(command, :dryrun => true)
|
448
|
+
# if $opts[:debug] do
|
449
|
+
# puts "+ Current list of files:"
|
450
|
+
# ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
|
451
|
+
# puts ret
|
452
|
+
# end
|
453
|
+
ret
|
454
|
+
end
|