storazzo 0.4.10 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +5 -4
- data/Rakefile +8 -11
- data/VERSION +1 -1
- data/bin/hello-storazzo +0 -1
- data/bin/ricdisk-magic +50 -56
- data/bin/stats-with-md5 +251 -210
- data/bin/storazzo +32 -0
- data/bin/storazzo-util +1 -0
- data/lib/storazzo/colors.rb +43 -45
- data/lib/storazzo/common.rb +52 -46
- data/lib/storazzo/debug.rb +5 -8
- data/lib/storazzo/hashify.rb +44 -43
- data/lib/storazzo/main.rb +40 -41
- data/lib/storazzo/media/abstract_ric_disk.rb +162 -151
- data/lib/storazzo/media/gcs_bucket.rb +52 -51
- data/lib/storazzo/media/local_folder.rb +43 -44
- data/lib/storazzo/media/mount_point.rb +8 -9
- data/lib/storazzo/ric_disk.rb +153 -170
- data/lib/storazzo/ric_disk_config.rb +224 -219
- data/lib/storazzo/ric_disk_sample_config.rb +12 -12
- data/lib/storazzo/ric_disk_statsfile.rb +17 -15
- data/lib/storazzo/ric_disk_ugly.rb +35 -38
- data/lib/storazzo/version.rb +7 -7
- data/lib/storazzo.rb +36 -36
- data/storazzo.gemspec +26 -21
- data/test/media/test_abstract_ric_disk.rb +12 -16
- data/test/media/test_gcs_bucket.rb +44 -47
- data/test/media/test_local_folder.rb +134 -136
- data/test/media/test_mount_point.rb +18 -19
- data/test/test_ric_disk.rb +12 -14
- data/test/test_ric_disk_config.rb +20 -24
- data/test/test_ric_disk_stats_file.rb +13 -14
- data/test/test_storazzo.rb +26 -26
- data/var/test/disks/disk02-full/Rakefile +4 -4
- metadata +7 -3
data/lib/storazzo/ric_disk.rb
CHANGED
@@ -4,54 +4,52 @@
|
|
4
4
|
require 'digest'
|
5
5
|
|
6
6
|
module Storazzo
|
7
|
-
class Storazzo::RicDisk
|
8
|
-
|
7
|
+
class Storazzo::RicDisk
|
9
8
|
include Hashify
|
10
|
-
include Storazzo::Common
|
11
|
-
extend Storazzo::Common
|
9
|
+
include Storazzo::Common
|
10
|
+
extend Storazzo::Common
|
12
11
|
extend Storazzo::Colors
|
13
12
|
require 'socket'
|
14
13
|
|
15
|
-
|
16
|
-
## Instance variables
|
14
|
+
## Instance variables
|
17
15
|
|
18
16
|
# in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
|
19
17
|
# These are the files I do accept.
|
20
|
-
ConfigFiles = %W{
|
21
|
-
DefaultConfigFile = "storazzo.yaml" # .ricdisk }
|
18
|
+
ConfigFiles = %W{ricdisk.yaml .ricdisk storazzo.yaml}
|
19
|
+
DefaultConfigFile = "storazzo.yaml" # .ricdisk }
|
22
20
|
RicdiskVersion = '2.1'
|
23
21
|
RicdiskHistory = [
|
24
22
|
'2022-07-29 2.1 Added timestamp',
|
25
23
|
'2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...',
|
26
|
-
|
24
|
+
]
|
27
25
|
DefaultGemfileTestDiskFolder = Storazzo.root + "/var/test/disks/" # was: @@default_gemfile_test_disks_folder
|
28
26
|
# Immutable
|
29
|
-
DefaultMediaFolders = %w{
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
36
|
-
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
27
|
+
DefaultMediaFolders = %w{
|
28
|
+
/Volumes/
|
29
|
+
/mnt/
|
30
|
+
}.append(DefaultGemfileTestDiskFolder).append("/media/#{ENV["USER"]}/")
|
31
|
+
|
32
|
+
# # todo substitute with protobuf..
|
33
|
+
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
34
|
+
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
37
35
|
:unique_hash # new 202207
|
38
36
|
|
37
|
+
################################
|
38
|
+
## INSTANCE methods
|
39
|
+
################################
|
39
40
|
|
40
|
-
|
41
|
-
## INSTANCE methods
|
42
|
-
################################
|
43
|
-
|
44
|
-
def initialize_old_way(path, opts={})
|
41
|
+
def initialize_old_way(path, opts = {})
|
45
42
|
raise "Now I dont want a string in input, I want an OBJECT < Storazzo::Media::AbstractRicDisk"
|
46
43
|
end
|
47
44
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
45
|
+
def initialize(ric_disk_object, opts = {})
|
46
|
+
verbose = opts.fetch :verbose, true
|
47
|
+
pverbose verbose,
|
48
|
+
"This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
|
52
49
|
raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
|
50
|
+
|
53
51
|
# ok back to business, now path is a String :)
|
54
|
-
path =
|
52
|
+
path = ric_disk_object.path
|
55
53
|
deb "RicDisk initialize.. path=#{path}"
|
56
54
|
deb "RicDisk initialize.. path=#{path}"
|
57
55
|
@local_mountpoint = File.expand_path(path)
|
@@ -62,16 +60,16 @@ module Storazzo
|
|
62
60
|
@ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
|
63
61
|
@label = path.split("/").last
|
64
62
|
@name = path.split("/").last
|
65
|
-
|
66
|
-
|
63
|
+
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
64
|
+
# @wr = writeable?
|
67
65
|
@tags = ['ricdisk', 'storazzo']
|
68
|
-
@size = RicDisk._compute_size_could_take_long(path)
|
66
|
+
@size = RicDisk._compute_size_could_take_long(path)
|
69
67
|
@unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
70
68
|
@computation_hostname = Socket.gethostname
|
71
69
|
@created_at = Time.now
|
72
70
|
|
73
71
|
@ricdisk_file_empty = ricdisk_file_empty?
|
74
|
-
|
72
|
+
|
75
73
|
# @config = RicDiskConfig.instance.get_config
|
76
74
|
# #puts @config if @config
|
77
75
|
# find_info_from_mount(path)
|
@@ -89,7 +87,6 @@ module Storazzo
|
|
89
87
|
not ricdisk_file.nil?
|
90
88
|
end
|
91
89
|
|
92
|
-
|
93
90
|
def analyze_local_system()
|
94
91
|
puts "TODO This should analyzze the WHOLE system. TODO(ricc): move to another object which has to do with the system/computer."
|
95
92
|
puts "1. Interesting Mounts: #{green interesting_mount_points}"
|
@@ -98,45 +95,47 @@ module Storazzo
|
|
98
95
|
# find_info_from_df()
|
99
96
|
end
|
100
97
|
|
101
|
-
def path
|
98
|
+
def path
|
102
99
|
local_mountpoint
|
103
100
|
end
|
104
101
|
|
105
|
-
def to_s
|
102
|
+
def to_s
|
106
103
|
"RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version}, ard=#{@ard})"
|
107
104
|
end
|
108
105
|
|
109
106
|
# could take long..
|
110
|
-
# def size
|
107
|
+
# def size
|
111
108
|
# `du -s '#{path}'`.split(/\s/)[0]
|
112
109
|
# end
|
113
|
-
def self._compute_size_could_take_long(my_path)
|
110
|
+
def self._compute_size_could_take_long(my_path)
|
114
111
|
deb "Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds."
|
115
112
|
puts azure('could take long. Please take precautions like forking with timeout of 5sec')
|
116
113
|
`du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
|
117
114
|
end
|
118
115
|
|
119
|
-
def writeable?()
|
120
|
-
#memoize
|
121
|
-
return @wr unless @wr.nil?
|
116
|
+
def writeable?()
|
117
|
+
# memoize
|
118
|
+
return @wr unless @wr.nil?
|
119
|
+
|
122
120
|
# NOW: CALCULATE it
|
123
121
|
# Now I can do ONCE an EXPENSIVE calculation
|
124
122
|
puts yellow("[RicDisk.writeable] TODO(ricc): Do expensive calculation if this FS is writeable: #{path} and write/memoize it on @wr once and for all")
|
125
123
|
puts yellow("[RicDisk.writeable] I have a feeling this should be delegated to praecipuus Storazzo::Media::Object we refer to (WR is different on GCS vs Local):") # infinite loop dammit #{self.to_verbose_s}")
|
126
|
-
puts("Dir: #{
|
124
|
+
puts("Dir: #{azure path}")
|
127
125
|
puts("absolute_path: #{azure absolute_path}")
|
128
126
|
puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
|
129
127
|
bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
|
130
128
|
puts("bash_output: #{azure bash_output}")
|
131
|
-
|
129
|
+
# @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
|
132
130
|
raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
|
131
|
+
|
133
132
|
@wr = File.writable?(absolute_path) # rescue false
|
134
133
|
return @wr
|
135
|
-
|
136
|
-
#false
|
134
|
+
# :boh_todo_fix_me_and_compute
|
135
|
+
# false
|
137
136
|
end
|
138
137
|
|
139
|
-
def to_verbose_s
|
138
|
+
def to_verbose_s
|
140
139
|
h = {}
|
141
140
|
h[:to_s] = self.to_s
|
142
141
|
h[:wr] = self.wr
|
@@ -146,60 +145,58 @@ module Storazzo
|
|
146
145
|
return h
|
147
146
|
end
|
148
147
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
148
|
+
################################
|
149
|
+
## CLASS methods
|
150
|
+
################################
|
153
151
|
|
154
152
|
# All places where to find for something :)
|
155
153
|
def self.default_media_folders
|
156
154
|
DefaultMediaFolders # was DEFAULT_MEDIA_FOLDERS
|
157
155
|
end
|
158
156
|
|
159
|
-
|
160
157
|
def self.test # _localgem_disks
|
161
|
-
d = RicDisk.new(
|
158
|
+
d = RicDisk.new(DefaultGemfileTestDiskFolder)
|
162
159
|
puts (d)
|
163
160
|
puts "do something with it: #{d}"
|
164
|
-
#d.find_active_dirs()
|
161
|
+
# d.find_active_dirs()
|
165
162
|
end
|
166
163
|
|
167
164
|
def absolute_path
|
168
|
-
|
165
|
+
# @local_mountpoint + "/" + @ricdisk_file
|
169
166
|
"#{local_mountpoint}/#{ricdisk_file}"
|
170
167
|
end
|
171
|
-
|
172
|
-
def self.find_active_dirs(base_dirs=nil, also_mountpoints=true)
|
168
|
+
|
169
|
+
def self.find_active_dirs(base_dirs = nil, also_mountpoints = true)
|
173
170
|
if base_dirs.nil?
|
174
|
-
base_dirs = default_media_folders
|
171
|
+
base_dirs = default_media_folders
|
175
172
|
puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
|
176
173
|
end
|
177
174
|
active_dirs = []
|
178
|
-
base_dirs.each do |ugly_dir|
|
175
|
+
base_dirs.each do |ugly_dir|
|
179
176
|
# https://stackoverflow.com/questions/1899072/getting-a-list-of-folders-in-a-directory#:~:text=Dir.chdir(%27/destination_directory%27)%0ADir.glob(%27*%27).select%20%7B%7Cf%7C%20File.directory%3F%20f%7D
|
180
177
|
dir = File.expand_path(ugly_dir)
|
181
178
|
begin
|
182
|
-
x=[]
|
183
|
-
|
179
|
+
x = []
|
180
|
+
# puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
|
184
181
|
unless Dir.exists?(dir)
|
185
182
|
deb "Dir doesnt exist, skipping: #{dir}"
|
186
|
-
next
|
183
|
+
next
|
187
184
|
end
|
188
185
|
Dir.chdir(dir)
|
189
|
-
x = Dir.glob('*').select {|f| File.directory? f}
|
190
|
-
subdirs = x.map{|subdir|
|
191
|
-
subdirs.each{|subdir|
|
186
|
+
x = Dir.glob('*').select { |f| File.directory? f }
|
187
|
+
subdirs = x.map { |subdir| "#{dir}#{subdir}" }
|
188
|
+
subdirs.each { |subdir|
|
192
189
|
deb "Subdir: #{subdir}"
|
193
190
|
puts `ls -al "#{subdir}"` # TODO refactor in exec
|
194
191
|
active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
|
195
192
|
}
|
196
|
-
#puts(white x)
|
193
|
+
# puts(white x)
|
197
194
|
rescue Exception => e # optionally: `rescue Exception => ex`
|
198
195
|
puts "Exception: '#{e}'"
|
199
196
|
ensure # will always get executed
|
200
|
-
#deb 'Always gets executed.'
|
201
|
-
#x = []
|
202
|
-
end
|
197
|
+
# deb 'Always gets executed.'
|
198
|
+
# x = []
|
199
|
+
end
|
203
200
|
end
|
204
201
|
end
|
205
202
|
|
@@ -210,16 +207,15 @@ module Storazzo
|
|
210
207
|
end
|
211
208
|
deb "[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once"
|
212
209
|
ConfigFiles.each do |papable_config_filename|
|
213
|
-
#return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
214
|
-
#return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
210
|
+
# return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
211
|
+
# return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
215
212
|
return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
216
213
|
end
|
217
214
|
deb "File not found! Neither #{ConfigFiles} exist.."
|
218
|
-
# return nil
|
215
|
+
# return nil
|
219
216
|
return DefaultConfigFile
|
220
217
|
end
|
221
218
|
|
222
|
-
|
223
219
|
# def self.compute_ricdisk_file_by_path_once(path)
|
224
220
|
# # unless @ricdisk_file.nil?
|
225
221
|
# # deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
|
@@ -234,7 +230,6 @@ module Storazzo
|
|
234
230
|
# return nil
|
235
231
|
# end
|
236
232
|
|
237
|
-
|
238
233
|
# # new
|
239
234
|
# def self.get_ricdisk_file_obsolete(path)
|
240
235
|
# if @ricdisk_file
|
@@ -250,9 +245,8 @@ module Storazzo
|
|
250
245
|
# return nil
|
251
246
|
# end
|
252
247
|
|
253
|
-
|
254
|
-
|
255
|
-
#https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
248
|
+
def self.interesting_mount_points(opts = {})
|
249
|
+
# https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
256
250
|
`mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
|
257
251
|
end
|
258
252
|
|
@@ -262,23 +256,23 @@ module Storazzo
|
|
262
256
|
deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{(Time.now - creation_time)} seconds ago")
|
263
257
|
(Time.now - creation_time).to_i
|
264
258
|
end
|
259
|
+
|
265
260
|
# maybe move to a RiccFile class? Maybe even INHERIT from FILE?
|
266
261
|
def obsolescence_days(file_path)
|
267
262
|
return obsolescence_seconds(file_path) / 86400
|
268
263
|
end
|
269
264
|
|
270
|
-
|
271
265
|
# FORMER SBRODOLA, now write_config_yaml_to_disk
|
272
|
-
#def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
273
|
-
def write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
266
|
+
# def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
267
|
+
def write_config_yaml_to_disk(subdir, opts = {}) # sbrodola_ricdisk(subdir)
|
274
268
|
# given a path, if .ricdisk exists i do stuff with it..
|
275
269
|
disk_info = nil
|
276
270
|
unless ok_dir? # self.ok_dir?(subdir)
|
277
271
|
warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
|
278
|
-
return
|
272
|
+
return
|
279
273
|
end
|
280
274
|
ConfigFiles.each do |papable_configfile_name|
|
281
|
-
if File.exists?(
|
275
|
+
if File.exists?("#{subdir}/#{papable_configfile_name}") and File.empty?("#{subdir}/#{papable_configfile_name}")
|
282
276
|
deb("Interesting. Empty file '#{papable_configfile_name}'! Now I write YAML with it.")
|
283
277
|
disk_info = RicDisk.new(subdir, papable_configfile_name)
|
284
278
|
end
|
@@ -314,8 +308,8 @@ module Storazzo
|
|
314
308
|
puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
|
315
309
|
end
|
316
310
|
|
317
|
-
#disk_info.absolute_path
|
318
|
-
#if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
311
|
+
# disk_info.absolute_path
|
312
|
+
# if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
319
313
|
# if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
|
320
314
|
# puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
|
321
315
|
# puts(white `cat "#{disk_info.absolute_path}"`)
|
@@ -327,42 +321,45 @@ module Storazzo
|
|
327
321
|
|
328
322
|
# TODO obsolete this as i should NOT be calling it from clas, but from method.
|
329
323
|
def self.ok_dir?(subdir)
|
330
|
-
File.exists?(
|
324
|
+
File.exists?("#{subdir}/.ricdisk") or File.exists?("#{subdir}/.ricdisk.yaml")
|
331
325
|
end
|
332
326
|
|
333
|
-
def compute_stats_files(opts={})
|
334
|
-
#Storazzo::RicDisk.calculate_stats_files(path, opts)
|
327
|
+
def compute_stats_files(opts = {})
|
328
|
+
# Storazzo::RicDisk.calculate_stats_files(path, opts)
|
335
329
|
opts_upload_to_gcs = opts.fetch :upload_to_gcs, false
|
336
330
|
opts_force_rewrite = opts.fetch :force, false
|
337
331
|
opts_stats_file = opts.fetch :stats_file, "ricdisk_stats_v11.rds" # default. TODO point to proper..
|
338
332
|
dir = path
|
339
333
|
puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy. stats_file=#{opts_stats_file} dir=#{dir}")
|
340
|
-
|
334
|
+
|
341
335
|
full_file_path = "#{dir}/#{opts_stats_file}"
|
342
336
|
deb "This refactor is for another day. Actually no, TODAY "
|
343
|
-
pverbose true,
|
337
|
+
pverbose true,
|
338
|
+
"TODO(ricc): you should compute more SMARTLY the full_file_path (#{full_file_path}): if its R/O it should be elsewhere.."
|
344
339
|
puts azure("- full_file_path: #{full_file_path}")
|
345
340
|
puts azure("- writeable?: #{writeable?}")
|
346
|
-
|
341
|
+
|
347
342
|
puts("compute_stats_files(#{white dir}): #{white full_file_path}")
|
348
343
|
deb "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
|
349
344
|
raise "Directory doesnt exist: #{dir}" unless File.directory?(dir)
|
345
|
+
|
350
346
|
Dir.chdir(dir)
|
351
347
|
puts azure `ls` # im curious
|
352
348
|
if File.exists?(full_file_path)
|
353
349
|
if opts_force_rewrite
|
354
|
-
#raise "TODO implement file exists and FORCE enabled"
|
350
|
+
# raise "TODO implement file exists and FORCE enabled"
|
355
351
|
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ReWrite enabled")
|
356
352
|
else # File.exists?(full_file_path) and (opts_force)
|
357
353
|
puts "File '#{opts_stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
|
358
354
|
# TODO check for file time...
|
359
|
-
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp
|
360
|
-
if obsolescence_days(full_file_path) > 7
|
361
|
-
#puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
355
|
+
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp}. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
|
356
|
+
if obsolescence_days(full_file_path) > 7
|
357
|
+
# puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
362
358
|
puts yellow("*** ACHTUNG *** FIle is pretty old. I'll force a rewrite")
|
363
|
-
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
|
359
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
|
360
|
+
"File older than 7 days. Indeed: #{obsolescence_days(full_file_path)}")
|
364
361
|
end
|
365
|
-
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
362
|
+
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
366
363
|
end
|
367
364
|
else
|
368
365
|
deb("File doesnt exist..")
|
@@ -370,10 +367,10 @@ module Storazzo
|
|
370
367
|
end
|
371
368
|
end
|
372
369
|
|
373
|
-
def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts={})
|
370
|
+
def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts = {})
|
374
371
|
max_lines = opts.fetch :max_lines, 42 # todo move to nil or -1
|
375
|
-
#full_file_path = "#{dir}/#{stats_file}"
|
376
|
-
puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
|
372
|
+
# full_file_path = "#{dir}/#{stats_file}"
|
373
|
+
puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
|
377
374
|
if max_lines < 0 # infinite
|
378
375
|
command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
379
376
|
else
|
@@ -383,7 +380,7 @@ module Storazzo
|
|
383
380
|
# xargs -r0 myscript.sh
|
384
381
|
if mac?
|
385
382
|
puts red("Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)")
|
386
|
-
spannometric_lines = (max_lines/10)
|
383
|
+
spannometric_lines = (max_lines / 10)
|
387
384
|
command = "find . -print0 | head -n '#{spannometric_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
388
385
|
else
|
389
386
|
command = "find . -print0 | head -z -n '#{max_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
@@ -393,79 +390,65 @@ module Storazzo
|
|
393
390
|
ret = backquote_execute(command)
|
394
391
|
puts "Done. #{ret.split("\n").count} files processed."
|
395
392
|
end
|
396
|
-
|
397
|
-
|
398
393
|
|
399
|
-
def self.calculate_stats_files_DUPLICATE_STATIC(dir, opts={})
|
394
|
+
def self.calculate_stats_files_DUPLICATE_STATIC(dir, opts = {})
|
400
395
|
raise "Please use object instead. If you cant, please move the object code to STATIC and dedupe code!"
|
401
|
-
end
|
402
|
-
|
403
|
-
|
404
|
-
# if also_mountpoints
|
405
|
-
# =begin
|
406
|
-
# Example output from mount:
|
407
|
-
|
408
|
-
# devfs on /dev (devfs, local, nobrowse)
|
409
|
-
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
410
|
-
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
411
|
-
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
412
|
-
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
413
|
-
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
414
|
-
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
415
|
-
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
416
|
-
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
417
|
-
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
418
|
-
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
419
|
-
# =end
|
420
|
-
# # add directories from current mountpoints...
|
421
|
-
# mount_table_lines = interesting_mount_points()
|
422
|
-
# mount_table_lines.each{|line|
|
423
|
-
# next if line =~ /^map /
|
424
|
-
# dev, on, path, mode = line.split(/ /)
|
425
|
-
# #puts line
|
426
|
-
# #deb yellow(path)
|
427
|
-
# active_dirs << path if self.ok_dir?(path)
|
428
|
-
# }
|
429
|
-
# end
|
430
|
-
# active_dirs.uniq!
|
431
|
-
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
432
|
-
# return active_dirs
|
433
|
-
# end
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
end #/Class
|
438
|
-
end #/Module
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
def backquote_execute(cmd, opts={})
|
449
|
-
dryrun = opts.fetch :dryrun, false
|
450
|
-
# executed a command wrapped by dryrun though
|
451
|
-
return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
|
452
|
-
`#{cmd}`
|
453
396
|
end
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
463
|
-
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
469
|
-
|
470
|
-
end
|
471
|
-
|
397
|
+
|
398
|
+
# if also_mountpoints
|
399
|
+
# =begin
|
400
|
+
# Example output from mount:
|
401
|
+
|
402
|
+
# devfs on /dev (devfs, local, nobrowse)
|
403
|
+
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
404
|
+
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
405
|
+
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
406
|
+
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
407
|
+
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
408
|
+
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
409
|
+
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
410
|
+
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
411
|
+
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
412
|
+
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
413
|
+
# =end
|
414
|
+
# # add directories from current mountpoints...
|
415
|
+
# mount_table_lines = interesting_mount_points()
|
416
|
+
# mount_table_lines.each{|line|
|
417
|
+
# next if line =~ /^map /
|
418
|
+
# dev, on, path, mode = line.split(/ /)
|
419
|
+
# #puts line
|
420
|
+
# #deb yellow(path)
|
421
|
+
# active_dirs << path if self.ok_dir?(path)
|
422
|
+
# }
|
423
|
+
# end
|
424
|
+
# active_dirs.uniq!
|
425
|
+
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
426
|
+
# return active_dirs
|
427
|
+
# end
|
428
|
+
end # /Class
|
429
|
+
end # /Module
|
430
|
+
|
431
|
+
def backquote_execute(cmd, opts = {})
|
432
|
+
dryrun = opts.fetch :dryrun, false
|
433
|
+
# executed a command wrapped by dryrun though
|
434
|
+
return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
|
435
|
+
|
436
|
+
`#{cmd}`
|
437
|
+
end
|
438
|
+
|
439
|
+
def upload_to_gcs(file, opts = {})
|
440
|
+
deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
|
441
|
+
mount_name = file.split('/')[-2]
|
442
|
+
filename = "#{mount_name}-#{File.basename file}"
|
443
|
+
hostname = Socket.gethostname[/^[^.]+/]
|
444
|
+
command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{hostname}-#{filename}"
|
445
|
+
deb("Command: #{command}")
|
446
|
+
puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
|
447
|
+
ret = backquote_execute(command, :dryrun => true)
|
448
|
+
# if $opts[:debug] do
|
449
|
+
# puts "+ Current list of files:"
|
450
|
+
# ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
|
451
|
+
# puts ret
|
452
|
+
# end
|
453
|
+
ret
|
454
|
+
end
|