storazzo 0.4.2 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +7 -4
- data/Makefile +20 -1
- data/README.md +23 -0
- data/Rakefile +35 -8
- data/VERSION +1 -1
- data/bin/hello-storazzo +5 -0
- data/bin/ricdisk-magic +50 -57
- data/bin/stats-with-md5 +268 -297
- data/lib/storazzo/colors.rb +43 -45
- data/lib/storazzo/common.rb +80 -17
- data/lib/storazzo/debug.rb +5 -8
- data/lib/storazzo/hashify.rb +44 -43
- data/lib/storazzo/main.rb +44 -40
- data/lib/storazzo/media/abstract_ric_disk.rb +163 -104
- data/lib/storazzo/media/gcs_bucket.rb +51 -15
- data/lib/storazzo/media/local_folder.rb +43 -46
- data/lib/storazzo/media/mount_point.rb +12 -2
- data/lib/storazzo/ric_disk.rb +223 -251
- data/lib/storazzo/ric_disk_config.rb +230 -193
- data/lib/storazzo/ric_disk_sample_config.rb +12 -16
- data/lib/storazzo/ric_disk_statsfile.rb +17 -16
- data/lib/storazzo/ric_disk_ugly.rb +35 -38
- data/lib/storazzo/version.rb +7 -7
- data/lib/storazzo.rb +34 -29
- data/storazzo.gemspec +22 -20
- data/test/media/test_abstract_ric_disk.rb +19 -0
- data/test/media/test_gcs_bucket.rb +58 -0
- data/test/media/test_local_folder.rb +145 -0
- data/test/media/test_mount_point.rb +25 -0
- data/test/test_ric_disk.rb +16 -0
- data/test/test_ric_disk_config.rb +20 -29
- data/test/test_ric_disk_stats_file.rb +13 -14
- data/test/test_storazzo.rb +26 -26
- data/var/dumps/file_stat.linux.yaml +15 -0
- data/var/dumps/file_stat.macosx.yaml +15 -0
- data/var/test/disks/disk02-full/Rakefile +13 -0
- data/var/test/disks/ricdisk_stats_v11.rds +11 -0
- metadata +38 -10
- data/test/test_gcs_bucket.rb +0 -70
- data/test/test_local_folder.rb +0 -121
data/lib/storazzo/ric_disk.rb
CHANGED
@@ -4,62 +4,72 @@
|
|
4
4
|
require 'digest'
|
5
5
|
|
6
6
|
module Storazzo
|
7
|
-
class Storazzo::RicDisk
|
8
|
-
|
7
|
+
class Storazzo::RicDisk
|
9
8
|
include Hashify
|
10
|
-
include Storazzo::Common
|
9
|
+
include Storazzo::Common
|
10
|
+
extend Storazzo::Common
|
11
11
|
extend Storazzo::Colors
|
12
12
|
require 'socket'
|
13
13
|
|
14
|
-
|
15
|
-
## Instance variables
|
14
|
+
## Instance variables
|
16
15
|
|
17
16
|
# in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
|
18
17
|
# These are the files I do accept.
|
19
|
-
ConfigFiles = %W{
|
20
|
-
DefaultConfigFile = "storazzo.yaml" # .ricdisk }
|
18
|
+
ConfigFiles = %W{ricdisk.yaml .ricdisk storazzo.yaml}
|
19
|
+
DefaultConfigFile = "storazzo.yaml" # .ricdisk }
|
21
20
|
RicdiskVersion = '2.1'
|
22
21
|
RicdiskHistory = [
|
23
22
|
'2022-07-29 2.1 Added timestamp',
|
24
23
|
'2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...',
|
25
|
-
|
24
|
+
]
|
26
25
|
DefaultGemfileTestDiskFolder = Storazzo.root + "/var/test/disks/" # was: @@default_gemfile_test_disks_folder
|
27
26
|
# Immutable
|
28
|
-
DefaultMediaFolders = %w{
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
35
|
-
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
27
|
+
DefaultMediaFolders = %w{
|
28
|
+
/Volumes/
|
29
|
+
/mnt/
|
30
|
+
}.append(DefaultGemfileTestDiskFolder).append("/media/#{ENV["USER"]}/")
|
31
|
+
|
32
|
+
# # todo substitute with protobuf..
|
33
|
+
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
34
|
+
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
36
35
|
:unique_hash # new 202207
|
37
36
|
|
37
|
+
################################
|
38
|
+
## INSTANCE methods
|
39
|
+
################################
|
38
40
|
|
39
|
-
|
40
|
-
|
41
|
-
|
41
|
+
def initialize_old_way(path, opts = {})
|
42
|
+
raise "Now I dont want a string in input, I want an OBJECT < Storazzo::Media::AbstractRicDisk"
|
43
|
+
end
|
42
44
|
|
45
|
+
def initialize(ric_disk_object, opts = {})
|
46
|
+
verbose = opts.fetch :verbose, true
|
47
|
+
pverbose verbose,
|
48
|
+
"This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
|
49
|
+
raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
|
43
50
|
|
44
|
-
|
51
|
+
# ok back to business, now path is a String :)
|
52
|
+
path = ric_disk_object.path
|
53
|
+
deb "RicDisk initialize.. path=#{path}"
|
45
54
|
deb "RicDisk initialize.. path=#{path}"
|
46
55
|
@local_mountpoint = File.expand_path(path)
|
56
|
+
@ard = ric_disk_object # AbstractRicDiskObject
|
47
57
|
@description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
|
48
58
|
@ricdisk_version = RicdiskVersion
|
49
59
|
@ricdisk_file = compute_ricdisk_file() # Storazzo::RicDisk.get_ricdisk_file(path)
|
50
60
|
@ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
|
51
61
|
@label = path.split("/").last
|
52
62
|
@name = path.split("/").last
|
53
|
-
|
54
|
-
|
63
|
+
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
64
|
+
# @wr = writeable?
|
55
65
|
@tags = ['ricdisk', 'storazzo']
|
56
|
-
@size =
|
66
|
+
@size = RicDisk._compute_size_could_take_long(path)
|
57
67
|
@unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
58
68
|
@computation_hostname = Socket.gethostname
|
59
69
|
@created_at = Time.now
|
60
70
|
|
61
71
|
@ricdisk_file_empty = ricdisk_file_empty?
|
62
|
-
|
72
|
+
|
63
73
|
# @config = RicDiskConfig.instance.get_config
|
64
74
|
# #puts @config if @config
|
65
75
|
# find_info_from_mount(path)
|
@@ -77,7 +87,6 @@ module Storazzo
|
|
77
87
|
not ricdisk_file.nil?
|
78
88
|
end
|
79
89
|
|
80
|
-
|
81
90
|
def analyze_local_system()
|
82
91
|
puts "TODO This should analyzze the WHOLE system. TODO(ricc): move to another object which has to do with the system/computer."
|
83
92
|
puts "1. Interesting Mounts: #{green interesting_mount_points}"
|
@@ -86,85 +95,108 @@ module Storazzo
|
|
86
95
|
# find_info_from_df()
|
87
96
|
end
|
88
97
|
|
89
|
-
def path
|
98
|
+
def path
|
90
99
|
local_mountpoint
|
91
100
|
end
|
92
101
|
|
93
|
-
def to_s
|
94
|
-
"RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version})"
|
102
|
+
def to_s
|
103
|
+
"RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version}, ard=#{@ard})"
|
95
104
|
end
|
96
105
|
|
97
106
|
# could take long..
|
98
|
-
# def size
|
107
|
+
# def size
|
99
108
|
# `du -s '#{path}'`.split(/\s/)[0]
|
100
109
|
# end
|
110
|
+
def self._compute_size_could_take_long(my_path)
|
111
|
+
deb "Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds."
|
112
|
+
puts azure('could take long. Please take precautions like forking with timeout of 5sec')
|
113
|
+
`du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
|
114
|
+
end
|
101
115
|
|
102
|
-
def writeable?()
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
116
|
+
def writeable?()
|
117
|
+
# memoize
|
118
|
+
return @wr unless @wr.nil?
|
119
|
+
|
120
|
+
# NOW: CALCULATE it
|
121
|
+
# Now I can do ONCE an EXPENSIVE calculation
|
122
|
+
puts yellow("[RicDisk.writeable] TODO(ricc): Do expensive calculation if this FS is writeable: #{path} and write/memoize it on @wr once and for all")
|
123
|
+
puts yellow("[RicDisk.writeable] I have a feeling this should be delegated to praecipuus Storazzo::Media::Object we refer to (WR is different on GCS vs Local):") # infinite loop dammit #{self.to_verbose_s}")
|
124
|
+
puts("Dir: #{azure path}")
|
125
|
+
puts("absolute_path: #{azure absolute_path}")
|
126
|
+
puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
|
127
|
+
bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
|
128
|
+
puts("bash_output: #{azure bash_output}")
|
129
|
+
# @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
|
107
130
|
raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
|
131
|
+
|
108
132
|
@wr = File.writable?(absolute_path) # rescue false
|
109
133
|
return @wr
|
110
|
-
|
111
|
-
#false
|
134
|
+
# :boh_todo_fix_me_and_compute
|
135
|
+
# false
|
112
136
|
end
|
113
137
|
|
114
|
-
|
115
|
-
|
116
|
-
|
138
|
+
def to_verbose_s
|
139
|
+
h = {}
|
140
|
+
h[:to_s] = self.to_s
|
141
|
+
h[:wr] = self.wr
|
142
|
+
h[:inspect] = self.inspect
|
143
|
+
h[:writeable] = self.writeable?
|
144
|
+
h[:ard] = @ard
|
145
|
+
return h
|
146
|
+
end
|
117
147
|
|
148
|
+
################################
|
149
|
+
## CLASS methods
|
150
|
+
################################
|
118
151
|
|
119
152
|
# All places where to find for something :)
|
120
153
|
def self.default_media_folders
|
121
154
|
DefaultMediaFolders # was DEFAULT_MEDIA_FOLDERS
|
122
155
|
end
|
123
156
|
|
124
|
-
|
125
157
|
def self.test # _localgem_disks
|
126
|
-
d = RicDisk.new(
|
158
|
+
d = RicDisk.new(DefaultGemfileTestDiskFolder)
|
127
159
|
puts (d)
|
128
160
|
puts "do something with it: #{d}"
|
129
|
-
#d.find_active_dirs()
|
161
|
+
# d.find_active_dirs()
|
130
162
|
end
|
131
163
|
|
132
164
|
def absolute_path
|
133
|
-
|
165
|
+
# @local_mountpoint + "/" + @ricdisk_file
|
134
166
|
"#{local_mountpoint}/#{ricdisk_file}"
|
135
167
|
end
|
136
|
-
|
137
|
-
def self.find_active_dirs(base_dirs=nil, also_mountpoints=true)
|
168
|
+
|
169
|
+
def self.find_active_dirs(base_dirs = nil, also_mountpoints = true)
|
138
170
|
if base_dirs.nil?
|
139
|
-
base_dirs = default_media_folders
|
171
|
+
base_dirs = default_media_folders
|
140
172
|
puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
|
141
173
|
end
|
142
174
|
active_dirs = []
|
143
|
-
base_dirs.each do |ugly_dir|
|
175
|
+
base_dirs.each do |ugly_dir|
|
144
176
|
# https://stackoverflow.com/questions/1899072/getting-a-list-of-folders-in-a-directory#:~:text=Dir.chdir(%27/destination_directory%27)%0ADir.glob(%27*%27).select%20%7B%7Cf%7C%20File.directory%3F%20f%7D
|
145
177
|
dir = File.expand_path(ugly_dir)
|
146
178
|
begin
|
147
|
-
x=[]
|
148
|
-
|
179
|
+
x = []
|
180
|
+
# puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
|
149
181
|
unless Dir.exists?(dir)
|
150
182
|
deb "Dir doesnt exist, skipping: #{dir}"
|
151
|
-
next
|
183
|
+
next
|
152
184
|
end
|
153
185
|
Dir.chdir(dir)
|
154
|
-
x = Dir.glob('*').select {|f| File.directory? f}
|
155
|
-
subdirs = x.map{|subdir|
|
156
|
-
subdirs.each{|subdir|
|
157
|
-
|
158
|
-
puts `ls -al "#{subdir}"`
|
186
|
+
x = Dir.glob('*').select { |f| File.directory? f }
|
187
|
+
subdirs = x.map { |subdir| "#{dir}#{subdir}" }
|
188
|
+
subdirs.each { |subdir|
|
189
|
+
deb "Subdir: #{subdir}"
|
190
|
+
puts `ls -al "#{subdir}"` # TODO refactor in exec
|
159
191
|
active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
|
160
192
|
}
|
161
|
-
#puts(white x)
|
193
|
+
# puts(white x)
|
162
194
|
rescue Exception => e # optionally: `rescue Exception => ex`
|
163
195
|
puts "Exception: '#{e}'"
|
164
196
|
ensure # will always get executed
|
165
|
-
#deb 'Always gets executed.'
|
166
|
-
#x = []
|
167
|
-
end
|
197
|
+
# deb 'Always gets executed.'
|
198
|
+
# x = []
|
199
|
+
end
|
168
200
|
end
|
169
201
|
end
|
170
202
|
|
@@ -175,16 +207,15 @@ module Storazzo
|
|
175
207
|
end
|
176
208
|
deb "[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once"
|
177
209
|
ConfigFiles.each do |papable_config_filename|
|
178
|
-
#return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
179
|
-
#return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
210
|
+
# return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
211
|
+
# return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
180
212
|
return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
181
213
|
end
|
182
214
|
deb "File not found! Neither #{ConfigFiles} exist.."
|
183
|
-
# return nil
|
215
|
+
# return nil
|
184
216
|
return DefaultConfigFile
|
185
217
|
end
|
186
218
|
|
187
|
-
|
188
219
|
# def self.compute_ricdisk_file_by_path_once(path)
|
189
220
|
# # unless @ricdisk_file.nil?
|
190
221
|
# # deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
|
@@ -199,7 +230,6 @@ module Storazzo
|
|
199
230
|
# return nil
|
200
231
|
# end
|
201
232
|
|
202
|
-
|
203
233
|
# # new
|
204
234
|
# def self.get_ricdisk_file_obsolete(path)
|
205
235
|
# if @ricdisk_file
|
@@ -215,9 +245,8 @@ module Storazzo
|
|
215
245
|
# return nil
|
216
246
|
# end
|
217
247
|
|
218
|
-
|
219
|
-
|
220
|
-
#https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
248
|
+
def self.interesting_mount_points(opts = {})
|
249
|
+
# https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
221
250
|
`mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
|
222
251
|
end
|
223
252
|
|
@@ -227,23 +256,23 @@ module Storazzo
|
|
227
256
|
deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{(Time.now - creation_time)} seconds ago")
|
228
257
|
(Time.now - creation_time).to_i
|
229
258
|
end
|
259
|
+
|
230
260
|
# maybe move to a RiccFile class? Maybe even INHERIT from FILE?
|
231
261
|
def obsolescence_days(file_path)
|
232
262
|
return obsolescence_seconds(file_path) / 86400
|
233
263
|
end
|
234
264
|
|
235
|
-
|
236
265
|
# FORMER SBRODOLA, now write_config_yaml_to_disk
|
237
|
-
#def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
238
|
-
def write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
266
|
+
# def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
267
|
+
def write_config_yaml_to_disk(subdir, opts = {}) # sbrodola_ricdisk(subdir)
|
239
268
|
# given a path, if .ricdisk exists i do stuff with it..
|
240
269
|
disk_info = nil
|
241
270
|
unless ok_dir? # self.ok_dir?(subdir)
|
242
|
-
|
243
|
-
return
|
271
|
+
warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
|
272
|
+
return
|
244
273
|
end
|
245
274
|
ConfigFiles.each do |papable_configfile_name|
|
246
|
-
if File.exists?(
|
275
|
+
if File.exists?("#{subdir}/#{papable_configfile_name}") and File.empty?("#{subdir}/#{papable_configfile_name}")
|
247
276
|
deb("Interesting. Empty file '#{papable_configfile_name}'! Now I write YAML with it.")
|
248
277
|
disk_info = RicDisk.new(subdir, papable_configfile_name)
|
249
278
|
end
|
@@ -258,7 +287,7 @@ module Storazzo
|
|
258
287
|
# puts(yellow disk_info.to_yaml)
|
259
288
|
# end
|
260
289
|
if disk_info.is_a?(RicDisk)
|
261
|
-
|
290
|
+
deb yellow("disk_info.class: #{disk_info.class}")
|
262
291
|
if File.empty?(disk_info.absolute_path) # and (disk_info.wr)
|
263
292
|
puts(green("yay, we can now write the file '#{disk_info.absolute_path}' (which is R/W, I just checked!) with proper YAML content.."))
|
264
293
|
if disk_info.wr
|
@@ -279,8 +308,8 @@ module Storazzo
|
|
279
308
|
puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
|
280
309
|
end
|
281
310
|
|
282
|
-
#disk_info.absolute_path
|
283
|
-
#if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
311
|
+
# disk_info.absolute_path
|
312
|
+
# if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
284
313
|
# if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
|
285
314
|
# puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
|
286
315
|
# puts(white `cat "#{disk_info.absolute_path}"`)
|
@@ -292,191 +321,134 @@ module Storazzo
|
|
292
321
|
|
293
322
|
# TODO obsolete this as i should NOT be calling it from clas, but from method.
|
294
323
|
def self.ok_dir?(subdir)
|
295
|
-
File.exists?(
|
324
|
+
File.exists?("#{subdir}/.ricdisk") or File.exists?("#{subdir}/.ricdisk.yaml")
|
296
325
|
end
|
297
326
|
|
298
|
-
def compute_stats_files(opts={})
|
299
|
-
|
300
|
-
|
301
|
-
|
327
|
+
def compute_stats_files(opts = {})
|
328
|
+
# Storazzo::RicDisk.calculate_stats_files(path, opts)
|
329
|
+
opts_upload_to_gcs = opts.fetch :upload_to_gcs, false
|
330
|
+
opts_force_rewrite = opts.fetch :force, false
|
331
|
+
opts_stats_file = opts.fetch :stats_file, "ricdisk_stats_v11.rds" # default. TODO point to proper..
|
302
332
|
dir = path
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
333
|
+
puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy. stats_file=#{opts_stats_file} dir=#{dir}")
|
334
|
+
|
335
|
+
full_file_path = "#{dir}/#{opts_stats_file}"
|
336
|
+
deb "This refactor is for another day. Actually no, TODAY "
|
337
|
+
pverbose true,
|
338
|
+
"TODO(ricc): you should compute more SMARTLY the full_file_path (#{full_file_path}): if its R/O it should be elsewhere.."
|
339
|
+
puts azure("- full_file_path: #{full_file_path}")
|
340
|
+
puts azure("- writeable?: #{writeable?}")
|
341
|
+
|
307
342
|
puts("compute_stats_files(#{white dir}): #{white full_file_path}")
|
308
|
-
|
343
|
+
deb "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
|
344
|
+
raise "Directory doesnt exist: #{dir}" unless File.directory?(dir)
|
345
|
+
|
309
346
|
Dir.chdir(dir)
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
347
|
+
puts azure `ls` # im curious
|
348
|
+
if File.exists?(full_file_path)
|
349
|
+
if opts_force_rewrite
|
350
|
+
# raise "TODO implement file exists and FORCE enabled"
|
351
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ReWrite enabled")
|
352
|
+
else # File.exists?(full_file_path) and (opts_force)
|
353
|
+
puts "File '#{opts_stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
|
354
|
+
# TODO check for file time...
|
355
|
+
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp}. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
|
356
|
+
if obsolescence_days(full_file_path) > 7
|
357
|
+
# puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
358
|
+
puts yellow("*** ACHTUNG *** FIle is pretty old. I'll force a rewrite")
|
359
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
|
360
|
+
"File older than 7 days. Indeed: #{obsolescence_days(full_file_path)}")
|
361
|
+
end
|
362
|
+
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
316
363
|
end
|
317
|
-
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
318
364
|
else
|
319
|
-
|
320
|
-
|
321
|
-
puts("[#{`pwd`.chomp}] Executing: #{azure command}")
|
322
|
-
ret = backquote_execute(command)
|
323
|
-
puts "Done. #{ret.split("\n").count} files processed."
|
365
|
+
deb("File doesnt exist..")
|
366
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ConfigFile doesn't exist")
|
324
367
|
end
|
325
368
|
end
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
full_file_path = "#{dir}/#{$stats_file}"
|
334
|
-
return "This refactor is for another day"
|
335
|
-
|
336
|
-
puts("calculate_stats_files(#{white dir}): #{white full_file_path}")
|
337
|
-
puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
|
338
|
-
Dir.chdir(dir)
|
339
|
-
if File.exists?(full_file_path) and ($opts[:force] == false)
|
340
|
-
puts "File '#{$stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
|
341
|
-
# TODO check for file time...
|
342
|
-
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp }. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
|
343
|
-
if obsolescence_days(full_file_path) > 7
|
344
|
-
puts("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
345
|
-
end
|
346
|
-
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
347
|
-
else
|
348
|
-
puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]"
|
369
|
+
|
370
|
+
def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts = {})
|
371
|
+
max_lines = opts.fetch :max_lines, 42 # todo move to nil or -1
|
372
|
+
# full_file_path = "#{dir}/#{stats_file}"
|
373
|
+
puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
|
374
|
+
if max_lines < 0 # infinite
|
349
375
|
command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
350
|
-
|
351
|
-
|
352
|
-
|
376
|
+
else
|
377
|
+
# WOW! https://stackoverflow.com/questions/68599963/reliably-stop-bash-find-after-n-matches
|
378
|
+
# find . -type f -iname "*.txt" -print0 |
|
379
|
+
# head -z -n 10 |
|
380
|
+
# xargs -r0 myscript.sh
|
381
|
+
if mac?
|
382
|
+
puts red("Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)")
|
383
|
+
spannometric_lines = (max_lines / 10)
|
384
|
+
command = "find . -print0 | head -n '#{spannometric_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
385
|
+
else
|
386
|
+
command = "find . -print0 | head -z -n '#{max_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
387
|
+
end
|
353
388
|
end
|
389
|
+
puts("[#{`pwd`.chomp}] Executing: #{azure command}")
|
390
|
+
ret = backquote_execute(command)
|
391
|
+
puts "Done. #{ret.split("\n").count} files processed."
|
354
392
|
end
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
# Example output from mount:
|
359
|
-
|
360
|
-
# devfs on /dev (devfs, local, nobrowse)
|
361
|
-
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
362
|
-
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
363
|
-
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
364
|
-
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
365
|
-
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
366
|
-
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
367
|
-
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
368
|
-
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
369
|
-
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
370
|
-
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
371
|
-
# =end
|
372
|
-
# # add directories from current mountpoints...
|
373
|
-
# mount_table_lines = interesting_mount_points()
|
374
|
-
# mount_table_lines.each{|line|
|
375
|
-
# next if line =~ /^map /
|
376
|
-
# dev, on, path, mode = line.split(/ /)
|
377
|
-
# #puts line
|
378
|
-
# #deb yellow(path)
|
379
|
-
# active_dirs << path if self.ok_dir?(path)
|
380
|
-
# }
|
381
|
-
# end
|
382
|
-
# active_dirs.uniq!
|
383
|
-
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
384
|
-
# return active_dirs
|
385
|
-
# end
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
end #/Class
|
390
|
-
end #/Module
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
# def initialize(path, ricdisk_file)
|
397
|
-
# puts "[DEB] RicDisk initialize.. path=#{path}"
|
398
|
-
# @local_mountpoint = path
|
399
|
-
# @description = "This is an automated RicDisk description from v.#{VERSION}. Riccardo feel free to edit away with characteristicshs of this device.. Created on #{Time.now}'"
|
400
|
-
# @ricdisk_version = VERSION
|
401
|
-
# @ricdisk_file = ricdisk_file
|
402
|
-
# #@questo_non_esiste = :sobenme
|
403
|
-
# @label = path.split("/").last
|
404
|
-
# @name = path.split("/").last
|
405
|
-
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
406
|
-
# @tags = 'ricdisk'
|
407
|
-
# puts :beleza
|
408
|
-
# @config = RicDiskConfig.instance.get_config
|
409
|
-
# #puts @config if @config
|
410
|
-
# find_info_from_mount(path)
|
411
|
-
# find_info_from_df()
|
412
|
-
# end
|
413
|
-
|
414
|
-
# def ricdisk_absolute_path
|
415
|
-
# @local_mountpoint + "/" + @ricdisk_file
|
416
|
-
# end
|
417
|
-
|
418
|
-
# def add_tag(tag)
|
419
|
-
# @tags += ", #{tag}"
|
420
|
-
# end
|
421
|
-
|
422
|
-
# # might have other things in the future...
|
423
|
-
# def find_info_from_mount(path)
|
424
|
-
# mount_table_lines = interesting_mount_points()
|
425
|
-
# mount_line = nil
|
426
|
-
# mount_table_lines.each do |line|
|
427
|
-
# next if line =~ /^map /
|
428
|
-
# dev, on, mount_path, mode = line.split(/ /)
|
429
|
-
# if mount_path==path
|
430
|
-
# mount_line = line
|
431
|
-
# else
|
432
|
-
# @info_from_mount = false
|
433
|
-
# end
|
434
|
-
# end
|
435
|
-
# @info_from_mount = ! (mount_line.nil?)
|
436
|
-
# if @info_from_mount
|
437
|
-
# #@mount_line = mount_line
|
438
|
-
# @description += "\nMount line:\n" + mount_line
|
439
|
-
# @remote_mountpoint = mount_line.split(/ /)[0]
|
440
|
-
# @fstype = mount_line.split(/ /)[3].gsub(/[\(,]/, '')
|
441
|
-
# add_tag(:synology) if @remote_mountpoint.match('1.0.1.10')
|
442
|
-
# end
|
443
|
-
# end
|
444
|
-
|
445
|
-
# def find_info_from_df()
|
446
|
-
# path = @local_mountpoint
|
447
|
-
# df_info = `df -h "#{path}"`
|
448
|
-
# @df_info = df_info
|
449
|
-
# lines = df_info.split(/\n+/)
|
450
|
-
# raise "I need exactly TWO lines! Or no info is served here..." unless lines.size == 2
|
451
|
-
# mount, @size_readable, used_size, avail_size, @disk_utilization, other = lines[1].split(/\s+/) # second line..
|
452
|
-
# end
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
def backquote_execute(cmd, opts={})
|
460
|
-
dryrun = opts.fetch :dryrun, false
|
461
|
-
# executed a command wrapped by dryrun though
|
462
|
-
return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
|
463
|
-
`#{cmd}`
|
464
|
-
end
|
465
|
-
|
466
|
-
def upload_to_gcs(file, opts={})
|
467
|
-
deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
|
468
|
-
mount_name = file.split('/')[-2]
|
469
|
-
filename = "#{mount_name}-#{File.basename file}"
|
470
|
-
hostname = Socket.gethostname[/^[^.]+/]
|
471
|
-
command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{ hostname }-#{filename}"
|
472
|
-
deb("Command: #{command}")
|
473
|
-
puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
|
474
|
-
ret = backquote_execute(command, :dryrun => true)
|
475
|
-
# if $opts[:debug] do
|
476
|
-
# puts "+ Current list of files:"
|
477
|
-
# ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
|
478
|
-
# puts ret
|
479
|
-
# end
|
480
|
-
ret
|
393
|
+
|
394
|
+
def self.calculate_stats_files_DUPLICATE_STATIC(dir, opts = {})
|
395
|
+
raise "Please use object instead. If you cant, please move the object code to STATIC and dedupe code!"
|
481
396
|
end
|
482
|
-
|
397
|
+
|
398
|
+
# if also_mountpoints
|
399
|
+
# =begin
|
400
|
+
# Example output from mount:
|
401
|
+
|
402
|
+
# devfs on /dev (devfs, local, nobrowse)
|
403
|
+
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
404
|
+
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
405
|
+
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
406
|
+
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
407
|
+
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
408
|
+
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
409
|
+
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
410
|
+
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
411
|
+
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
412
|
+
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
413
|
+
# =end
|
414
|
+
# # add directories from current mountpoints...
|
415
|
+
# mount_table_lines = interesting_mount_points()
|
416
|
+
# mount_table_lines.each{|line|
|
417
|
+
# next if line =~ /^map /
|
418
|
+
# dev, on, path, mode = line.split(/ /)
|
419
|
+
# #puts line
|
420
|
+
# #deb yellow(path)
|
421
|
+
# active_dirs << path if self.ok_dir?(path)
|
422
|
+
# }
|
423
|
+
# end
|
424
|
+
# active_dirs.uniq!
|
425
|
+
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
426
|
+
# return active_dirs
|
427
|
+
# end
|
428
|
+
end # /Class
|
429
|
+
end # /Module
|
430
|
+
|
431
|
+
def backquote_execute(cmd, opts = {})
|
432
|
+
dryrun = opts.fetch :dryrun, false
|
433
|
+
# executed a command wrapped by dryrun though
|
434
|
+
return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
|
435
|
+
|
436
|
+
`#{cmd}`
|
437
|
+
end
|
438
|
+
|
439
|
+
def upload_to_gcs(file, opts = {})
|
440
|
+
deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
|
441
|
+
mount_name = file.split('/')[-2]
|
442
|
+
filename = "#{mount_name}-#{File.basename file}"
|
443
|
+
hostname = Socket.gethostname[/^[^.]+/]
|
444
|
+
command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{hostname}-#{filename}"
|
445
|
+
deb("Command: #{command}")
|
446
|
+
puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
|
447
|
+
ret = backquote_execute(command, :dryrun => true)
|
448
|
+
# if $opts[:debug] do
|
449
|
+
# puts "+ Current list of files:"
|
450
|
+
# ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
|
451
|
+
# puts ret
|
452
|
+
# end
|
453
|
+
ret
|
454
|
+
end
|