storazzo 0.4.2 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +7 -4
- data/Makefile +20 -1
- data/README.md +23 -0
- data/Rakefile +35 -8
- data/VERSION +1 -1
- data/bin/hello-storazzo +5 -0
- data/bin/ricdisk-magic +50 -57
- data/bin/stats-with-md5 +268 -297
- data/lib/storazzo/colors.rb +43 -45
- data/lib/storazzo/common.rb +80 -17
- data/lib/storazzo/debug.rb +5 -8
- data/lib/storazzo/hashify.rb +44 -43
- data/lib/storazzo/main.rb +44 -40
- data/lib/storazzo/media/abstract_ric_disk.rb +163 -104
- data/lib/storazzo/media/gcs_bucket.rb +51 -15
- data/lib/storazzo/media/local_folder.rb +43 -46
- data/lib/storazzo/media/mount_point.rb +12 -2
- data/lib/storazzo/ric_disk.rb +223 -251
- data/lib/storazzo/ric_disk_config.rb +230 -193
- data/lib/storazzo/ric_disk_sample_config.rb +12 -16
- data/lib/storazzo/ric_disk_statsfile.rb +17 -16
- data/lib/storazzo/ric_disk_ugly.rb +35 -38
- data/lib/storazzo/version.rb +7 -7
- data/lib/storazzo.rb +34 -29
- data/storazzo.gemspec +22 -20
- data/test/media/test_abstract_ric_disk.rb +19 -0
- data/test/media/test_gcs_bucket.rb +58 -0
- data/test/media/test_local_folder.rb +145 -0
- data/test/media/test_mount_point.rb +25 -0
- data/test/test_ric_disk.rb +16 -0
- data/test/test_ric_disk_config.rb +20 -29
- data/test/test_ric_disk_stats_file.rb +13 -14
- data/test/test_storazzo.rb +26 -26
- data/var/dumps/file_stat.linux.yaml +15 -0
- data/var/dumps/file_stat.macosx.yaml +15 -0
- data/var/test/disks/disk02-full/Rakefile +13 -0
- data/var/test/disks/ricdisk_stats_v11.rds +11 -0
- metadata +38 -10
- data/test/test_gcs_bucket.rb +0 -70
- data/test/test_local_folder.rb +0 -121
data/lib/storazzo/ric_disk.rb
CHANGED
@@ -4,62 +4,72 @@
|
|
4
4
|
require 'digest'
|
5
5
|
|
6
6
|
module Storazzo
|
7
|
-
class Storazzo::RicDisk
|
8
|
-
|
7
|
+
class Storazzo::RicDisk
|
9
8
|
include Hashify
|
10
|
-
include Storazzo::Common
|
9
|
+
include Storazzo::Common
|
10
|
+
extend Storazzo::Common
|
11
11
|
extend Storazzo::Colors
|
12
12
|
require 'socket'
|
13
13
|
|
14
|
-
|
15
|
-
## Instance variables
|
14
|
+
## Instance variables
|
16
15
|
|
17
16
|
# in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
|
18
17
|
# These are the files I do accept.
|
19
|
-
ConfigFiles = %W{
|
20
|
-
DefaultConfigFile = "storazzo.yaml" # .ricdisk }
|
18
|
+
ConfigFiles = %W{ricdisk.yaml .ricdisk storazzo.yaml}
|
19
|
+
DefaultConfigFile = "storazzo.yaml" # .ricdisk }
|
21
20
|
RicdiskVersion = '2.1'
|
22
21
|
RicdiskHistory = [
|
23
22
|
'2022-07-29 2.1 Added timestamp',
|
24
23
|
'2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...',
|
25
|
-
|
24
|
+
]
|
26
25
|
DefaultGemfileTestDiskFolder = Storazzo.root + "/var/test/disks/" # was: @@default_gemfile_test_disks_folder
|
27
26
|
# Immutable
|
28
|
-
DefaultMediaFolders = %w{
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
35
|
-
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
27
|
+
DefaultMediaFolders = %w{
|
28
|
+
/Volumes/
|
29
|
+
/mnt/
|
30
|
+
}.append(DefaultGemfileTestDiskFolder).append("/media/#{ENV["USER"]}/")
|
31
|
+
|
32
|
+
# # todo substitute with protobuf..
|
33
|
+
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
34
|
+
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
36
35
|
:unique_hash # new 202207
|
37
36
|
|
37
|
+
################################
|
38
|
+
## INSTANCE methods
|
39
|
+
################################
|
38
40
|
|
39
|
-
|
40
|
-
|
41
|
-
|
41
|
+
def initialize_old_way(path, opts = {})
|
42
|
+
raise "Now I dont want a string in input, I want an OBJECT < Storazzo::Media::AbstractRicDisk"
|
43
|
+
end
|
42
44
|
|
45
|
+
def initialize(ric_disk_object, opts = {})
|
46
|
+
verbose = opts.fetch :verbose, true
|
47
|
+
pverbose verbose,
|
48
|
+
"This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
|
49
|
+
raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
|
43
50
|
|
44
|
-
|
51
|
+
# ok back to business, now path is a String :)
|
52
|
+
path = ric_disk_object.path
|
53
|
+
deb "RicDisk initialize.. path=#{path}"
|
45
54
|
deb "RicDisk initialize.. path=#{path}"
|
46
55
|
@local_mountpoint = File.expand_path(path)
|
56
|
+
@ard = ric_disk_object # AbstractRicDiskObject
|
47
57
|
@description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
|
48
58
|
@ricdisk_version = RicdiskVersion
|
49
59
|
@ricdisk_file = compute_ricdisk_file() # Storazzo::RicDisk.get_ricdisk_file(path)
|
50
60
|
@ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
|
51
61
|
@label = path.split("/").last
|
52
62
|
@name = path.split("/").last
|
53
|
-
|
54
|
-
|
63
|
+
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
64
|
+
# @wr = writeable?
|
55
65
|
@tags = ['ricdisk', 'storazzo']
|
56
|
-
@size =
|
66
|
+
@size = RicDisk._compute_size_could_take_long(path)
|
57
67
|
@unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
58
68
|
@computation_hostname = Socket.gethostname
|
59
69
|
@created_at = Time.now
|
60
70
|
|
61
71
|
@ricdisk_file_empty = ricdisk_file_empty?
|
62
|
-
|
72
|
+
|
63
73
|
# @config = RicDiskConfig.instance.get_config
|
64
74
|
# #puts @config if @config
|
65
75
|
# find_info_from_mount(path)
|
@@ -77,7 +87,6 @@ module Storazzo
|
|
77
87
|
not ricdisk_file.nil?
|
78
88
|
end
|
79
89
|
|
80
|
-
|
81
90
|
def analyze_local_system()
|
82
91
|
puts "TODO This should analyzze the WHOLE system. TODO(ricc): move to another object which has to do with the system/computer."
|
83
92
|
puts "1. Interesting Mounts: #{green interesting_mount_points}"
|
@@ -86,85 +95,108 @@ module Storazzo
|
|
86
95
|
# find_info_from_df()
|
87
96
|
end
|
88
97
|
|
89
|
-
def path
|
98
|
+
def path
|
90
99
|
local_mountpoint
|
91
100
|
end
|
92
101
|
|
93
|
-
def to_s
|
94
|
-
"RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version})"
|
102
|
+
def to_s
|
103
|
+
"RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version}, ard=#{@ard})"
|
95
104
|
end
|
96
105
|
|
97
106
|
# could take long..
|
98
|
-
# def size
|
107
|
+
# def size
|
99
108
|
# `du -s '#{path}'`.split(/\s/)[0]
|
100
109
|
# end
|
110
|
+
def self._compute_size_could_take_long(my_path)
|
111
|
+
deb "Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds."
|
112
|
+
puts azure('could take long. Please take precautions like forking with timeout of 5sec')
|
113
|
+
`du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
|
114
|
+
end
|
101
115
|
|
102
|
-
def writeable?()
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
116
|
+
def writeable?()
|
117
|
+
# memoize
|
118
|
+
return @wr unless @wr.nil?
|
119
|
+
|
120
|
+
# NOW: CALCULATE it
|
121
|
+
# Now I can do ONCE an EXPENSIVE calculation
|
122
|
+
puts yellow("[RicDisk.writeable] TODO(ricc): Do expensive calculation if this FS is writeable: #{path} and write/memoize it on @wr once and for all")
|
123
|
+
puts yellow("[RicDisk.writeable] I have a feeling this should be delegated to praecipuus Storazzo::Media::Object we refer to (WR is different on GCS vs Local):") # infinite loop dammit #{self.to_verbose_s}")
|
124
|
+
puts("Dir: #{azure path}")
|
125
|
+
puts("absolute_path: #{azure absolute_path}")
|
126
|
+
puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
|
127
|
+
bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
|
128
|
+
puts("bash_output: #{azure bash_output}")
|
129
|
+
# @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
|
107
130
|
raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
|
131
|
+
|
108
132
|
@wr = File.writable?(absolute_path) # rescue false
|
109
133
|
return @wr
|
110
|
-
|
111
|
-
#false
|
134
|
+
# :boh_todo_fix_me_and_compute
|
135
|
+
# false
|
112
136
|
end
|
113
137
|
|
114
|
-
|
115
|
-
|
116
|
-
|
138
|
+
def to_verbose_s
|
139
|
+
h = {}
|
140
|
+
h[:to_s] = self.to_s
|
141
|
+
h[:wr] = self.wr
|
142
|
+
h[:inspect] = self.inspect
|
143
|
+
h[:writeable] = self.writeable?
|
144
|
+
h[:ard] = @ard
|
145
|
+
return h
|
146
|
+
end
|
117
147
|
|
148
|
+
################################
|
149
|
+
## CLASS methods
|
150
|
+
################################
|
118
151
|
|
119
152
|
# All places where to find for something :)
|
120
153
|
def self.default_media_folders
|
121
154
|
DefaultMediaFolders # was DEFAULT_MEDIA_FOLDERS
|
122
155
|
end
|
123
156
|
|
124
|
-
|
125
157
|
def self.test # _localgem_disks
|
126
|
-
d = RicDisk.new(
|
158
|
+
d = RicDisk.new(DefaultGemfileTestDiskFolder)
|
127
159
|
puts (d)
|
128
160
|
puts "do something with it: #{d}"
|
129
|
-
#d.find_active_dirs()
|
161
|
+
# d.find_active_dirs()
|
130
162
|
end
|
131
163
|
|
132
164
|
def absolute_path
|
133
|
-
|
165
|
+
# @local_mountpoint + "/" + @ricdisk_file
|
134
166
|
"#{local_mountpoint}/#{ricdisk_file}"
|
135
167
|
end
|
136
|
-
|
137
|
-
def self.find_active_dirs(base_dirs=nil, also_mountpoints=true)
|
168
|
+
|
169
|
+
def self.find_active_dirs(base_dirs = nil, also_mountpoints = true)
|
138
170
|
if base_dirs.nil?
|
139
|
-
base_dirs = default_media_folders
|
171
|
+
base_dirs = default_media_folders
|
140
172
|
puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
|
141
173
|
end
|
142
174
|
active_dirs = []
|
143
|
-
base_dirs.each do |ugly_dir|
|
175
|
+
base_dirs.each do |ugly_dir|
|
144
176
|
# https://stackoverflow.com/questions/1899072/getting-a-list-of-folders-in-a-directory#:~:text=Dir.chdir(%27/destination_directory%27)%0ADir.glob(%27*%27).select%20%7B%7Cf%7C%20File.directory%3F%20f%7D
|
145
177
|
dir = File.expand_path(ugly_dir)
|
146
178
|
begin
|
147
|
-
x=[]
|
148
|
-
|
179
|
+
x = []
|
180
|
+
# puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
|
149
181
|
unless Dir.exists?(dir)
|
150
182
|
deb "Dir doesnt exist, skipping: #{dir}"
|
151
|
-
next
|
183
|
+
next
|
152
184
|
end
|
153
185
|
Dir.chdir(dir)
|
154
|
-
x = Dir.glob('*').select {|f| File.directory? f}
|
155
|
-
subdirs = x.map{|subdir|
|
156
|
-
subdirs.each{|subdir|
|
157
|
-
|
158
|
-
puts `ls -al "#{subdir}"`
|
186
|
+
x = Dir.glob('*').select { |f| File.directory? f }
|
187
|
+
subdirs = x.map { |subdir| "#{dir}#{subdir}" }
|
188
|
+
subdirs.each { |subdir|
|
189
|
+
deb "Subdir: #{subdir}"
|
190
|
+
puts `ls -al "#{subdir}"` # TODO refactor in exec
|
159
191
|
active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
|
160
192
|
}
|
161
|
-
#puts(white x)
|
193
|
+
# puts(white x)
|
162
194
|
rescue Exception => e # optionally: `rescue Exception => ex`
|
163
195
|
puts "Exception: '#{e}'"
|
164
196
|
ensure # will always get executed
|
165
|
-
#deb 'Always gets executed.'
|
166
|
-
#x = []
|
167
|
-
end
|
197
|
+
# deb 'Always gets executed.'
|
198
|
+
# x = []
|
199
|
+
end
|
168
200
|
end
|
169
201
|
end
|
170
202
|
|
@@ -175,16 +207,15 @@ module Storazzo
|
|
175
207
|
end
|
176
208
|
deb "[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once"
|
177
209
|
ConfigFiles.each do |papable_config_filename|
|
178
|
-
#return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
179
|
-
#return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
210
|
+
# return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
211
|
+
# return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
180
212
|
return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
181
213
|
end
|
182
214
|
deb "File not found! Neither #{ConfigFiles} exist.."
|
183
|
-
# return nil
|
215
|
+
# return nil
|
184
216
|
return DefaultConfigFile
|
185
217
|
end
|
186
218
|
|
187
|
-
|
188
219
|
# def self.compute_ricdisk_file_by_path_once(path)
|
189
220
|
# # unless @ricdisk_file.nil?
|
190
221
|
# # deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
|
@@ -199,7 +230,6 @@ module Storazzo
|
|
199
230
|
# return nil
|
200
231
|
# end
|
201
232
|
|
202
|
-
|
203
233
|
# # new
|
204
234
|
# def self.get_ricdisk_file_obsolete(path)
|
205
235
|
# if @ricdisk_file
|
@@ -215,9 +245,8 @@ module Storazzo
|
|
215
245
|
# return nil
|
216
246
|
# end
|
217
247
|
|
218
|
-
|
219
|
-
|
220
|
-
#https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
248
|
+
def self.interesting_mount_points(opts = {})
|
249
|
+
# https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
221
250
|
`mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
|
222
251
|
end
|
223
252
|
|
@@ -227,23 +256,23 @@ module Storazzo
|
|
227
256
|
deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{(Time.now - creation_time)} seconds ago")
|
228
257
|
(Time.now - creation_time).to_i
|
229
258
|
end
|
259
|
+
|
230
260
|
# maybe move to a RiccFile class? Maybe even INHERIT from FILE?
|
231
261
|
def obsolescence_days(file_path)
|
232
262
|
return obsolescence_seconds(file_path) / 86400
|
233
263
|
end
|
234
264
|
|
235
|
-
|
236
265
|
# FORMER SBRODOLA, now write_config_yaml_to_disk
|
237
|
-
#def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
238
|
-
def write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
266
|
+
# def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
267
|
+
def write_config_yaml_to_disk(subdir, opts = {}) # sbrodola_ricdisk(subdir)
|
239
268
|
# given a path, if .ricdisk exists i do stuff with it..
|
240
269
|
disk_info = nil
|
241
270
|
unless ok_dir? # self.ok_dir?(subdir)
|
242
|
-
|
243
|
-
return
|
271
|
+
warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
|
272
|
+
return
|
244
273
|
end
|
245
274
|
ConfigFiles.each do |papable_configfile_name|
|
246
|
-
if File.exists?(
|
275
|
+
if File.exists?("#{subdir}/#{papable_configfile_name}") and File.empty?("#{subdir}/#{papable_configfile_name}")
|
247
276
|
deb("Interesting. Empty file '#{papable_configfile_name}'! Now I write YAML with it.")
|
248
277
|
disk_info = RicDisk.new(subdir, papable_configfile_name)
|
249
278
|
end
|
@@ -258,7 +287,7 @@ module Storazzo
|
|
258
287
|
# puts(yellow disk_info.to_yaml)
|
259
288
|
# end
|
260
289
|
if disk_info.is_a?(RicDisk)
|
261
|
-
|
290
|
+
deb yellow("disk_info.class: #{disk_info.class}")
|
262
291
|
if File.empty?(disk_info.absolute_path) # and (disk_info.wr)
|
263
292
|
puts(green("yay, we can now write the file '#{disk_info.absolute_path}' (which is R/W, I just checked!) with proper YAML content.."))
|
264
293
|
if disk_info.wr
|
@@ -279,8 +308,8 @@ module Storazzo
|
|
279
308
|
puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
|
280
309
|
end
|
281
310
|
|
282
|
-
#disk_info.absolute_path
|
283
|
-
#if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
311
|
+
# disk_info.absolute_path
|
312
|
+
# if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
284
313
|
# if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
|
285
314
|
# puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
|
286
315
|
# puts(white `cat "#{disk_info.absolute_path}"`)
|
@@ -292,191 +321,134 @@ module Storazzo
|
|
292
321
|
|
293
322
|
# TODO obsolete this as i should NOT be calling it from clas, but from method.
|
294
323
|
def self.ok_dir?(subdir)
|
295
|
-
File.exists?(
|
324
|
+
File.exists?("#{subdir}/.ricdisk") or File.exists?("#{subdir}/.ricdisk.yaml")
|
296
325
|
end
|
297
326
|
|
298
|
-
def compute_stats_files(opts={})
|
299
|
-
|
300
|
-
|
301
|
-
|
327
|
+
def compute_stats_files(opts = {})
|
328
|
+
# Storazzo::RicDisk.calculate_stats_files(path, opts)
|
329
|
+
opts_upload_to_gcs = opts.fetch :upload_to_gcs, false
|
330
|
+
opts_force_rewrite = opts.fetch :force, false
|
331
|
+
opts_stats_file = opts.fetch :stats_file, "ricdisk_stats_v11.rds" # default. TODO point to proper..
|
302
332
|
dir = path
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
333
|
+
puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy. stats_file=#{opts_stats_file} dir=#{dir}")
|
334
|
+
|
335
|
+
full_file_path = "#{dir}/#{opts_stats_file}"
|
336
|
+
deb "This refactor is for another day. Actually no, TODAY "
|
337
|
+
pverbose true,
|
338
|
+
"TODO(ricc): you should compute more SMARTLY the full_file_path (#{full_file_path}): if its R/O it should be elsewhere.."
|
339
|
+
puts azure("- full_file_path: #{full_file_path}")
|
340
|
+
puts azure("- writeable?: #{writeable?}")
|
341
|
+
|
307
342
|
puts("compute_stats_files(#{white dir}): #{white full_file_path}")
|
308
|
-
|
343
|
+
deb "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
|
344
|
+
raise "Directory doesnt exist: #{dir}" unless File.directory?(dir)
|
345
|
+
|
309
346
|
Dir.chdir(dir)
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
347
|
+
puts azure `ls` # im curious
|
348
|
+
if File.exists?(full_file_path)
|
349
|
+
if opts_force_rewrite
|
350
|
+
# raise "TODO implement file exists and FORCE enabled"
|
351
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ReWrite enabled")
|
352
|
+
else # File.exists?(full_file_path) and (opts_force)
|
353
|
+
puts "File '#{opts_stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
|
354
|
+
# TODO check for file time...
|
355
|
+
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp}. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
|
356
|
+
if obsolescence_days(full_file_path) > 7
|
357
|
+
# puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
358
|
+
puts yellow("*** ACHTUNG *** FIle is pretty old. I'll force a rewrite")
|
359
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
|
360
|
+
"File older than 7 days. Indeed: #{obsolescence_days(full_file_path)}")
|
361
|
+
end
|
362
|
+
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
316
363
|
end
|
317
|
-
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
318
364
|
else
|
319
|
-
|
320
|
-
|
321
|
-
puts("[#{`pwd`.chomp}] Executing: #{azure command}")
|
322
|
-
ret = backquote_execute(command)
|
323
|
-
puts "Done. #{ret.split("\n").count} files processed."
|
365
|
+
deb("File doesnt exist..")
|
366
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ConfigFile doesn't exist")
|
324
367
|
end
|
325
368
|
end
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
full_file_path = "#{dir}/#{$stats_file}"
|
334
|
-
return "This refactor is for another day"
|
335
|
-
|
336
|
-
puts("calculate_stats_files(#{white dir}): #{white full_file_path}")
|
337
|
-
puts "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
|
338
|
-
Dir.chdir(dir)
|
339
|
-
if File.exists?(full_file_path) and ($opts[:force] == false)
|
340
|
-
puts "File '#{$stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
|
341
|
-
# TODO check for file time...
|
342
|
-
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp }. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
|
343
|
-
if obsolescence_days(full_file_path) > 7
|
344
|
-
puts("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
345
|
-
end
|
346
|
-
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
347
|
-
else
|
348
|
-
puts "Crunching data stats from '#{dir}' into '#{$stats_file}' ... please bear with me.. [maybe file didnt exist, maybe $opts[:force] is true]"
|
369
|
+
|
370
|
+
def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts = {})
|
371
|
+
max_lines = opts.fetch :max_lines, 42 # todo move to nil or -1
|
372
|
+
# full_file_path = "#{dir}/#{stats_file}"
|
373
|
+
puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
|
374
|
+
if max_lines < 0 # infinite
|
349
375
|
command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
350
|
-
|
351
|
-
|
352
|
-
|
376
|
+
else
|
377
|
+
# WOW! https://stackoverflow.com/questions/68599963/reliably-stop-bash-find-after-n-matches
|
378
|
+
# find . -type f -iname "*.txt" -print0 |
|
379
|
+
# head -z -n 10 |
|
380
|
+
# xargs -r0 myscript.sh
|
381
|
+
if mac?
|
382
|
+
puts red("Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)")
|
383
|
+
spannometric_lines = (max_lines / 10)
|
384
|
+
command = "find . -print0 | head -n '#{spannometric_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
385
|
+
else
|
386
|
+
command = "find . -print0 | head -z -n '#{max_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
387
|
+
end
|
353
388
|
end
|
389
|
+
puts("[#{`pwd`.chomp}] Executing: #{azure command}")
|
390
|
+
ret = backquote_execute(command)
|
391
|
+
puts "Done. #{ret.split("\n").count} files processed."
|
354
392
|
end
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
# Example output from mount:
|
359
|
-
|
360
|
-
# devfs on /dev (devfs, local, nobrowse)
|
361
|
-
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
362
|
-
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
363
|
-
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
364
|
-
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
365
|
-
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
366
|
-
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
367
|
-
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
368
|
-
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
369
|
-
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
370
|
-
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
371
|
-
# =end
|
372
|
-
# # add directories from current mountpoints...
|
373
|
-
# mount_table_lines = interesting_mount_points()
|
374
|
-
# mount_table_lines.each{|line|
|
375
|
-
# next if line =~ /^map /
|
376
|
-
# dev, on, path, mode = line.split(/ /)
|
377
|
-
# #puts line
|
378
|
-
# #deb yellow(path)
|
379
|
-
# active_dirs << path if self.ok_dir?(path)
|
380
|
-
# }
|
381
|
-
# end
|
382
|
-
# active_dirs.uniq!
|
383
|
-
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
384
|
-
# return active_dirs
|
385
|
-
# end
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
end #/Class
|
390
|
-
end #/Module
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
# def initialize(path, ricdisk_file)
|
397
|
-
# puts "[DEB] RicDisk initialize.. path=#{path}"
|
398
|
-
# @local_mountpoint = path
|
399
|
-
# @description = "This is an automated RicDisk description from v.#{VERSION}. Riccardo feel free to edit away with characteristicshs of this device.. Created on #{Time.now}'"
|
400
|
-
# @ricdisk_version = VERSION
|
401
|
-
# @ricdisk_file = ricdisk_file
|
402
|
-
# #@questo_non_esiste = :sobenme
|
403
|
-
# @label = path.split("/").last
|
404
|
-
# @name = path.split("/").last
|
405
|
-
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
406
|
-
# @tags = 'ricdisk'
|
407
|
-
# puts :beleza
|
408
|
-
# @config = RicDiskConfig.instance.get_config
|
409
|
-
# #puts @config if @config
|
410
|
-
# find_info_from_mount(path)
|
411
|
-
# find_info_from_df()
|
412
|
-
# end
|
413
|
-
|
414
|
-
# def ricdisk_absolute_path
|
415
|
-
# @local_mountpoint + "/" + @ricdisk_file
|
416
|
-
# end
|
417
|
-
|
418
|
-
# def add_tag(tag)
|
419
|
-
# @tags += ", #{tag}"
|
420
|
-
# end
|
421
|
-
|
422
|
-
# # might have other things in the future...
|
423
|
-
# def find_info_from_mount(path)
|
424
|
-
# mount_table_lines = interesting_mount_points()
|
425
|
-
# mount_line = nil
|
426
|
-
# mount_table_lines.each do |line|
|
427
|
-
# next if line =~ /^map /
|
428
|
-
# dev, on, mount_path, mode = line.split(/ /)
|
429
|
-
# if mount_path==path
|
430
|
-
# mount_line = line
|
431
|
-
# else
|
432
|
-
# @info_from_mount = false
|
433
|
-
# end
|
434
|
-
# end
|
435
|
-
# @info_from_mount = ! (mount_line.nil?)
|
436
|
-
# if @info_from_mount
|
437
|
-
# #@mount_line = mount_line
|
438
|
-
# @description += "\nMount line:\n" + mount_line
|
439
|
-
# @remote_mountpoint = mount_line.split(/ /)[0]
|
440
|
-
# @fstype = mount_line.split(/ /)[3].gsub(/[\(,]/, '')
|
441
|
-
# add_tag(:synology) if @remote_mountpoint.match('1.0.1.10')
|
442
|
-
# end
|
443
|
-
# end
|
444
|
-
|
445
|
-
# def find_info_from_df()
|
446
|
-
# path = @local_mountpoint
|
447
|
-
# df_info = `df -h "#{path}"`
|
448
|
-
# @df_info = df_info
|
449
|
-
# lines = df_info.split(/\n+/)
|
450
|
-
# raise "I need exactly TWO lines! Or no info is served here..." unless lines.size == 2
|
451
|
-
# mount, @size_readable, used_size, avail_size, @disk_utilization, other = lines[1].split(/\s+/) # second line..
|
452
|
-
# end
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
458
|
-
|
459
|
-
def backquote_execute(cmd, opts={})
|
460
|
-
dryrun = opts.fetch :dryrun, false
|
461
|
-
# executed a command wrapped by dryrun though
|
462
|
-
return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
|
463
|
-
`#{cmd}`
|
464
|
-
end
|
465
|
-
|
466
|
-
def upload_to_gcs(file, opts={})
|
467
|
-
deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
|
468
|
-
mount_name = file.split('/')[-2]
|
469
|
-
filename = "#{mount_name}-#{File.basename file}"
|
470
|
-
hostname = Socket.gethostname[/^[^.]+/]
|
471
|
-
command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{ hostname }-#{filename}"
|
472
|
-
deb("Command: #{command}")
|
473
|
-
puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
|
474
|
-
ret = backquote_execute(command, :dryrun => true)
|
475
|
-
# if $opts[:debug] do
|
476
|
-
# puts "+ Current list of files:"
|
477
|
-
# ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
|
478
|
-
# puts ret
|
479
|
-
# end
|
480
|
-
ret
|
393
|
+
|
394
|
+
def self.calculate_stats_files_DUPLICATE_STATIC(dir, opts = {})
|
395
|
+
raise "Please use object instead. If you cant, please move the object code to STATIC and dedupe code!"
|
481
396
|
end
|
482
|
-
|
397
|
+
|
398
|
+
# if also_mountpoints
|
399
|
+
# =begin
|
400
|
+
# Example output from mount:
|
401
|
+
|
402
|
+
# devfs on /dev (devfs, local, nobrowse)
|
403
|
+
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
404
|
+
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
405
|
+
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
406
|
+
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
407
|
+
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
408
|
+
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
409
|
+
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
410
|
+
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
411
|
+
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
412
|
+
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
413
|
+
# =end
|
414
|
+
# # add directories from current mountpoints...
|
415
|
+
# mount_table_lines = interesting_mount_points()
|
416
|
+
# mount_table_lines.each{|line|
|
417
|
+
# next if line =~ /^map /
|
418
|
+
# dev, on, path, mode = line.split(/ /)
|
419
|
+
# #puts line
|
420
|
+
# #deb yellow(path)
|
421
|
+
# active_dirs << path if self.ok_dir?(path)
|
422
|
+
# }
|
423
|
+
# end
|
424
|
+
# active_dirs.uniq!
|
425
|
+
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
426
|
+
# return active_dirs
|
427
|
+
# end
|
428
|
+
end # /Class
|
429
|
+
end # /Module
|
430
|
+
|
431
|
+
def backquote_execute(cmd, opts = {})
|
432
|
+
dryrun = opts.fetch :dryrun, false
|
433
|
+
# executed a command wrapped by dryrun though
|
434
|
+
return "DRYRUN backquote_execute(#{cmd})" if dryrun # $opts[:dryrun]
|
435
|
+
|
436
|
+
`#{cmd}`
|
437
|
+
end
|
438
|
+
|
439
|
+
def upload_to_gcs(file, opts = {})
|
440
|
+
deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
|
441
|
+
mount_name = file.split('/')[-2]
|
442
|
+
filename = "#{mount_name}-#{File.basename file}"
|
443
|
+
hostname = Socket.gethostname[/^[^.]+/]
|
444
|
+
command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{hostname}-#{filename}"
|
445
|
+
deb("Command: #{command}")
|
446
|
+
puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
|
447
|
+
ret = backquote_execute(command, :dryrun => true)
|
448
|
+
# if $opts[:debug] do
|
449
|
+
# puts "+ Current list of files:"
|
450
|
+
# ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
|
451
|
+
# puts ret
|
452
|
+
# end
|
453
|
+
ret
|
454
|
+
end
|