storazzo 0.5.7 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +7 -5
- data/Makefile +8 -6
- data/Rakefile +15 -13
- data/VERSION +1 -1
- data/bin/hello-storazzo +1 -0
- data/bin/ricdisk-magic +33 -30
- data/bin/stats-with-md5 +182 -157
- data/bin/storazzo +112 -114
- data/bin/storazzo-symlink.rb +1 -0
- data/lib/storazzo/colors.rb +77 -36
- data/lib/storazzo/common.rb +72 -68
- data/lib/storazzo/debug.rb +2 -0
- data/lib/storazzo/hashify.rb +7 -5
- data/lib/storazzo/main.rb +59 -49
- data/lib/storazzo/media/abstract_ric_disk.rb +188 -179
- data/lib/storazzo/media/gcs_bucket.rb +76 -58
- data/lib/storazzo/media/local_folder.rb +56 -42
- data/lib/storazzo/media/mount_point.rb +20 -12
- data/lib/storazzo/ric_disk.rb +386 -383
- data/lib/storazzo/ric_disk_config.rb +227 -209
- data/lib/storazzo/ric_disk_sample_config.rb +26 -24
- data/lib/storazzo/ric_disk_statsfile.rb +19 -17
- data/lib/storazzo/ric_disk_ugly.rb +1 -0
- data/lib/storazzo/version.rb +3 -3
- data/lib/storazzo.rb +6 -7
- data/storazzo.gemspec +24 -16
- data/test/benchmark/for_future_use.rb +4 -2
- data/test/benchmark/test_hashing_functions-speed.rb +17 -0
- data/test/bin/new-idea.rb +17 -0
- data/test/bin/storazzo.rb +22 -25
- data/test/media/test_abstract_ric_disk.rb +5 -3
- data/test/media/test_gcs_bucket.rb +24 -23
- data/test/media/test_local_folder.rb +24 -23
- data/test/media/test_mount_point.rb +6 -5
- data/test/test_ric_disk.rb +6 -4
- data/test/test_ric_disk_config.rb +12 -11
- data/test/test_ric_disk_stats_file.rb +5 -3
- data/test/test_storazzo.rb +6 -4
- data/var/test/disks/disk02-full/Rakefile +7 -5
- data/var/test/disks/ricdisk_stats_v11.rds +11 -0
- metadata +13 -21
data/lib/storazzo/ric_disk.rb
CHANGED
@@ -1,432 +1,436 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# A RicDisk wraps a local mount/disk/folder
|
2
4
|
# it's considered interesting if there's a ".ricdisk/.ricdisk"
|
3
5
|
|
4
6
|
require 'digest'
|
5
7
|
|
6
8
|
module Storazzo
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
def initialize(ric_disk_object, opts = {})
|
46
|
-
verbose = opts.fetch :verbose, true
|
47
|
-
pverbose verbose,
|
48
|
-
"This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
|
49
|
-
raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
|
50
|
-
|
51
|
-
# ok back to business, now path is a String :)
|
52
|
-
path = ric_disk_object.path
|
53
|
-
deb "RicDisk initialize.. path=#{path}"
|
54
|
-
deb "RicDisk initialize.. path=#{path}"
|
55
|
-
@local_mountpoint = File.expand_path(path)
|
56
|
-
@ard = ric_disk_object # AbstractRicDiskObject
|
57
|
-
@description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
|
58
|
-
@ricdisk_version = RicdiskVersion
|
59
|
-
@ricdisk_file = compute_ricdisk_file() # Storazzo::RicDisk.get_ricdisk_file(path)
|
60
|
-
@ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
|
61
|
-
@label = path.split("/").last
|
62
|
-
@name = path.split("/").last
|
63
|
-
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
64
|
-
# @wr = writeable?
|
65
|
-
@tags = ['ricdisk', 'storazzo']
|
66
|
-
@size = RicDisk._compute_size_could_take_long(path)
|
67
|
-
@unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
68
|
-
@computation_hostname = Socket.gethostname
|
69
|
-
@created_at = Time.now
|
70
|
-
|
71
|
-
@ricdisk_file_empty = ricdisk_file_empty?
|
72
|
-
|
73
|
-
# @config = RicDiskConfig.instance.get_config
|
74
|
-
# #puts @config if @config
|
75
|
-
# find_info_from_mount(path)
|
76
|
-
deb "RicDisk initialize. to_s: #{self}"
|
77
|
-
# find_info_from_df()
|
78
|
-
end
|
9
|
+
module Storazzo
|
10
|
+
class RicDisk
|
11
|
+
include Hashify
|
12
|
+
include Storazzo::Common
|
13
|
+
extend Storazzo::Common
|
14
|
+
extend Storazzo::Colors
|
15
|
+
require 'socket'
|
16
|
+
|
17
|
+
## Instance variables
|
18
|
+
|
19
|
+
# in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
|
20
|
+
# These are the files I do accept.
|
21
|
+
ConfigFiles = %w[ricdisk.yaml .ricdisk storazzo.yaml].freeze
|
22
|
+
DefaultConfigFile = 'storazzo.yaml' # .ricdisk }
|
23
|
+
RicdiskVersion = '2.1'
|
24
|
+
RicdiskHistory = [
|
25
|
+
'2022-07-29 2.1 Added timestamp',
|
26
|
+
'2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...'
|
27
|
+
].freeze
|
28
|
+
DefaultGemfileTestDiskFolder = "#{Storazzo.root}/var/test/disks/" # was: @@default_gemfile_test_disks_folder
|
29
|
+
# Immutable
|
30
|
+
DefaultMediaFolders = %w[
|
31
|
+
/Volumes/
|
32
|
+
/mnt/
|
33
|
+
].append(DefaultGemfileTestDiskFolder).append("/media/#{ENV['USER']}/")
|
34
|
+
|
35
|
+
# # todo substitute with protobuf..
|
36
|
+
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
37
|
+
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
38
|
+
:unique_hash # new 202207
|
39
|
+
|
40
|
+
################################
|
41
|
+
## INSTANCE methods
|
42
|
+
################################
|
43
|
+
|
44
|
+
def initialize_old_way(_path, _opts = {})
|
45
|
+
raise 'Now I dont want a string in input, I want an OBJECT < Storazzo::Media::AbstractRicDisk'
|
46
|
+
end
|
79
47
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
48
|
+
def initialize(ric_disk_object, opts = {})
|
49
|
+
verbose = opts.fetch :verbose, true
|
50
|
+
pverbose verbose,
|
51
|
+
"This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
|
52
|
+
raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
|
53
|
+
|
54
|
+
# ok back to business, now path is a String :)
|
55
|
+
path = ric_disk_object.path
|
56
|
+
deb "RicDisk initialize.. path=#{path}"
|
57
|
+
deb "RicDisk initialize.. path=#{path}"
|
58
|
+
@local_mountpoint = File.expand_path(path)
|
59
|
+
@ard = ric_disk_object # AbstractRicDiskObject
|
60
|
+
@description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
|
61
|
+
@ricdisk_version = RicdiskVersion
|
62
|
+
@ricdisk_file = compute_ricdisk_file # Storazzo::RicDisk.get_ricdisk_file(path)
|
63
|
+
@ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
|
64
|
+
@label = path.split('/').last
|
65
|
+
@name = path.split('/').last
|
66
|
+
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
67
|
+
# @wr = writeable?
|
68
|
+
@tags = %w[ricdisk storazzo]
|
69
|
+
@size = RicDisk._compute_size_could_take_long(path)
|
70
|
+
@unique_hash = "MD5::#{Digest::MD5.hexdigest(File.expand_path(path))}" # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
71
|
+
@computation_hostname = Socket.gethostname
|
72
|
+
@created_at = Time.now
|
73
|
+
|
74
|
+
@ricdisk_file_empty = ricdisk_file_empty?
|
75
|
+
|
76
|
+
# @config = RicDiskConfig.instance.get_config
|
77
|
+
# #puts @config if @config
|
78
|
+
# find_info_from_mount(path)
|
79
|
+
deb "RicDisk initialize. to_s: #{self}"
|
80
|
+
# find_info_from_df()
|
81
|
+
end
|
85
82
|
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
def ricdisk_file_empty?
|
84
|
+
# File.empty?("#{local_mountpoint}/.ricdisk.yaml")
|
85
|
+
puts "compute_ricdisk_file: #{compute_ricdisk_file}"
|
86
|
+
File.empty?(compute_ricdisk_file.to_s) # was (get_ricdisk_file)
|
87
|
+
end
|
89
88
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
puts "2. Sbrodoling everything: :TODO"
|
94
|
-
# find_info_from_mount(path)
|
95
|
-
# find_info_from_df()
|
96
|
-
end
|
89
|
+
def ok_dir?
|
90
|
+
!ricdisk_file.nil?
|
91
|
+
end
|
97
92
|
|
98
|
-
|
99
|
-
|
100
|
-
|
93
|
+
def analyze_local_system
|
94
|
+
puts 'TODO This should analyzze the WHOLE system. TODO(ricc): move to another object which has to do with the system/computer.'
|
95
|
+
puts "1. Interesting Mounts: #{green interesting_mount_points}"
|
96
|
+
puts '2. Sbrodoling everything: :TODO'
|
97
|
+
# find_info_from_mount(path)
|
98
|
+
# find_info_from_df()
|
99
|
+
end
|
101
100
|
|
102
|
-
|
103
|
-
|
104
|
-
|
101
|
+
def path
|
102
|
+
local_mountpoint
|
103
|
+
end
|
105
104
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
# end
|
110
|
-
def self._compute_size_could_take_long(my_path)
|
111
|
-
deb "Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds."
|
112
|
-
puts azure('could take long. Please take precautions like forking with timeout of 5sec')
|
113
|
-
`du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
|
114
|
-
end
|
105
|
+
def to_s
|
106
|
+
"RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version}, ard=#{@ard})"
|
107
|
+
end
|
115
108
|
|
116
|
-
|
117
|
-
#
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
puts("absolute_path: #{azure absolute_path}")
|
126
|
-
puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
|
127
|
-
bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
|
128
|
-
puts("bash_output: #{azure bash_output}")
|
129
|
-
# @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
|
130
|
-
raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
|
131
|
-
|
132
|
-
@wr = File.writable?(absolute_path) # rescue false
|
133
|
-
return @wr
|
134
|
-
# :boh_todo_fix_me_and_compute
|
135
|
-
# false
|
136
|
-
end
|
109
|
+
# could take long..
|
110
|
+
# def size
|
111
|
+
# `du -s '#{path}'`.split(/\s/)[0]
|
112
|
+
# end
|
113
|
+
def self._compute_size_could_take_long(my_path)
|
114
|
+
deb 'Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds.'
|
115
|
+
puts azure('could take long. Please take precautions like forking with timeout of 5sec')
|
116
|
+
`du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
|
117
|
+
end
|
137
118
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
119
|
+
def writeable?
|
120
|
+
# memoize
|
121
|
+
return @wr unless @wr.nil?
|
122
|
+
|
123
|
+
# NOW: CALCULATE it
|
124
|
+
# Now I can do ONCE an EXPENSIVE calculation
|
125
|
+
puts yellow("[RicDisk.writeable] TODO(ricc): Do expensive calculation if this FS is writeable: #{path} and write/memoize it on @wr once and for all")
|
126
|
+
puts yellow('[RicDisk.writeable] I have a feeling this should be delegated to praecipuus Storazzo::Media::Object we refer to (WR is different on GCS vs Local):') # infinite loop dammit #{self.to_verbose_s}")
|
127
|
+
puts("Dir: #{azure path}")
|
128
|
+
puts("absolute_path: #{azure absolute_path}")
|
129
|
+
puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
|
130
|
+
bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
|
131
|
+
puts("bash_output: #{azure bash_output}")
|
132
|
+
# @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
|
133
|
+
raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
|
134
|
+
|
135
|
+
@wr = File.writable?(absolute_path) # rescue false
|
136
|
+
@wr
|
137
|
+
# :boh_todo_fix_me_and_compute
|
138
|
+
# false
|
139
|
+
end
|
147
140
|
|
148
|
-
|
149
|
-
|
150
|
-
|
141
|
+
def to_verbose_s
|
142
|
+
h = {}
|
143
|
+
h[:to_s] = to_s
|
144
|
+
h[:wr] = wr
|
145
|
+
h[:inspect] = inspect
|
146
|
+
h[:writeable] = writeable?
|
147
|
+
h[:ard] = @ard
|
148
|
+
h
|
149
|
+
end
|
151
150
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
end
|
151
|
+
################################
|
152
|
+
## CLASS methods
|
153
|
+
################################
|
156
154
|
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
# d.find_active_dirs()
|
162
|
-
end
|
155
|
+
# All places where to find for something :)
|
156
|
+
def self.default_media_folders
|
157
|
+
DefaultMediaFolders # was DEFAULT_MEDIA_FOLDERS
|
158
|
+
end
|
163
159
|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
160
|
+
# _localgem_disks
|
161
|
+
def self.test
|
162
|
+
d = RicDisk.new(DefaultGemfileTestDiskFolder)
|
163
|
+
puts(d)
|
164
|
+
puts "do something with it: #{d}"
|
165
|
+
# d.find_active_dirs()
|
166
|
+
end
|
168
167
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
|
168
|
+
def absolute_path
|
169
|
+
# @local_mountpoint + "/" + @ricdisk_file
|
170
|
+
"#{local_mountpoint}/#{ricdisk_file}"
|
173
171
|
end
|
174
|
-
|
175
|
-
base_dirs
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
172
|
+
|
173
|
+
def self.find_active_dirs(base_dirs = nil, _also_mountpoints = true)
|
174
|
+
if base_dirs.nil?
|
175
|
+
base_dirs = default_media_folders
|
176
|
+
puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
|
177
|
+
end
|
178
|
+
active_dirs = []
|
179
|
+
base_dirs.each do |ugly_dir|
|
180
|
+
# https://stackoverflow.com/questions/1899072/getting-a-list-of-folders-in-a-directory#:~:text=Dir.chdir(%27/destination_directory%27)%0ADir.glob(%27*%27).select%20%7B%7Cf%7C%20File.directory%3F%20f%7D
|
181
|
+
dir = File.expand_path(ugly_dir)
|
182
|
+
begin
|
183
|
+
x = []
|
184
|
+
# puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
|
185
|
+
unless Dir.exist?(dir)
|
186
|
+
deb "Dir doesnt exist, skipping: #{dir}"
|
187
|
+
next
|
188
|
+
end
|
189
|
+
Dir.chdir(dir)
|
190
|
+
x = Dir.glob('*').select { |f| File.directory? f }
|
191
|
+
subdirs = x.map { |subdir| "#{dir}#{subdir}" }
|
192
|
+
subdirs.each do |subdir|
|
193
|
+
deb "Subdir: #{subdir}"
|
194
|
+
puts `ls -al "#{subdir}"` # TODO: refactor in exec
|
195
|
+
active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
|
196
|
+
end
|
197
|
+
# puts(white x)
|
198
|
+
rescue Exception => e # optionally: `rescue Exception => ex`
|
199
|
+
puts "Exception: '#{e}'"
|
200
|
+
# will always get executed
|
201
|
+
# deb 'Always gets executed.'
|
202
|
+
# x = []
|
184
203
|
end
|
185
|
-
Dir.chdir(dir)
|
186
|
-
x = Dir.glob('*').select { |f| File.directory? f }
|
187
|
-
subdirs = x.map { |subdir| "#{dir}#{subdir}" }
|
188
|
-
subdirs.each { |subdir|
|
189
|
-
deb "Subdir: #{subdir}"
|
190
|
-
puts `ls -al "#{subdir}"` # TODO refactor in exec
|
191
|
-
active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
|
192
|
-
}
|
193
|
-
# puts(white x)
|
194
|
-
rescue Exception => e # optionally: `rescue Exception => ex`
|
195
|
-
puts "Exception: '#{e}'"
|
196
|
-
ensure # will always get executed
|
197
|
-
# deb 'Always gets executed.'
|
198
|
-
# x = []
|
199
204
|
end
|
200
205
|
end
|
201
|
-
end
|
202
206
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
207
|
+
def compute_ricdisk_file
|
208
|
+
unless @ricdisk_file.nil?
|
209
|
+
deb '[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)'
|
210
|
+
return @ricdisk_file
|
211
|
+
end
|
212
|
+
deb '[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once'
|
213
|
+
ConfigFiles.each do |papable_config_filename|
|
214
|
+
# return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
215
|
+
# return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
216
|
+
return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}")
|
217
|
+
end
|
218
|
+
deb "File not found! Neither #{ConfigFiles} exist.."
|
219
|
+
# return nil
|
220
|
+
DefaultConfigFile
|
213
221
|
end
|
214
|
-
deb "File not found! Neither #{ConfigFiles} exist.."
|
215
|
-
# return nil
|
216
|
-
return DefaultConfigFile
|
217
|
-
end
|
218
222
|
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
# # new
|
234
|
-
# def self.get_ricdisk_file_obsolete(path)
|
235
|
-
# if @ricdisk_file
|
236
|
-
# puts "[CACHE HIT] ricdisk_file"
|
237
|
-
# return @ricdisk_file
|
238
|
-
# end
|
239
|
-
# puts "RICC_WARNING This requires cmputation I wanna do it almost once"
|
240
|
-
# ConfigFiles.each do |papable_config_filename|
|
241
|
-
# #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
242
|
-
# #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
243
|
-
# return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
244
|
-
# end
|
245
|
-
# return nil
|
246
|
-
# end
|
247
|
-
|
248
|
-
def self.interesting_mount_points(opts = {})
|
249
|
-
# https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
250
|
-
`mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
|
251
|
-
end
|
223
|
+
# def self.compute_ricdisk_file_by_path_once(path)
|
224
|
+
# # unless @ricdisk_file.nil?
|
225
|
+
# # deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
|
226
|
+
# # return @ricdisk_file
|
227
|
+
# # end
|
228
|
+
# warn "RICC_WARNING This requires cmputation I wanna do it almost once"
|
229
|
+
# ConfigFiles.each do |papable_config_filename|
|
230
|
+
# #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
231
|
+
# #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
232
|
+
# return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
233
|
+
# end
|
234
|
+
# return nil
|
235
|
+
# end
|
252
236
|
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
237
|
+
# # new
|
238
|
+
# def self.get_ricdisk_file_obsolete(path)
|
239
|
+
# if @ricdisk_file
|
240
|
+
# puts "[CACHE HIT] ricdisk_file"
|
241
|
+
# return @ricdisk_file
|
242
|
+
# end
|
243
|
+
# puts "RICC_WARNING This requires cmputation I wanna do it almost once"
|
244
|
+
# ConfigFiles.each do |papable_config_filename|
|
245
|
+
# #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
246
|
+
# #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
247
|
+
# return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
248
|
+
# end
|
249
|
+
# return nil
|
250
|
+
# end
|
259
251
|
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
252
|
+
def self.interesting_mount_points(_opts = {})
|
253
|
+
# https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
254
|
+
`mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
|
255
|
+
end
|
264
256
|
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
unless ok_dir? # self.ok_dir?(subdir)
|
271
|
-
warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
|
272
|
-
return
|
257
|
+
# maybe move to a RiccFile class? Maybe even INHERIT from FILE?
|
258
|
+
def obsolescence_seconds(file_path)
|
259
|
+
creation_time = File.stat(file_path).ctime
|
260
|
+
deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{Time.now - creation_time} seconds ago")
|
261
|
+
(Time.now - creation_time).to_i
|
273
262
|
end
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
end
|
263
|
+
|
264
|
+
# maybe move to a RiccFile class? Maybe even INHERIT from FILE?
|
265
|
+
def obsolescence_days(file_path)
|
266
|
+
obsolescence_seconds(file_path) / 86_400
|
279
267
|
end
|
280
|
-
|
281
|
-
#
|
282
|
-
#
|
283
|
-
#
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
268
|
+
|
269
|
+
# FORMER SBRODOLA, now write_config_yaml_to_disk
|
270
|
+
# def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
271
|
+
# sbrodola_ricdisk(subdir)
|
272
|
+
def write_config_yaml_to_disk(subdir, _opts = {})
|
273
|
+
# given a path, if .ricdisk exists i do stuff with it..
|
274
|
+
disk_info = nil
|
275
|
+
unless ok_dir? # self.ok_dir?(subdir)
|
276
|
+
warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
|
277
|
+
return
|
278
|
+
end
|
279
|
+
ConfigFiles.each do |papable_configfile_name|
|
280
|
+
if File.exist?("#{subdir}/#{papable_configfile_name}") && File.empty?("#{subdir}/#{papable_configfile_name}")
|
281
|
+
deb("Interesting. Empty file '#{papable_configfile_name}'! Now I write YAML with it.")
|
282
|
+
disk_info = RicDisk.new(subdir, papable_configfile_name)
|
283
|
+
end
|
284
|
+
end
|
285
|
+
# if File.exists?( "#{subdir}/.ricdisk") and File.empty?( "#{subdir}/.ricdisk")
|
286
|
+
# deb("Interesting1. Empty file! Now I write YAML with it.")
|
287
|
+
# disk_info = RicDisk.new(subdir, '.ricdisk')
|
288
|
+
# end
|
289
|
+
# if File.exists?( "#{subdir}/.ricdisk.yaml") and File.empty?( "#{subdir}/.ricdisk.yaml")
|
290
|
+
# deb("Interesting2. Empty file! TODO write YAML with it.")
|
291
|
+
# disk_info = RicDisk.new(subdir, '.ricdisk.yaml')
|
292
|
+
# puts(yellow disk_info.to_yaml)
|
293
|
+
# end
|
294
|
+
if disk_info.is_a?(RicDisk)
|
295
|
+
deb yellow("disk_info.class: #{disk_info.class}")
|
296
|
+
if File.empty?(disk_info.absolute_path) # and (disk_info.wr)
|
297
|
+
puts(green("yay, we can now write the file '#{disk_info.absolute_path}' (which is R/W, I just checked!) with proper YAML content.."))
|
298
|
+
if disk_info.wr
|
299
|
+
ret = File.write(disk_info.absolute_path, disk_info.obj_to_yaml)
|
300
|
+
puts green("Written file! ret=#{ret}")
|
301
|
+
else
|
302
|
+
raise 'TODO_IMPLEMENT: write in proper place in config dir'
|
303
|
+
puts red('TODO implement me')
|
304
|
+
end
|
296
305
|
else
|
297
|
-
|
298
|
-
puts
|
306
|
+
puts(red("Something not right here: either file is NOT empty or disk is NOT writeable.. #{File.empty?(disk_info.absolute_path)}"))
|
307
|
+
puts("File size: #{File.size(disk_info.absolute_path)}")
|
308
|
+
puts(disk_info.to_s)
|
309
|
+
puts(disk_info.obj_to_hash)
|
310
|
+
puts(disk_info.obj_to_yaml)
|
299
311
|
end
|
300
|
-
else
|
301
|
-
puts
|
302
|
-
puts("File size: #{File.size(disk_info.absolute_path)}")
|
303
|
-
puts(disk_info.to_s)
|
304
|
-
puts(disk_info.obj_to_hash)
|
305
|
-
puts(disk_info.obj_to_yaml)
|
312
|
+
else # not a RicDisk..
|
313
|
+
puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
|
306
314
|
end
|
307
|
-
else # not a RicDisk..
|
308
|
-
puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
|
309
|
-
end
|
310
315
|
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
316
|
+
# disk_info.absolute_path
|
317
|
+
# if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
318
|
+
# if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
|
319
|
+
# puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
|
320
|
+
# puts(white `cat "#{disk_info.absolute_path}"`)
|
321
|
+
# else
|
322
|
+
# puts "WRITING NOW. [I BELIEVE THIS IS DUPE CODE - see a few lines above!] disk_info.obj_to_yaml .. to #{compute_ricdisk_file}"
|
323
|
+
# File.open(ricdisk_file_full, 'w').write(disk_info.obj_to_yaml)
|
324
|
+
# end
|
325
|
+
end
|
321
326
|
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
327
|
+
# TODO: obsolete this as i should NOT be calling it from clas, but from method.
|
328
|
+
def self.ok_dir?(subdir)
|
329
|
+
File.exist?("#{subdir}/.ricdisk") or File.exist?("#{subdir}/.ricdisk.yaml")
|
330
|
+
end
|
326
331
|
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
332
|
+
def compute_stats_files(opts = {})
|
333
|
+
# Storazzo::RicDisk.calculate_stats_files(path, opts)
|
334
|
+
opts_upload_to_gcs = opts.fetch :upload_to_gcs, false
|
335
|
+
opts_force_rewrite = opts.fetch :force, false
|
336
|
+
opts_stats_file = opts.fetch :stats_file, 'ricdisk_stats_v11.rds' # default. TODO point to proper..
|
337
|
+
dir = path
|
338
|
+
puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy. stats_file=#{opts_stats_file} dir=#{dir}")
|
339
|
+
|
340
|
+
full_file_path = "#{dir}/#{opts_stats_file}"
|
341
|
+
deb 'This refactor is for another day. Actually no, TODAY '
|
342
|
+
pverbose true,
|
343
|
+
"TODO(ricc): you should compute more SMARTLY the full_file_path (#{full_file_path}): if its R/O it should be elsewhere.."
|
344
|
+
puts azure("- full_file_path: #{full_file_path}")
|
345
|
+
puts azure("- writeable?: #{writeable?}")
|
346
|
+
|
347
|
+
puts("compute_stats_files(#{white dir}): #{white full_file_path}")
|
348
|
+
deb "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
|
349
|
+
raise "Directory doesnt exist: #{dir}" unless File.directory?(dir)
|
350
|
+
|
351
|
+
Dir.chdir(dir)
|
352
|
+
puts azure `ls` # im curious
|
353
|
+
if File.exist?(full_file_path)
|
354
|
+
if opts_force_rewrite
|
355
|
+
# raise "TODO implement file exists and FORCE enabled"
|
356
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, 'ReWrite enabled')
|
357
|
+
else # File.exists?(full_file_path) and (opts_force)
|
358
|
+
puts "File '#{opts_stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
|
359
|
+
# TODO check for file time...
|
360
|
+
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp}. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
|
361
|
+
if obsolescence_days(full_file_path) > 7
|
362
|
+
# puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
363
|
+
puts yellow("*** ACHTUNG *** FIle is pretty old. I'll force a rewrite")
|
364
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
|
365
|
+
"File older than 7 days. Indeed: #{obsolescence_days(full_file_path)}")
|
366
|
+
end
|
367
|
+
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
361
368
|
end
|
362
|
-
|
369
|
+
else
|
370
|
+
deb('File doesnt exist..')
|
371
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ConfigFile doesn't exist")
|
363
372
|
end
|
364
|
-
else
|
365
|
-
deb("File doesnt exist..")
|
366
|
-
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ConfigFile doesn't exist")
|
367
373
|
end
|
368
|
-
end
|
369
374
|
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
puts red("Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)")
|
375
|
+
def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts = {})
|
376
|
+
max_lines = opts.fetch :max_lines, 42 # TODO: move to nil or -1
|
377
|
+
# full_file_path = "#{dir}/#{stats_file}"
|
378
|
+
puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
|
379
|
+
if max_lines.negative? # infinite
|
380
|
+
command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
381
|
+
elsif mac?
|
382
|
+
# WOW! https://stackoverflow.com/questions/68599963/reliably-stop-bash-find-after-n-matches
|
383
|
+
# find . -type f -iname "*.txt" -print0 |
|
384
|
+
# head -z -n 10 |
|
385
|
+
# xargs -r0 myscript.sh
|
386
|
+
puts red('Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)')
|
383
387
|
spannometric_lines = (max_lines / 10)
|
384
388
|
command = "find . -print0 | head -n '#{spannometric_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
385
389
|
else
|
386
390
|
command = "find . -print0 | head -z -n '#{max_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
387
391
|
end
|
392
|
+
puts("[#{`pwd`.chomp}] Executing: #{azure command}")
|
393
|
+
ret = backquote_execute(command)
|
394
|
+
puts "Done. #{ret.split("\n").count} files processed."
|
388
395
|
end
|
389
|
-
puts("[#{`pwd`.chomp}] Executing: #{azure command}")
|
390
|
-
ret = backquote_execute(command)
|
391
|
-
puts "Done. #{ret.split("\n").count} files processed."
|
392
|
-
end
|
393
396
|
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
+
def self.calculate_stats_files_DUPLICATE_STATIC(_dir, _opts = {})
|
398
|
+
raise 'Please use object instead. If you cant, please move the object code to STATIC and dedupe code!'
|
399
|
+
end
|
397
400
|
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
end
|
401
|
+
# if also_mountpoints
|
402
|
+
# =begin
|
403
|
+
# Example output from mount:
|
404
|
+
|
405
|
+
# devfs on /dev (devfs, local, nobrowse)
|
406
|
+
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
407
|
+
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
408
|
+
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
409
|
+
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
410
|
+
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
411
|
+
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
412
|
+
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
413
|
+
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
414
|
+
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
415
|
+
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
416
|
+
# =end
|
417
|
+
# # add directories from current mountpoints...
|
418
|
+
# mount_table_lines = interesting_mount_points()
|
419
|
+
# mount_table_lines.each{|line|
|
420
|
+
# next if line =~ /^map /
|
421
|
+
# dev, on, path, mode = line.split(/ /)
|
422
|
+
# #puts line
|
423
|
+
# #deb yellow(path)
|
424
|
+
# active_dirs << path if self.ok_dir?(path)
|
425
|
+
# }
|
426
|
+
# end
|
427
|
+
# active_dirs.uniq!
|
428
|
+
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
429
|
+
# return active_dirs
|
430
|
+
# end
|
431
|
+
end
|
432
|
+
end
|
433
|
+
end
|
430
434
|
|
431
435
|
def backquote_execute(cmd, opts = {})
|
432
436
|
dryrun = opts.fetch :dryrun, false
|
@@ -436,7 +440,7 @@ def backquote_execute(cmd, opts = {})
|
|
436
440
|
`#{cmd}`
|
437
441
|
end
|
438
442
|
|
439
|
-
def upload_to_gcs(file,
|
443
|
+
def upload_to_gcs(file, _opts = {})
|
440
444
|
deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
|
441
445
|
mount_name = file.split('/')[-2]
|
442
446
|
filename = "#{mount_name}-#{File.basename file}"
|
@@ -444,11 +448,10 @@ def upload_to_gcs(file, opts = {})
|
|
444
448
|
command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{hostname}-#{filename}"
|
445
449
|
deb("Command: #{command}")
|
446
450
|
puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
|
447
|
-
|
451
|
+
backquote_execute(command, dryrun: true)
|
448
452
|
# if $opts[:debug] do
|
449
453
|
# puts "+ Current list of files:"
|
450
454
|
# ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
|
451
455
|
# puts ret
|
452
456
|
# end
|
453
|
-
ret
|
454
457
|
end
|