storazzo 0.5.7 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +7 -5
- data/Makefile +8 -6
- data/Rakefile +15 -13
- data/VERSION +1 -1
- data/bin/hello-storazzo +1 -0
- data/bin/ricdisk-magic +33 -30
- data/bin/stats-with-md5 +182 -157
- data/bin/storazzo +112 -114
- data/bin/storazzo-symlink.rb +1 -0
- data/lib/storazzo/colors.rb +77 -36
- data/lib/storazzo/common.rb +72 -68
- data/lib/storazzo/debug.rb +2 -0
- data/lib/storazzo/hashify.rb +7 -5
- data/lib/storazzo/main.rb +59 -49
- data/lib/storazzo/media/abstract_ric_disk.rb +188 -179
- data/lib/storazzo/media/gcs_bucket.rb +76 -58
- data/lib/storazzo/media/local_folder.rb +56 -42
- data/lib/storazzo/media/mount_point.rb +20 -12
- data/lib/storazzo/ric_disk.rb +386 -383
- data/lib/storazzo/ric_disk_config.rb +227 -209
- data/lib/storazzo/ric_disk_sample_config.rb +26 -24
- data/lib/storazzo/ric_disk_statsfile.rb +19 -17
- data/lib/storazzo/ric_disk_ugly.rb +1 -0
- data/lib/storazzo/version.rb +3 -3
- data/lib/storazzo.rb +6 -7
- data/storazzo.gemspec +24 -16
- data/test/benchmark/for_future_use.rb +4 -2
- data/test/benchmark/test_hashing_functions-speed.rb +17 -0
- data/test/bin/new-idea.rb +17 -0
- data/test/bin/storazzo.rb +22 -25
- data/test/media/test_abstract_ric_disk.rb +5 -3
- data/test/media/test_gcs_bucket.rb +24 -23
- data/test/media/test_local_folder.rb +24 -23
- data/test/media/test_mount_point.rb +6 -5
- data/test/test_ric_disk.rb +6 -4
- data/test/test_ric_disk_config.rb +12 -11
- data/test/test_ric_disk_stats_file.rb +5 -3
- data/test/test_storazzo.rb +6 -4
- data/var/test/disks/disk02-full/Rakefile +7 -5
- data/var/test/disks/ricdisk_stats_v11.rds +11 -0
- metadata +13 -21
data/lib/storazzo/ric_disk.rb
CHANGED
@@ -1,432 +1,436 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# A RicDisk wraps a local mount/disk/folder
|
2
4
|
# it's considered interesting if there's a ".ricdisk/.ricdisk"
|
3
5
|
|
4
6
|
require 'digest'
|
5
7
|
|
6
8
|
module Storazzo
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
def initialize(ric_disk_object, opts = {})
|
46
|
-
verbose = opts.fetch :verbose, true
|
47
|
-
pverbose verbose,
|
48
|
-
"This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
|
49
|
-
raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
|
50
|
-
|
51
|
-
# ok back to business, now path is a String :)
|
52
|
-
path = ric_disk_object.path
|
53
|
-
deb "RicDisk initialize.. path=#{path}"
|
54
|
-
deb "RicDisk initialize.. path=#{path}"
|
55
|
-
@local_mountpoint = File.expand_path(path)
|
56
|
-
@ard = ric_disk_object # AbstractRicDiskObject
|
57
|
-
@description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
|
58
|
-
@ricdisk_version = RicdiskVersion
|
59
|
-
@ricdisk_file = compute_ricdisk_file() # Storazzo::RicDisk.get_ricdisk_file(path)
|
60
|
-
@ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
|
61
|
-
@label = path.split("/").last
|
62
|
-
@name = path.split("/").last
|
63
|
-
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
64
|
-
# @wr = writeable?
|
65
|
-
@tags = ['ricdisk', 'storazzo']
|
66
|
-
@size = RicDisk._compute_size_could_take_long(path)
|
67
|
-
@unique_hash = "MD5::" + Digest::MD5.hexdigest(File.expand_path(path)) # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
68
|
-
@computation_hostname = Socket.gethostname
|
69
|
-
@created_at = Time.now
|
70
|
-
|
71
|
-
@ricdisk_file_empty = ricdisk_file_empty?
|
72
|
-
|
73
|
-
# @config = RicDiskConfig.instance.get_config
|
74
|
-
# #puts @config if @config
|
75
|
-
# find_info_from_mount(path)
|
76
|
-
deb "RicDisk initialize. to_s: #{self}"
|
77
|
-
# find_info_from_df()
|
78
|
-
end
|
9
|
+
module Storazzo
|
10
|
+
class RicDisk
|
11
|
+
include Hashify
|
12
|
+
include Storazzo::Common
|
13
|
+
extend Storazzo::Common
|
14
|
+
extend Storazzo::Colors
|
15
|
+
require 'socket'
|
16
|
+
|
17
|
+
## Instance variables
|
18
|
+
|
19
|
+
# in order of finding, so the first will be the one we actually READ and use. I could looknat the date but cmon...
|
20
|
+
# These are the files I do accept.
|
21
|
+
ConfigFiles = %w[ricdisk.yaml .ricdisk storazzo.yaml].freeze
|
22
|
+
DefaultConfigFile = 'storazzo.yaml' # .ricdisk }
|
23
|
+
RicdiskVersion = '2.1'
|
24
|
+
RicdiskHistory = [
|
25
|
+
'2022-07-29 2.1 Added timestamp',
|
26
|
+
'2022-07-28 2.0 Added tags, siz, unique_hash, computation_hostname, wr, ...'
|
27
|
+
].freeze
|
28
|
+
DefaultGemfileTestDiskFolder = "#{Storazzo.root}/var/test/disks/" # was: @@default_gemfile_test_disks_folder
|
29
|
+
# Immutable
|
30
|
+
DefaultMediaFolders = %w[
|
31
|
+
/Volumes/
|
32
|
+
/mnt/
|
33
|
+
].append(DefaultGemfileTestDiskFolder).append("/media/#{ENV['USER']}/")
|
34
|
+
|
35
|
+
# # todo substitute with protobuf..
|
36
|
+
attr_accessor :name, :description, :ricdisk_file, :ricdisk_file_full, :local_mountpoint, :wr, :path,
|
37
|
+
:ricdisk_file_empty, :size, :active_dirs, :ricdisk_version,
|
38
|
+
:unique_hash # new 202207
|
39
|
+
|
40
|
+
################################
|
41
|
+
## INSTANCE methods
|
42
|
+
################################
|
43
|
+
|
44
|
+
def initialize_old_way(_path, _opts = {})
|
45
|
+
raise 'Now I dont want a string in input, I want an OBJECT < Storazzo::Media::AbstractRicDisk'
|
46
|
+
end
|
79
47
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
48
|
+
def initialize(ric_disk_object, opts = {})
|
49
|
+
verbose = opts.fetch :verbose, true
|
50
|
+
pverbose verbose,
|
51
|
+
"This needs an object of type Storazzo::Media::AbstractRicDisk now (this case: #{ric_disk_object.class})"
|
52
|
+
raise "Woopsie, not a Storazzo::Media::AbstractRicDisk! Intead its a #{ric_disk_object.class}" unless ric_disk_object.class.superclass == Storazzo::Media::AbstractRicDisk
|
53
|
+
|
54
|
+
# ok back to business, now path is a String :)
|
55
|
+
path = ric_disk_object.path
|
56
|
+
deb "RicDisk initialize.. path=#{path}"
|
57
|
+
deb "RicDisk initialize.. path=#{path}"
|
58
|
+
@local_mountpoint = File.expand_path(path)
|
59
|
+
@ard = ric_disk_object # AbstractRicDiskObject
|
60
|
+
@description = "This is an automated RicDisk description from v.#{RicdiskVersion}. Created on #{Time.now}'"
|
61
|
+
@ricdisk_version = RicdiskVersion
|
62
|
+
@ricdisk_file = compute_ricdisk_file # Storazzo::RicDisk.get_ricdisk_file(path)
|
63
|
+
@ricdisk_file_full = "#{@local_mountpoint}/#{@ricdisk_file}"
|
64
|
+
@label = path.split('/').last
|
65
|
+
@name = path.split('/').last
|
66
|
+
# @wr = File.writable?("#{path}/#{ricdisk_file}" ) # .writeable?
|
67
|
+
# @wr = writeable?
|
68
|
+
@tags = %w[ricdisk storazzo]
|
69
|
+
@size = RicDisk._compute_size_could_take_long(path)
|
70
|
+
@unique_hash = "MD5::#{Digest::MD5.hexdigest(File.expand_path(path))}" # hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
71
|
+
@computation_hostname = Socket.gethostname
|
72
|
+
@created_at = Time.now
|
73
|
+
|
74
|
+
@ricdisk_file_empty = ricdisk_file_empty?
|
75
|
+
|
76
|
+
# @config = RicDiskConfig.instance.get_config
|
77
|
+
# #puts @config if @config
|
78
|
+
# find_info_from_mount(path)
|
79
|
+
deb "RicDisk initialize. to_s: #{self}"
|
80
|
+
# find_info_from_df()
|
81
|
+
end
|
85
82
|
|
86
|
-
|
87
|
-
|
88
|
-
|
83
|
+
def ricdisk_file_empty?
|
84
|
+
# File.empty?("#{local_mountpoint}/.ricdisk.yaml")
|
85
|
+
puts "compute_ricdisk_file: #{compute_ricdisk_file}"
|
86
|
+
File.empty?(compute_ricdisk_file.to_s) # was (get_ricdisk_file)
|
87
|
+
end
|
89
88
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
puts "2. Sbrodoling everything: :TODO"
|
94
|
-
# find_info_from_mount(path)
|
95
|
-
# find_info_from_df()
|
96
|
-
end
|
89
|
+
def ok_dir?
|
90
|
+
!ricdisk_file.nil?
|
91
|
+
end
|
97
92
|
|
98
|
-
|
99
|
-
|
100
|
-
|
93
|
+
def analyze_local_system
|
94
|
+
puts 'TODO This should analyzze the WHOLE system. TODO(ricc): move to another object which has to do with the system/computer.'
|
95
|
+
puts "1. Interesting Mounts: #{green interesting_mount_points}"
|
96
|
+
puts '2. Sbrodoling everything: :TODO'
|
97
|
+
# find_info_from_mount(path)
|
98
|
+
# find_info_from_df()
|
99
|
+
end
|
101
100
|
|
102
|
-
|
103
|
-
|
104
|
-
|
101
|
+
def path
|
102
|
+
local_mountpoint
|
103
|
+
end
|
105
104
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
# end
|
110
|
-
def self._compute_size_could_take_long(my_path)
|
111
|
-
deb "Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds."
|
112
|
-
puts azure('could take long. Please take precautions like forking with timeout of 5sec')
|
113
|
-
`du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
|
114
|
-
end
|
105
|
+
def to_s
|
106
|
+
"RicDisk(paz=#{path}, r/w=#{writeable?}, size=#{size}B, f=#{ricdisk_file}, v#{ricdisk_version}, ard=#{@ard})"
|
107
|
+
end
|
115
108
|
|
116
|
-
|
117
|
-
#
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
puts("absolute_path: #{azure absolute_path}")
|
126
|
-
puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
|
127
|
-
bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
|
128
|
-
puts("bash_output: #{azure bash_output}")
|
129
|
-
# @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
|
130
|
-
raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
|
131
|
-
|
132
|
-
@wr = File.writable?(absolute_path) # rescue false
|
133
|
-
return @wr
|
134
|
-
# :boh_todo_fix_me_and_compute
|
135
|
-
# false
|
136
|
-
end
|
109
|
+
# could take long..
|
110
|
+
# def size
|
111
|
+
# `du -s '#{path}'`.split(/\s/)[0]
|
112
|
+
# end
|
113
|
+
def self._compute_size_could_take_long(my_path)
|
114
|
+
deb 'Could take long. TODO(ricc): add some sort of cutoff/timeout to 5 seconds.'
|
115
|
+
puts azure('could take long. Please take precautions like forking with timeout of 5sec')
|
116
|
+
`du -s '#{my_path}' 2>/dev/null`.chomp.split(/\s/)[0] # self.size
|
117
|
+
end
|
137
118
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
119
|
+
def writeable?
|
120
|
+
# memoize
|
121
|
+
return @wr unless @wr.nil?
|
122
|
+
|
123
|
+
# NOW: CALCULATE it
|
124
|
+
# Now I can do ONCE an EXPENSIVE calculation
|
125
|
+
puts yellow("[RicDisk.writeable] TODO(ricc): Do expensive calculation if this FS is writeable: #{path} and write/memoize it on @wr once and for all")
|
126
|
+
puts yellow('[RicDisk.writeable] I have a feeling this should be delegated to praecipuus Storazzo::Media::Object we refer to (WR is different on GCS vs Local):') # infinite loop dammit #{self.to_verbose_s}")
|
127
|
+
puts("Dir: #{azure path}")
|
128
|
+
puts("absolute_path: #{azure absolute_path}")
|
129
|
+
puts("File.writable?(absolute_path): #{azure File.writable?(absolute_path)}")
|
130
|
+
bash_output = `if [ -w "#{absolute_path}" ]; then echo "WRITABLE"; else echo "NOT WRITABLE"; fi`
|
131
|
+
puts("bash_output: #{azure bash_output}")
|
132
|
+
# @wr = File.writable?(File.expand_path(@ricdisk_file)) # rescue false
|
133
|
+
raise "for some reason an important info (ricdisk_file='#{absolute_path}') is missing!" if ricdisk_file.nil?
|
134
|
+
|
135
|
+
@wr = File.writable?(absolute_path) # rescue false
|
136
|
+
@wr
|
137
|
+
# :boh_todo_fix_me_and_compute
|
138
|
+
# false
|
139
|
+
end
|
147
140
|
|
148
|
-
|
149
|
-
|
150
|
-
|
141
|
+
def to_verbose_s
|
142
|
+
h = {}
|
143
|
+
h[:to_s] = to_s
|
144
|
+
h[:wr] = wr
|
145
|
+
h[:inspect] = inspect
|
146
|
+
h[:writeable] = writeable?
|
147
|
+
h[:ard] = @ard
|
148
|
+
h
|
149
|
+
end
|
151
150
|
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
end
|
151
|
+
################################
|
152
|
+
## CLASS methods
|
153
|
+
################################
|
156
154
|
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
# d.find_active_dirs()
|
162
|
-
end
|
155
|
+
# All places where to find for something :)
|
156
|
+
def self.default_media_folders
|
157
|
+
DefaultMediaFolders # was DEFAULT_MEDIA_FOLDERS
|
158
|
+
end
|
163
159
|
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
160
|
+
# _localgem_disks
|
161
|
+
def self.test
|
162
|
+
d = RicDisk.new(DefaultGemfileTestDiskFolder)
|
163
|
+
puts(d)
|
164
|
+
puts "do something with it: #{d}"
|
165
|
+
# d.find_active_dirs()
|
166
|
+
end
|
168
167
|
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
|
168
|
+
def absolute_path
|
169
|
+
# @local_mountpoint + "/" + @ricdisk_file
|
170
|
+
"#{local_mountpoint}/#{ricdisk_file}"
|
173
171
|
end
|
174
|
-
|
175
|
-
base_dirs
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
172
|
+
|
173
|
+
def self.find_active_dirs(base_dirs = nil, _also_mountpoints = true)
|
174
|
+
if base_dirs.nil?
|
175
|
+
base_dirs = default_media_folders
|
176
|
+
puts "find_active_dirs with empty input -> using default_media_folders: #{yellow default_media_folders}"
|
177
|
+
end
|
178
|
+
active_dirs = []
|
179
|
+
base_dirs.each do |ugly_dir|
|
180
|
+
# https://stackoverflow.com/questions/1899072/getting-a-list-of-folders-in-a-directory#:~:text=Dir.chdir(%27/destination_directory%27)%0ADir.glob(%27*%27).select%20%7B%7Cf%7C%20File.directory%3F%20f%7D
|
181
|
+
dir = File.expand_path(ugly_dir)
|
182
|
+
begin
|
183
|
+
x = []
|
184
|
+
# puts "TEST2 DIR EXISTS: #{dir} -> #{Dir.exists?(dir)}"
|
185
|
+
unless Dir.exist?(dir)
|
186
|
+
deb "Dir doesnt exist, skipping: #{dir}"
|
187
|
+
next
|
188
|
+
end
|
189
|
+
Dir.chdir(dir)
|
190
|
+
x = Dir.glob('*').select { |f| File.directory? f }
|
191
|
+
subdirs = x.map { |subdir| "#{dir}#{subdir}" }
|
192
|
+
subdirs.each do |subdir|
|
193
|
+
deb "Subdir: #{subdir}"
|
194
|
+
puts `ls -al "#{subdir}"` # TODO: refactor in exec
|
195
|
+
active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
|
196
|
+
end
|
197
|
+
# puts(white x)
|
198
|
+
rescue Exception => e # optionally: `rescue Exception => ex`
|
199
|
+
puts "Exception: '#{e}'"
|
200
|
+
# will always get executed
|
201
|
+
# deb 'Always gets executed.'
|
202
|
+
# x = []
|
184
203
|
end
|
185
|
-
Dir.chdir(dir)
|
186
|
-
x = Dir.glob('*').select { |f| File.directory? f }
|
187
|
-
subdirs = x.map { |subdir| "#{dir}#{subdir}" }
|
188
|
-
subdirs.each { |subdir|
|
189
|
-
deb "Subdir: #{subdir}"
|
190
|
-
puts `ls -al "#{subdir}"` # TODO refactor in exec
|
191
|
-
active_dirs << subdir if ok_dir? # self.ok_dir?(subdir)
|
192
|
-
}
|
193
|
-
# puts(white x)
|
194
|
-
rescue Exception => e # optionally: `rescue Exception => ex`
|
195
|
-
puts "Exception: '#{e}'"
|
196
|
-
ensure # will always get executed
|
197
|
-
# deb 'Always gets executed.'
|
198
|
-
# x = []
|
199
204
|
end
|
200
205
|
end
|
201
|
-
end
|
202
206
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
207
|
+
def compute_ricdisk_file
|
208
|
+
unless @ricdisk_file.nil?
|
209
|
+
deb '[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)'
|
210
|
+
return @ricdisk_file
|
211
|
+
end
|
212
|
+
deb '[compute_ricdisk_file] RICC_WARNING This requires cmputation I wanna do it almost once'
|
213
|
+
ConfigFiles.each do |papable_config_filename|
|
214
|
+
# return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
215
|
+
# return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
216
|
+
return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}")
|
217
|
+
end
|
218
|
+
deb "File not found! Neither #{ConfigFiles} exist.."
|
219
|
+
# return nil
|
220
|
+
DefaultConfigFile
|
213
221
|
end
|
214
|
-
deb "File not found! Neither #{ConfigFiles} exist.."
|
215
|
-
# return nil
|
216
|
-
return DefaultConfigFile
|
217
|
-
end
|
218
222
|
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
# # new
|
234
|
-
# def self.get_ricdisk_file_obsolete(path)
|
235
|
-
# if @ricdisk_file
|
236
|
-
# puts "[CACHE HIT] ricdisk_file"
|
237
|
-
# return @ricdisk_file
|
238
|
-
# end
|
239
|
-
# puts "RICC_WARNING This requires cmputation I wanna do it almost once"
|
240
|
-
# ConfigFiles.each do |papable_config_filename|
|
241
|
-
# #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
242
|
-
# #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
243
|
-
# return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
244
|
-
# end
|
245
|
-
# return nil
|
246
|
-
# end
|
247
|
-
|
248
|
-
def self.interesting_mount_points(opts = {})
|
249
|
-
# https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
250
|
-
`mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
|
251
|
-
end
|
223
|
+
# def self.compute_ricdisk_file_by_path_once(path)
|
224
|
+
# # unless @ricdisk_file.nil?
|
225
|
+
# # deb "[CACHE HIT] ricdisk_file (didnt have to recompute it - yay!)"
|
226
|
+
# # return @ricdisk_file
|
227
|
+
# # end
|
228
|
+
# warn "RICC_WARNING This requires cmputation I wanna do it almost once"
|
229
|
+
# ConfigFiles.each do |papable_config_filename|
|
230
|
+
# #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
231
|
+
# #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
232
|
+
# return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
233
|
+
# end
|
234
|
+
# return nil
|
235
|
+
# end
|
252
236
|
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
237
|
+
# # new
|
238
|
+
# def self.get_ricdisk_file_obsolete(path)
|
239
|
+
# if @ricdisk_file
|
240
|
+
# puts "[CACHE HIT] ricdisk_file"
|
241
|
+
# return @ricdisk_file
|
242
|
+
# end
|
243
|
+
# puts "RICC_WARNING This requires cmputation I wanna do it almost once"
|
244
|
+
# ConfigFiles.each do |papable_config_filename|
|
245
|
+
# #return ".ricdisk.yaml" if File.exist?("#{path}/.ricdisk.yaml") #and File.empty?( "#{path}/.ricdisk.yaml")
|
246
|
+
# #return ".ricdisk" if File.exist?("#{path}/.ricdisk") # and File.empty?( "#{path}/.ricdisk")
|
247
|
+
# return papable_config_filename if File.exist?("#{path}/#{papable_config_filename}") # and File.empty?( "#{path}/.ricdisk")
|
248
|
+
# end
|
249
|
+
# return nil
|
250
|
+
# end
|
259
251
|
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
252
|
+
def self.interesting_mount_points(_opts = {})
|
253
|
+
# https://unix.stackexchange.com/questions/177014/showing-only-interesting-mount-points-filtering-non-interesting-types
|
254
|
+
`mount | grep -Ev 'type (proc|sysfs|tmpfs|devpts|debugfs|rpc_pipefs|nfsd|securityfs|fusectl|devtmpfs) '`.split(/\n+/)
|
255
|
+
end
|
264
256
|
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
unless ok_dir? # self.ok_dir?(subdir)
|
271
|
-
warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
|
272
|
-
return
|
257
|
+
# maybe move to a RiccFile class? Maybe even INHERIT from FILE?
|
258
|
+
def obsolescence_seconds(file_path)
|
259
|
+
creation_time = File.stat(file_path).ctime
|
260
|
+
deb("[obsolescence_seconds] File #{file_path}: #{creation_time} - #{Time.now - creation_time} seconds ago")
|
261
|
+
(Time.now - creation_time).to_i
|
273
262
|
end
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
end
|
263
|
+
|
264
|
+
# maybe move to a RiccFile class? Maybe even INHERIT from FILE?
|
265
|
+
def obsolescence_days(file_path)
|
266
|
+
obsolescence_seconds(file_path) / 86_400
|
279
267
|
end
|
280
|
-
|
281
|
-
#
|
282
|
-
#
|
283
|
-
#
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
268
|
+
|
269
|
+
# FORMER SBRODOLA, now write_config_yaml_to_disk
|
270
|
+
# def self.write_config_yaml_to_disk(subdir, opts={}) # sbrodola_ricdisk(subdir)
|
271
|
+
# sbrodola_ricdisk(subdir)
|
272
|
+
def write_config_yaml_to_disk(subdir, _opts = {})
|
273
|
+
# given a path, if .ricdisk exists i do stuff with it..
|
274
|
+
disk_info = nil
|
275
|
+
unless ok_dir? # self.ok_dir?(subdir)
|
276
|
+
warn("[write_config_yaml_to_disk] Nothing for me here: '#{subdir}'. Existing")
|
277
|
+
return
|
278
|
+
end
|
279
|
+
ConfigFiles.each do |papable_configfile_name|
|
280
|
+
if File.exist?("#{subdir}/#{papable_configfile_name}") && File.empty?("#{subdir}/#{papable_configfile_name}")
|
281
|
+
deb("Interesting. Empty file '#{papable_configfile_name}'! Now I write YAML with it.")
|
282
|
+
disk_info = RicDisk.new(subdir, papable_configfile_name)
|
283
|
+
end
|
284
|
+
end
|
285
|
+
# if File.exists?( "#{subdir}/.ricdisk") and File.empty?( "#{subdir}/.ricdisk")
|
286
|
+
# deb("Interesting1. Empty file! Now I write YAML with it.")
|
287
|
+
# disk_info = RicDisk.new(subdir, '.ricdisk')
|
288
|
+
# end
|
289
|
+
# if File.exists?( "#{subdir}/.ricdisk.yaml") and File.empty?( "#{subdir}/.ricdisk.yaml")
|
290
|
+
# deb("Interesting2. Empty file! TODO write YAML with it.")
|
291
|
+
# disk_info = RicDisk.new(subdir, '.ricdisk.yaml')
|
292
|
+
# puts(yellow disk_info.to_yaml)
|
293
|
+
# end
|
294
|
+
if disk_info.is_a?(RicDisk)
|
295
|
+
deb yellow("disk_info.class: #{disk_info.class}")
|
296
|
+
if File.empty?(disk_info.absolute_path) # and (disk_info.wr)
|
297
|
+
puts(green("yay, we can now write the file '#{disk_info.absolute_path}' (which is R/W, I just checked!) with proper YAML content.."))
|
298
|
+
if disk_info.wr
|
299
|
+
ret = File.write(disk_info.absolute_path, disk_info.obj_to_yaml)
|
300
|
+
puts green("Written file! ret=#{ret}")
|
301
|
+
else
|
302
|
+
raise 'TODO_IMPLEMENT: write in proper place in config dir'
|
303
|
+
puts red('TODO implement me')
|
304
|
+
end
|
296
305
|
else
|
297
|
-
|
298
|
-
puts
|
306
|
+
puts(red("Something not right here: either file is NOT empty or disk is NOT writeable.. #{File.empty?(disk_info.absolute_path)}"))
|
307
|
+
puts("File size: #{File.size(disk_info.absolute_path)}")
|
308
|
+
puts(disk_info.to_s)
|
309
|
+
puts(disk_info.obj_to_hash)
|
310
|
+
puts(disk_info.obj_to_yaml)
|
299
311
|
end
|
300
|
-
else
|
301
|
-
puts
|
302
|
-
puts("File size: #{File.size(disk_info.absolute_path)}")
|
303
|
-
puts(disk_info.to_s)
|
304
|
-
puts(disk_info.obj_to_hash)
|
305
|
-
puts(disk_info.obj_to_yaml)
|
312
|
+
else # not a RicDisk..
|
313
|
+
puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
|
306
314
|
end
|
307
|
-
else # not a RicDisk..
|
308
|
-
puts "[write_config_yaml_to_disk] No DiskInfo found across #{ConfigFiles}. I leave this function empty-handed."
|
309
|
-
end
|
310
315
|
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
316
|
+
# disk_info.absolute_path
|
317
|
+
# if File.exists?( "#{subdir}/.ricdisk") and ! File.empty?( "#{subdir}/.ricdisk")
|
318
|
+
# if File.exists?(disk_info.absolute_path) and ! File.empty?(disk_info.absolute_path)
|
319
|
+
# puts("Config File found with old-style name: '#{subdir}/.ricdisk' ! Please move it to .ricdisk.yaml!")
|
320
|
+
# puts(white `cat "#{disk_info.absolute_path}"`)
|
321
|
+
# else
|
322
|
+
# puts "WRITING NOW. [I BELIEVE THIS IS DUPE CODE - see a few lines above!] disk_info.obj_to_yaml .. to #{compute_ricdisk_file}"
|
323
|
+
# File.open(ricdisk_file_full, 'w').write(disk_info.obj_to_yaml)
|
324
|
+
# end
|
325
|
+
end
|
321
326
|
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
327
|
+
# TODO: obsolete this as i should NOT be calling it from clas, but from method.
|
328
|
+
def self.ok_dir?(subdir)
|
329
|
+
File.exist?("#{subdir}/.ricdisk") or File.exist?("#{subdir}/.ricdisk.yaml")
|
330
|
+
end
|
326
331
|
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
332
|
+
def compute_stats_files(opts = {})
|
333
|
+
# Storazzo::RicDisk.calculate_stats_files(path, opts)
|
334
|
+
opts_upload_to_gcs = opts.fetch :upload_to_gcs, false
|
335
|
+
opts_force_rewrite = opts.fetch :force, false
|
336
|
+
opts_stats_file = opts.fetch :stats_file, 'ricdisk_stats_v11.rds' # default. TODO point to proper..
|
337
|
+
dir = path
|
338
|
+
puts azure("[compute_stats_files] TODO implement natively. Now I'm being lazy. stats_file=#{opts_stats_file} dir=#{dir}")
|
339
|
+
|
340
|
+
full_file_path = "#{dir}/#{opts_stats_file}"
|
341
|
+
deb 'This refactor is for another day. Actually no, TODAY '
|
342
|
+
pverbose true,
|
343
|
+
"TODO(ricc): you should compute more SMARTLY the full_file_path (#{full_file_path}): if its R/O it should be elsewhere.."
|
344
|
+
puts azure("- full_file_path: #{full_file_path}")
|
345
|
+
puts azure("- writeable?: #{writeable?}")
|
346
|
+
|
347
|
+
puts("compute_stats_files(#{white dir}): #{white full_file_path}")
|
348
|
+
deb "TEST1 DIR EXISTS: #{dir} -> #{File.directory? dir}"
|
349
|
+
raise "Directory doesnt exist: #{dir}" unless File.directory?(dir)
|
350
|
+
|
351
|
+
Dir.chdir(dir)
|
352
|
+
puts azure `ls` # im curious
|
353
|
+
if File.exist?(full_file_path)
|
354
|
+
if opts_force_rewrite
|
355
|
+
# raise "TODO implement file exists and FORCE enabled"
|
356
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, 'ReWrite enabled')
|
357
|
+
else # File.exists?(full_file_path) and (opts_force)
|
358
|
+
puts "File '#{opts_stats_file}' exists already." # - now should see if its too old, like more than 1 week old"
|
359
|
+
# TODO check for file time...
|
360
|
+
print "Lines found: #{yellow `wc -l "#{full_file_path}" `.chomp}. File obsolescence (days): #{yellow obsolescence_days(full_file_path)}."
|
361
|
+
if obsolescence_days(full_file_path) > 7
|
362
|
+
# puts yellow("*** ACHTUNG *** FIle is pretty old. You might consider rotating: #{yellow "mv #{full_file_path} #{full_file_path}_old"}. Or invoke with --force")
|
363
|
+
puts yellow("*** ACHTUNG *** FIle is pretty old. I'll force a rewrite")
|
364
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path,
|
365
|
+
"File older than 7 days. Indeed: #{obsolescence_days(full_file_path)}")
|
366
|
+
end
|
367
|
+
upload_to_gcs(full_file_path) if opts_upload_to_gcs
|
361
368
|
end
|
362
|
-
|
369
|
+
else
|
370
|
+
deb('File doesnt exist..')
|
371
|
+
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ConfigFile doesn't exist")
|
363
372
|
end
|
364
|
-
else
|
365
|
-
deb("File doesnt exist..")
|
366
|
-
RicDisk.compute_stats_for_dir_into_file(dir, full_file_path, "ConfigFile doesn't exist")
|
367
373
|
end
|
368
|
-
end
|
369
374
|
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
puts red("Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)")
|
375
|
+
def self.compute_stats_for_dir_into_file(dir, full_file_path, reason, opts = {})
|
376
|
+
max_lines = opts.fetch :max_lines, 42 # TODO: move to nil or -1
|
377
|
+
# full_file_path = "#{dir}/#{stats_file}"
|
378
|
+
puts "Crunching data stats from '#{dir}' into '#{full_file_path}' ... please bear with me.. [reason: '#{reason}']"
|
379
|
+
if max_lines.negative? # infinite
|
380
|
+
command = "find . -print0 | xargs -0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
381
|
+
elsif mac?
|
382
|
+
# WOW! https://stackoverflow.com/questions/68599963/reliably-stop-bash-find-after-n-matches
|
383
|
+
# find . -type f -iname "*.txt" -print0 |
|
384
|
+
# head -z -n 10 |
|
385
|
+
# xargs -r0 myscript.sh
|
386
|
+
puts red('Sorry head -z doesnt work on Mac :/ so this head -N will be VERY approximate. Probably you should divide by ten or so :)')
|
383
387
|
spannometric_lines = (max_lines / 10)
|
384
388
|
command = "find . -print0 | head -n '#{spannometric_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
385
389
|
else
|
386
390
|
command = "find . -print0 | head -z -n '#{max_lines}' | xargs -r0 stats-with-md5 --no-color | tee '#{full_file_path}'"
|
387
391
|
end
|
392
|
+
puts("[#{`pwd`.chomp}] Executing: #{azure command}")
|
393
|
+
ret = backquote_execute(command)
|
394
|
+
puts "Done. #{ret.split("\n").count} files processed."
|
388
395
|
end
|
389
|
-
puts("[#{`pwd`.chomp}] Executing: #{azure command}")
|
390
|
-
ret = backquote_execute(command)
|
391
|
-
puts "Done. #{ret.split("\n").count} files processed."
|
392
|
-
end
|
393
396
|
|
394
|
-
|
395
|
-
|
396
|
-
|
397
|
+
def self.calculate_stats_files_DUPLICATE_STATIC(_dir, _opts = {})
|
398
|
+
raise 'Please use object instead. If you cant, please move the object code to STATIC and dedupe code!'
|
399
|
+
end
|
397
400
|
|
398
|
-
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
424
|
-
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
end
|
401
|
+
# if also_mountpoints
|
402
|
+
# =begin
|
403
|
+
# Example output from mount:
|
404
|
+
|
405
|
+
# devfs on /dev (devfs, local, nobrowse)
|
406
|
+
# /dev/disk3s6 on /System/Volumes/VM (apfs, local, noexec, journaled, noatime, nobrowse)
|
407
|
+
# /dev/disk3s2 on /System/Volumes/Preboot (apfs, local, journaled, nobrowse)
|
408
|
+
# /dev/disk3s4 on /System/Volumes/Update (apfs, local, journaled, nobrowse)
|
409
|
+
# /dev/disk1s2 on /System/Volumes/xarts (apfs, local, noexec, journaled, noatime, nobrowse)
|
410
|
+
# /dev/disk1s1 on /System/Volumes/iSCPreboot (apfs, local, journaled, nobrowse)
|
411
|
+
# /dev/disk1s3 on /System/Volumes/Hardware (apfs, local, journaled, nobrowse)
|
412
|
+
# /dev/disk3s5 on /System/Volumes/Data (apfs, local, journaled, nobrowse, protect)
|
413
|
+
# map auto_home on /System/Volumes/Data/home (autofs, automounted, nobrowse)
|
414
|
+
# //riccardo@1.0.1.10/video on /Volumes/video (afpfs, nodev, nosuid, mounted by ricc)
|
415
|
+
# //riccardo@1.0.1.10/photo on /Volumes/photo (afpfs, nodev, nosuid, mounted by ricc)
|
416
|
+
# =end
|
417
|
+
# # add directories from current mountpoints...
|
418
|
+
# mount_table_lines = interesting_mount_points()
|
419
|
+
# mount_table_lines.each{|line|
|
420
|
+
# next if line =~ /^map /
|
421
|
+
# dev, on, path, mode = line.split(/ /)
|
422
|
+
# #puts line
|
423
|
+
# #deb yellow(path)
|
424
|
+
# active_dirs << path if self.ok_dir?(path)
|
425
|
+
# }
|
426
|
+
# end
|
427
|
+
# active_dirs.uniq!
|
428
|
+
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
429
|
+
# return active_dirs
|
430
|
+
# end
|
431
|
+
end
|
432
|
+
end
|
433
|
+
end
|
430
434
|
|
431
435
|
def backquote_execute(cmd, opts = {})
|
432
436
|
dryrun = opts.fetch :dryrun, false
|
@@ -436,7 +440,7 @@ def backquote_execute(cmd, opts = {})
|
|
436
440
|
`#{cmd}`
|
437
441
|
end
|
438
442
|
|
439
|
-
def upload_to_gcs(file,
|
443
|
+
def upload_to_gcs(file, _opts = {})
|
440
444
|
deb("upload_to_gcs(#{file}). TODO(ricc) after breafast upload to GCS : #{file}")
|
441
445
|
mount_name = file.split('/')[-2]
|
442
446
|
filename = "#{mount_name}-#{File.basename file}"
|
@@ -444,11 +448,10 @@ def upload_to_gcs(file, opts = {})
|
|
444
448
|
command = "gsutil cp '#{file}' gs://#{$gcs_bucket}/backup/ricdisk-magic/#{hostname}-#{filename}"
|
445
449
|
deb("Command: #{command}")
|
446
450
|
puts azure("GCS upload disabled until I know if it works :) command='#{command}'")
|
447
|
-
|
451
|
+
backquote_execute(command, dryrun: true)
|
448
452
|
# if $opts[:debug] do
|
449
453
|
# puts "+ Current list of files:"
|
450
454
|
# ret = backquote_execute("gsutil ls -al gs://#{$gcs_bucket}/backup/ricdisk-magic/")
|
451
455
|
# puts ret
|
452
456
|
# end
|
453
|
-
ret
|
454
457
|
end
|