storazzo 0.4.10 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +5 -4
- data/Rakefile +8 -11
- data/VERSION +1 -1
- data/bin/hello-storazzo +0 -1
- data/bin/ricdisk-magic +50 -56
- data/bin/stats-with-md5 +251 -210
- data/bin/storazzo +32 -0
- data/bin/storazzo-util +1 -0
- data/lib/storazzo/colors.rb +43 -45
- data/lib/storazzo/common.rb +52 -46
- data/lib/storazzo/debug.rb +5 -8
- data/lib/storazzo/hashify.rb +44 -43
- data/lib/storazzo/main.rb +40 -41
- data/lib/storazzo/media/abstract_ric_disk.rb +162 -151
- data/lib/storazzo/media/gcs_bucket.rb +52 -51
- data/lib/storazzo/media/local_folder.rb +43 -44
- data/lib/storazzo/media/mount_point.rb +8 -9
- data/lib/storazzo/ric_disk.rb +153 -170
- data/lib/storazzo/ric_disk_config.rb +224 -219
- data/lib/storazzo/ric_disk_sample_config.rb +12 -12
- data/lib/storazzo/ric_disk_statsfile.rb +17 -15
- data/lib/storazzo/ric_disk_ugly.rb +35 -38
- data/lib/storazzo/version.rb +7 -7
- data/lib/storazzo.rb +36 -36
- data/storazzo.gemspec +26 -21
- data/test/media/test_abstract_ric_disk.rb +12 -16
- data/test/media/test_gcs_bucket.rb +44 -47
- data/test/media/test_local_folder.rb +134 -136
- data/test/media/test_mount_point.rb +18 -19
- data/test/test_ric_disk.rb +12 -14
- data/test/test_ric_disk_config.rb +20 -24
- data/test/test_ric_disk_stats_file.rb +13 -14
- data/test/test_storazzo.rb +26 -26
- data/var/test/disks/disk02-full/Rakefile +4 -4
- metadata +7 -3
@@ -1,12 +1,11 @@
|
|
1
1
|
require 'singleton'
|
2
2
|
require 'yaml'
|
3
3
|
|
4
|
-
#require 'storazzo/media/abstract_ric_disk'
|
5
|
-
Dir[File.dirname(__FILE__) + '/../lib/*.rb'].each do |file|
|
6
|
-
|
4
|
+
# require 'storazzo/media/abstract_ric_disk'
|
5
|
+
Dir[File.dirname(__FILE__) + '/../lib/*.rb'].each do |file|
|
6
|
+
require File.basename(file, File.extname(file))
|
7
7
|
end
|
8
|
-
#require_all 'media/directory'
|
9
|
-
|
8
|
+
# require_all 'media/directory'
|
10
9
|
|
11
10
|
=begin
|
12
11
|
This is a singleton class. You call me this way..
|
@@ -23,220 +22,226 @@ end
|
|
23
22
|
=end
|
24
23
|
|
25
24
|
module Storazzo
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
public
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
end
|
98
|
-
|
99
|
-
# Obsolete, call another class instead.
|
100
|
-
def load_sample_version
|
101
|
-
raise "DEPRECATED! USE SampleRicDiskConfig.load() instead!"
|
102
|
-
end
|
103
|
-
|
104
|
-
def config_ver
|
105
|
-
raise "I cant compute Version since I cant compute @config. Are you sure you didnt instance this Singleton without calling load?" if @config.nil?
|
106
|
-
@config['apiVersion'] # rescue :StillUnknown
|
107
|
-
#config['ConfigVersion']
|
108
|
-
end
|
109
|
-
def config_default_folder
|
110
|
-
#self.
|
111
|
-
@config['Config']['DefaultFolder'] #rescue "Unknown config_default_folder: #{$!}"
|
112
|
-
end
|
113
|
-
def config_project_id
|
114
|
-
@config['Config']['Backends']['GoogleCloudStorage']['ProjectId']
|
115
|
-
end
|
116
|
-
def already_loaded?
|
117
|
-
#return
|
118
|
-
load_called == true
|
119
|
-
end
|
120
|
-
|
121
|
-
def to_s
|
122
|
-
size = File.size(@config_file) rescue -1
|
123
|
-
#puts yellow "DEB: #{@config["apiVersion"]}"
|
124
|
-
#"RicDiskConfig(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
125
|
-
"POLY_#{self.class}_(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
126
|
-
end
|
127
|
-
|
128
|
-
def to_verbose_s
|
129
|
-
h = {}
|
130
|
-
h[:description] = "This is a Verbose Hash describing a RicDiskConfig or its child RicDiskSampleConfig to understand why it keeps failing.."
|
131
|
-
h[:to_s] = self.to_s
|
132
|
-
h[:class] = self.class
|
133
|
-
h[:file] = __FILE__
|
134
|
-
h[:id] = self.object_id
|
135
|
-
h[:get_bucket_paths] = self.get_bucket_paths()
|
136
|
-
h[:get_local_folders] = self.get_local_folders()
|
137
|
-
h[:config_project_id] = self.config_project_id()
|
138
|
-
return h
|
139
|
-
end
|
140
|
-
|
141
|
-
def get_config(opts={})
|
142
|
-
return load(opts) if @config.nil?
|
143
|
-
@config
|
144
|
-
end
|
145
|
-
|
146
|
-
def self.gem_default_config_path
|
147
|
-
Storazzo.root + "/etc/storazzo_config.sample.yaml"
|
25
|
+
# class Storazzo::Blah
|
26
|
+
# end
|
27
|
+
|
28
|
+
class Storazzo::RicDiskConfig
|
29
|
+
include Singleton
|
30
|
+
include Storazzo::Common
|
31
|
+
include Storazzo::Colors
|
32
|
+
|
33
|
+
DefaultConfigLocation = File.expand_path "~/.storazzo.yaml"
|
34
|
+
|
35
|
+
DefaultConfigLocations = [
|
36
|
+
File.expand_path("~/.storazzo.yaml"), # HOME
|
37
|
+
File.expand_path("./.storazzo.yaml"), # LOCAL DIR
|
38
|
+
]
|
39
|
+
|
40
|
+
DefaultGemLocationForTests =
|
41
|
+
File.expand_path('../../../', __FILE__) +
|
42
|
+
"/etc/storazzo_config.sample.yaml"
|
43
|
+
|
44
|
+
attr_accessor :config, :config_file, :load_called
|
45
|
+
|
46
|
+
public
|
47
|
+
|
48
|
+
# Load from the first valid config.
|
49
|
+
def load(config_path = nil, opts = {})
|
50
|
+
verbose = opts.fetch :verbose, false
|
51
|
+
|
52
|
+
if already_loaded? # and not self.config.nil?
|
53
|
+
pverbose verbose, "[#{self.class}] VERBOSE load: already loaded"
|
54
|
+
return self.config
|
55
|
+
end
|
56
|
+
pverbose verbose, "Storazzo::RicDiskConfig.load(): BEGIN"
|
57
|
+
# trying default location
|
58
|
+
raise "DefaultConfigLocation is not a string" unless DefaultConfigLocation.is_a?(String)
|
59
|
+
|
60
|
+
possible_locations = DefaultConfigLocations # [ default_config_locations .. , "./.storazzo.yaml"]
|
61
|
+
deb "[Config.load] Possible Locations: #{possible_locations}"
|
62
|
+
if config_path.is_a?(String)
|
63
|
+
# possible_locations = [config_path] + possible_locations # .append()
|
64
|
+
possible_locations = possible_locations.unshift(config_path) # append to front
|
65
|
+
# OR: possible_locations.instert(0, config_path)
|
66
|
+
pverbose verbose, "[LOAD] possible_locations: #{possible_locations}"
|
67
|
+
end
|
68
|
+
puts "[VERBOSE] Searching these paths in order: #{possible_locations}" if verbose
|
69
|
+
# bug "This is not always an array of sTRINGS."
|
70
|
+
raise "possible_locations is not an array" unless possible_locations.is_a?(Array)
|
71
|
+
|
72
|
+
possible_locations.each do |possible_path|
|
73
|
+
# ASSERT is a string
|
74
|
+
raise "possible_path is not a string" unless possible_path.is_a?(String)
|
75
|
+
|
76
|
+
deb "before buggy expand_path paz: '#{possible_path}''"
|
77
|
+
paz = File.expand_path(possible_path) rescue possible_path
|
78
|
+
raise "Not a string: #{paz}" unless paz.is_a?(String)
|
79
|
+
|
80
|
+
if File.exists?(paz)
|
81
|
+
@config_file = paz
|
82
|
+
@config = YAML.load(File.read paz) # YAML.load(File.read("file_path"))
|
83
|
+
|
84
|
+
unless (@config["kind"] == 'StorazzoConfig' rescue false)
|
85
|
+
puts white "RicDiskConfig.load(): Sorry this is wrong Config File. Kind=#{@config["kind"] rescue $!}"
|
86
|
+
next
|
87
|
+
end
|
88
|
+
#
|
89
|
+
# pp @config if verbose
|
90
|
+
config_ver = @config["apiVersion"]
|
91
|
+
# puts @config[:ConfigVersion]
|
92
|
+
deb("OK. Storazzo::RicDiskConfig v'#{config_ver}' parsed correctly")
|
93
|
+
# puts "[VERBOSE] RicDiskConfig.to_s: #{self}" if verbose
|
94
|
+
@load_called = true
|
95
|
+
return self.config
|
148
96
|
end
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
#
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
97
|
+
end
|
98
|
+
@load_called = true
|
99
|
+
# only get here if nothing is found
|
100
|
+
raise "No config found across these locations: #{possible_locations}. Consider copying and editing: #{RicDiskConfig.gem_default_config_path}"
|
101
|
+
end
|
102
|
+
|
103
|
+
# Obsolete, call another class instead.
|
104
|
+
def load_sample_version
|
105
|
+
raise "DEPRECATED! USE SampleRicDiskConfig.load() instead!"
|
106
|
+
end
|
107
|
+
|
108
|
+
def config_ver
|
109
|
+
raise "I cant compute Version since I cant compute @config. Are you sure you didnt instance this Singleton without calling load?" if @config.nil?
|
110
|
+
|
111
|
+
@config['apiVersion'] # rescue :StillUnknown
|
112
|
+
# config['ConfigVersion']
|
113
|
+
end
|
114
|
+
|
115
|
+
def config_default_folder
|
116
|
+
# self.
|
117
|
+
@config['Config']['DefaultFolder'] # rescue "Unknown config_default_folder: #{$!}"
|
118
|
+
end
|
119
|
+
|
120
|
+
def config_project_id
|
121
|
+
@config['Config']['Backends']['GoogleCloudStorage']['ProjectId']
|
122
|
+
end
|
123
|
+
|
124
|
+
def already_loaded?
|
125
|
+
# return
|
126
|
+
load_called == true
|
127
|
+
end
|
128
|
+
|
129
|
+
def to_s
|
130
|
+
size = File.size(@config_file) rescue -1
|
131
|
+
# puts yellow "DEB: #{@config["apiVersion"]}"
|
132
|
+
# "RicDiskConfig(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
133
|
+
"POLY_#{self.class}_(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
134
|
+
end
|
135
|
+
|
136
|
+
def to_verbose_s
|
137
|
+
h = {}
|
138
|
+
h[:description] =
|
139
|
+
"This is a Verbose Hash describing a RicDiskConfig or its child RicDiskSampleConfig to understand why it keeps failing.."
|
140
|
+
h[:to_s] = self.to_s
|
141
|
+
h[:class] = self.class
|
142
|
+
h[:file] = __FILE__
|
143
|
+
h[:id] = self.object_id
|
144
|
+
h[:get_bucket_paths] = self.get_bucket_paths()
|
145
|
+
h[:get_local_folders] = self.get_local_folders()
|
146
|
+
h[:config_project_id] = self.config_project_id()
|
147
|
+
return h
|
148
|
+
end
|
149
|
+
|
150
|
+
def get_config(opts = {})
|
151
|
+
return load(opts) if @config.nil?
|
152
|
+
|
153
|
+
@config
|
154
|
+
end
|
155
|
+
|
156
|
+
def self.gem_default_config_path
|
157
|
+
Storazzo.root + "/etc/storazzo_config.sample.yaml"
|
158
|
+
end
|
159
|
+
|
160
|
+
# Storazzo::RicDiskConfig
|
161
|
+
|
162
|
+
# returns all folders from file which are Directories
|
163
|
+
# This method is FLAKY! Sometimes gives error.
|
164
|
+
# LocalFolderTest#test_show_all_shouldnt_fail_and_should_return_a_non_empty_array:
|
165
|
+
# TypeError: no implicit conversion of Hash into String
|
166
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:38:in `expand_path'
|
167
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:38:in `block in load'
|
168
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:37:in `each'
|
169
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:37:in `load'
|
170
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:83:in `get_config'
|
171
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:95:in `get_local_folders'
|
172
|
+
def get_local_folders
|
173
|
+
config = get_config
|
174
|
+
# puts config['Config']['AdditionalMountDirs']
|
175
|
+
config['Config']['AdditionalMountDirs'].map { |folder|
|
176
|
+
File.expand_path(folder) rescue folder # TypeError: no implicit conversion of Hash into String
|
177
|
+
}.filter { |f| File.directory?(f) }
|
178
|
+
end
|
179
|
+
|
180
|
+
def get_bucket_paths
|
181
|
+
get_config['Config']['Backends']['GoogleCloudStorage']['BucketPaths'].map { |complex_gcs_struct|
|
182
|
+
complex_gcs_struct['path']
|
183
|
+
}
|
184
|
+
end
|
185
|
+
|
186
|
+
# UGLY CODE, copipasted from binary for ARGV, ex autosbrodola
|
187
|
+
def iterate_through_file_list_for_disks(files_list = [], opts = {})
|
188
|
+
verbose = opts.fetch :verbose, false
|
189
|
+
raise "[iterate_through_file_list_for_disks] Wrong input, I need an array here: #{files_list} " unless files_list.is_a?(Array)
|
190
|
+
|
191
|
+
# I decided i wont accept an emopty list, this is not how you use the gem, you lazy XXX!
|
192
|
+
# if files_list == [] # or files_list.nil? # empty -> ALL
|
193
|
+
# deb "iterate_through_file_list_for_disks(): no args provided"
|
194
|
+
# dirs = RicDisk.find_active_dirs()
|
195
|
+
# puts "DEB find_active_dirs: #{green dirs}"
|
196
|
+
# dirs.each {|dir|
|
197
|
+
# RicDisk.write_config_yaml_to_disk(dir)
|
198
|
+
# RicDisk.calculate_stats_files(dir) # dir is inutile
|
199
|
+
# } # TODO refactor in option sbrodola_afterwards=true. :)
|
200
|
+
# else
|
201
|
+
puts "iterate_through_file_list_for_disks(): I consider files_list as a list of directories to parse :)" if verbose
|
202
|
+
|
203
|
+
# dirs = RicDisk.find_active_dirs()
|
204
|
+
files_list.each do |dir|
|
205
|
+
dir = File.expand_path(dir)
|
206
|
+
if File.directory?(dir)
|
207
|
+
# if dirs.include?(dir)
|
208
|
+
puts "iterate_through_file_list_for_disks() Legit dir: #{green dir}" if verbose
|
209
|
+
rd = RicDisk.new(Storazzo::Media::AbstractRicDisk.DirFactory(dir))
|
210
|
+
pverbose true, "RicDisk from Factory (woohoo): #{rd}"
|
211
|
+
rd.write_config_yaml_to_disk(dir)
|
212
|
+
# RicDisk.write_config_yaml_to_disk(dir)
|
213
|
+
# RicDisk.calculate_stats_files (CLASS) => will become OBJECT compute_stats_files
|
214
|
+
rd.compute_stats_files() # dir is inutile # TODO
|
215
|
+
else
|
216
|
+
raise("Doesnt seem a dir to me, quitting: #{dir}")
|
222
217
|
end
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
218
|
+
end
|
219
|
+
# end
|
220
|
+
end # /iterate_through_file_list_for_disks
|
221
|
+
|
222
|
+
def config_hash
|
223
|
+
config['Config']
|
224
|
+
end
|
225
|
+
|
226
|
+
def self.safe_instance()
|
227
|
+
puts "This is a safe instance :)"
|
228
|
+
my_config = self.instance()
|
229
|
+
my_config.load
|
230
|
+
my_config
|
231
|
+
end
|
232
|
+
|
233
|
+
def self.get_config
|
234
|
+
self.instance.load unless self.instance.load_called
|
235
|
+
self.instance.get_config
|
236
|
+
end
|
237
|
+
|
238
|
+
# # gem 'after_do'
|
239
|
+
# require 'after_do'
|
240
|
+
# Storazzo::RicDiskConfig.extend AfterDo
|
241
|
+
# Storazzo::RicDiskConfig.after :instance do # |activity| #:pause, :finish, :resurrect, :do_today, :do_another_day
|
242
|
+
# puts yellow("after INSTANCE() has been called I call LOAD!!!")
|
243
|
+
# self.load
|
244
|
+
# #persistor.save activity
|
245
|
+
# end
|
246
|
+
end # class Storazzo::RicDiskConfig
|
241
247
|
end # module Storazzo
|
242
|
-
|
@@ -13,19 +13,19 @@ require_relative "./ric_disk_config"
|
|
13
13
|
=end
|
14
14
|
|
15
15
|
module Storazzo
|
16
|
-
|
17
|
-
|
16
|
+
class Storazzo::RicDiskSampleConfig < Storazzo::RicDiskConfig
|
17
|
+
# include Storazzo::Common
|
18
18
|
|
19
19
|
public
|
20
|
-
def load # _sample_version
|
21
|
-
deb("[RicDiskSampleConfig] Wheew 1! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
22
|
-
super(DefaultGemLocationForTests, :verbose => false )
|
23
|
-
end
|
24
|
-
def load_sample_version
|
25
|
-
deb white("[RicDiskSampleConfig] Wheew 2! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
26
|
-
super(DefaultGemLocationForTests, :verbose => false )
|
27
|
-
end
|
28
|
-
end
|
29
20
|
|
30
|
-
|
21
|
+
def load # _sample_version
|
22
|
+
deb("[RicDiskSampleConfig] Wheew 1! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
23
|
+
super(DefaultGemLocationForTests, :verbose => false)
|
24
|
+
end
|
31
25
|
|
26
|
+
def load_sample_version
|
27
|
+
deb white("[RicDiskSampleConfig] Wheew 2! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
28
|
+
super(DefaultGemLocationForTests, :verbose => false)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end # module Storazzo
|
@@ -1,22 +1,24 @@
|
|
1
1
|
# This class wraps the RDS file: we're going to write this RDS file
|
2
2
|
# directly in the disk: /mount/
|
3
3
|
module Storazzo
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
class Storazzo::RicDiskStatsFile
|
5
|
+
# Please keep these two in sync, until you fix them and DRY the behaviour.
|
6
|
+
DefaultName = "ricdisk_stats_v11.rds" # => RicDiskStatsFile
|
7
|
+
Version = "1.1" # @@version
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
# AttrAccessor for class - thanks StackOverflow from Android since Im in roaming :)
|
10
|
+
class << self
|
11
|
+
attr_accessor :default_name, :version
|
12
|
+
end
|
13
13
|
|
14
14
|
public
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
15
|
+
|
16
|
+
def self.default_name
|
17
|
+
DefaultName
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.version
|
21
|
+
Version
|
21
22
|
end
|
22
|
-
end
|
23
|
+
end
|
24
|
+
end
|