storazzo 0.4.9 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +5 -4
- data/Rakefile +8 -11
- data/VERSION +1 -1
- data/bin/hello-storazzo +0 -1
- data/bin/ricdisk-magic +50 -56
- data/bin/stats-with-md5 +288 -220
- data/lib/storazzo/colors.rb +43 -45
- data/lib/storazzo/common.rb +52 -46
- data/lib/storazzo/debug.rb +5 -8
- data/lib/storazzo/hashify.rb +44 -43
- data/lib/storazzo/main.rb +40 -41
- data/lib/storazzo/media/abstract_ric_disk.rb +162 -151
- data/lib/storazzo/media/gcs_bucket.rb +52 -51
- data/lib/storazzo/media/local_folder.rb +43 -44
- data/lib/storazzo/media/mount_point.rb +8 -9
- data/lib/storazzo/ric_disk.rb +153 -170
- data/lib/storazzo/ric_disk_config.rb +224 -219
- data/lib/storazzo/ric_disk_sample_config.rb +12 -12
- data/lib/storazzo/ric_disk_statsfile.rb +17 -15
- data/lib/storazzo/ric_disk_ugly.rb +35 -38
- data/lib/storazzo/version.rb +7 -7
- data/lib/storazzo.rb +36 -36
- data/storazzo.gemspec +22 -21
- data/test/media/test_abstract_ric_disk.rb +12 -16
- data/test/media/test_gcs_bucket.rb +44 -47
- data/test/media/test_local_folder.rb +134 -136
- data/test/media/test_mount_point.rb +18 -19
- data/test/test_ric_disk.rb +12 -13
- data/test/test_ric_disk_config.rb +20 -24
- data/test/test_ric_disk_stats_file.rb +13 -14
- data/test/test_storazzo.rb +26 -26
- data/var/test/disks/disk02-full/Rakefile +4 -4
- metadata +2 -2
@@ -1,12 +1,11 @@
|
|
1
1
|
require 'singleton'
|
2
2
|
require 'yaml'
|
3
3
|
|
4
|
-
#require 'storazzo/media/abstract_ric_disk'
|
5
|
-
Dir[File.dirname(__FILE__) + '/../lib/*.rb'].each do |file|
|
6
|
-
|
4
|
+
# require 'storazzo/media/abstract_ric_disk'
|
5
|
+
Dir[File.dirname(__FILE__) + '/../lib/*.rb'].each do |file|
|
6
|
+
require File.basename(file, File.extname(file))
|
7
7
|
end
|
8
|
-
#require_all 'media/directory'
|
9
|
-
|
8
|
+
# require_all 'media/directory'
|
10
9
|
|
11
10
|
=begin
|
12
11
|
This is a singleton class. You call me this way..
|
@@ -23,220 +22,226 @@ end
|
|
23
22
|
=end
|
24
23
|
|
25
24
|
module Storazzo
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
public
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
end
|
98
|
-
|
99
|
-
# Obsolete, call another class instead.
|
100
|
-
def load_sample_version
|
101
|
-
raise "DEPRECATED! USE SampleRicDiskConfig.load() instead!"
|
102
|
-
end
|
103
|
-
|
104
|
-
def config_ver
|
105
|
-
raise "I cant compute Version since I cant compute @config. Are you sure you didnt instance this Singleton without calling load?" if @config.nil?
|
106
|
-
@config['apiVersion'] # rescue :StillUnknown
|
107
|
-
#config['ConfigVersion']
|
108
|
-
end
|
109
|
-
def config_default_folder
|
110
|
-
#self.
|
111
|
-
@config['Config']['DefaultFolder'] #rescue "Unknown config_default_folder: #{$!}"
|
112
|
-
end
|
113
|
-
def config_project_id
|
114
|
-
@config['Config']['Backends']['GoogleCloudStorage']['ProjectId']
|
115
|
-
end
|
116
|
-
def already_loaded?
|
117
|
-
#return
|
118
|
-
load_called == true
|
119
|
-
end
|
120
|
-
|
121
|
-
def to_s
|
122
|
-
size = File.size(@config_file) rescue -1
|
123
|
-
#puts yellow "DEB: #{@config["apiVersion"]}"
|
124
|
-
#"RicDiskConfig(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
125
|
-
"POLY_#{self.class}_(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
126
|
-
end
|
127
|
-
|
128
|
-
def to_verbose_s
|
129
|
-
h = {}
|
130
|
-
h[:description] = "This is a Verbose Hash describing a RicDiskConfig or its child RicDiskSampleConfig to understand why it keeps failing.."
|
131
|
-
h[:to_s] = self.to_s
|
132
|
-
h[:class] = self.class
|
133
|
-
h[:file] = __FILE__
|
134
|
-
h[:id] = self.object_id
|
135
|
-
h[:get_bucket_paths] = self.get_bucket_paths()
|
136
|
-
h[:get_local_folders] = self.get_local_folders()
|
137
|
-
h[:config_project_id] = self.config_project_id()
|
138
|
-
return h
|
139
|
-
end
|
140
|
-
|
141
|
-
def get_config(opts={})
|
142
|
-
return load(opts) if @config.nil?
|
143
|
-
@config
|
144
|
-
end
|
145
|
-
|
146
|
-
def self.gem_default_config_path
|
147
|
-
Storazzo.root + "/etc/storazzo_config.sample.yaml"
|
25
|
+
# class Storazzo::Blah
|
26
|
+
# end
|
27
|
+
|
28
|
+
class Storazzo::RicDiskConfig
|
29
|
+
include Singleton
|
30
|
+
include Storazzo::Common
|
31
|
+
include Storazzo::Colors
|
32
|
+
|
33
|
+
DefaultConfigLocation = File.expand_path "~/.storazzo.yaml"
|
34
|
+
|
35
|
+
DefaultConfigLocations = [
|
36
|
+
File.expand_path("~/.storazzo.yaml"), # HOME
|
37
|
+
File.expand_path("./.storazzo.yaml"), # LOCAL DIR
|
38
|
+
]
|
39
|
+
|
40
|
+
DefaultGemLocationForTests =
|
41
|
+
File.expand_path('../../../', __FILE__) +
|
42
|
+
"/etc/storazzo_config.sample.yaml"
|
43
|
+
|
44
|
+
attr_accessor :config, :config_file, :load_called
|
45
|
+
|
46
|
+
public
|
47
|
+
|
48
|
+
# Load from the first valid config.
|
49
|
+
def load(config_path = nil, opts = {})
|
50
|
+
verbose = opts.fetch :verbose, false
|
51
|
+
|
52
|
+
if already_loaded? # and not self.config.nil?
|
53
|
+
pverbose verbose, "[#{self.class}] VERBOSE load: already loaded"
|
54
|
+
return self.config
|
55
|
+
end
|
56
|
+
pverbose verbose, "Storazzo::RicDiskConfig.load(): BEGIN"
|
57
|
+
# trying default location
|
58
|
+
raise "DefaultConfigLocation is not a string" unless DefaultConfigLocation.is_a?(String)
|
59
|
+
|
60
|
+
possible_locations = DefaultConfigLocations # [ default_config_locations .. , "./.storazzo.yaml"]
|
61
|
+
deb "[Config.load] Possible Locations: #{possible_locations}"
|
62
|
+
if config_path.is_a?(String)
|
63
|
+
# possible_locations = [config_path] + possible_locations # .append()
|
64
|
+
possible_locations = possible_locations.unshift(config_path) # append to front
|
65
|
+
# OR: possible_locations.instert(0, config_path)
|
66
|
+
pverbose verbose, "[LOAD] possible_locations: #{possible_locations}"
|
67
|
+
end
|
68
|
+
puts "[VERBOSE] Searching these paths in order: #{possible_locations}" if verbose
|
69
|
+
# bug "This is not always an array of sTRINGS."
|
70
|
+
raise "possible_locations is not an array" unless possible_locations.is_a?(Array)
|
71
|
+
|
72
|
+
possible_locations.each do |possible_path|
|
73
|
+
# ASSERT is a string
|
74
|
+
raise "possible_path is not a string" unless possible_path.is_a?(String)
|
75
|
+
|
76
|
+
deb "before buggy expand_path paz: '#{possible_path}''"
|
77
|
+
paz = File.expand_path(possible_path) rescue possible_path
|
78
|
+
raise "Not a string: #{paz}" unless paz.is_a?(String)
|
79
|
+
|
80
|
+
if File.exists?(paz)
|
81
|
+
@config_file = paz
|
82
|
+
@config = YAML.load(File.read paz) # YAML.load(File.read("file_path"))
|
83
|
+
|
84
|
+
unless (@config["kind"] == 'StorazzoConfig' rescue false)
|
85
|
+
puts white "RicDiskConfig.load(): Sorry this is wrong Config File. Kind=#{@config["kind"] rescue $!}"
|
86
|
+
next
|
87
|
+
end
|
88
|
+
#
|
89
|
+
# pp @config if verbose
|
90
|
+
config_ver = @config["apiVersion"]
|
91
|
+
# puts @config[:ConfigVersion]
|
92
|
+
deb("OK. Storazzo::RicDiskConfig v'#{config_ver}' parsed correctly")
|
93
|
+
# puts "[VERBOSE] RicDiskConfig.to_s: #{self}" if verbose
|
94
|
+
@load_called = true
|
95
|
+
return self.config
|
148
96
|
end
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
#
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
97
|
+
end
|
98
|
+
@load_called = true
|
99
|
+
# only get here if nothing is found
|
100
|
+
raise "No config found across these locations: #{possible_locations}. Consider copying and editing: #{RicDiskConfig.gem_default_config_path}"
|
101
|
+
end
|
102
|
+
|
103
|
+
# Obsolete, call another class instead.
|
104
|
+
def load_sample_version
|
105
|
+
raise "DEPRECATED! USE SampleRicDiskConfig.load() instead!"
|
106
|
+
end
|
107
|
+
|
108
|
+
def config_ver
|
109
|
+
raise "I cant compute Version since I cant compute @config. Are you sure you didnt instance this Singleton without calling load?" if @config.nil?
|
110
|
+
|
111
|
+
@config['apiVersion'] # rescue :StillUnknown
|
112
|
+
# config['ConfigVersion']
|
113
|
+
end
|
114
|
+
|
115
|
+
def config_default_folder
|
116
|
+
# self.
|
117
|
+
@config['Config']['DefaultFolder'] # rescue "Unknown config_default_folder: #{$!}"
|
118
|
+
end
|
119
|
+
|
120
|
+
def config_project_id
|
121
|
+
@config['Config']['Backends']['GoogleCloudStorage']['ProjectId']
|
122
|
+
end
|
123
|
+
|
124
|
+
def already_loaded?
|
125
|
+
# return
|
126
|
+
load_called == true
|
127
|
+
end
|
128
|
+
|
129
|
+
def to_s
|
130
|
+
size = File.size(@config_file) rescue -1
|
131
|
+
# puts yellow "DEB: #{@config["apiVersion"]}"
|
132
|
+
# "RicDiskConfig(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
133
|
+
"POLY_#{self.class}_(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
134
|
+
end
|
135
|
+
|
136
|
+
def to_verbose_s
|
137
|
+
h = {}
|
138
|
+
h[:description] =
|
139
|
+
"This is a Verbose Hash describing a RicDiskConfig or its child RicDiskSampleConfig to understand why it keeps failing.."
|
140
|
+
h[:to_s] = self.to_s
|
141
|
+
h[:class] = self.class
|
142
|
+
h[:file] = __FILE__
|
143
|
+
h[:id] = self.object_id
|
144
|
+
h[:get_bucket_paths] = self.get_bucket_paths()
|
145
|
+
h[:get_local_folders] = self.get_local_folders()
|
146
|
+
h[:config_project_id] = self.config_project_id()
|
147
|
+
return h
|
148
|
+
end
|
149
|
+
|
150
|
+
def get_config(opts = {})
|
151
|
+
return load(opts) if @config.nil?
|
152
|
+
|
153
|
+
@config
|
154
|
+
end
|
155
|
+
|
156
|
+
def self.gem_default_config_path
|
157
|
+
Storazzo.root + "/etc/storazzo_config.sample.yaml"
|
158
|
+
end
|
159
|
+
|
160
|
+
# Storazzo::RicDiskConfig
|
161
|
+
|
162
|
+
# returns all folders from file which are Directories
|
163
|
+
# This method is FLAKY! Sometimes gives error.
|
164
|
+
# LocalFolderTest#test_show_all_shouldnt_fail_and_should_return_a_non_empty_array:
|
165
|
+
# TypeError: no implicit conversion of Hash into String
|
166
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:38:in `expand_path'
|
167
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:38:in `block in load'
|
168
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:37:in `each'
|
169
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:37:in `load'
|
170
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:83:in `get_config'
|
171
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:95:in `get_local_folders'
|
172
|
+
def get_local_folders
|
173
|
+
config = get_config
|
174
|
+
# puts config['Config']['AdditionalMountDirs']
|
175
|
+
config['Config']['AdditionalMountDirs'].map { |folder|
|
176
|
+
File.expand_path(folder) rescue folder # TypeError: no implicit conversion of Hash into String
|
177
|
+
}.filter { |f| File.directory?(f) }
|
178
|
+
end
|
179
|
+
|
180
|
+
def get_bucket_paths
|
181
|
+
get_config['Config']['Backends']['GoogleCloudStorage']['BucketPaths'].map { |complex_gcs_struct|
|
182
|
+
complex_gcs_struct['path']
|
183
|
+
}
|
184
|
+
end
|
185
|
+
|
186
|
+
# UGLY CODE, copipasted from binary for ARGV, ex autosbrodola
|
187
|
+
def iterate_through_file_list_for_disks(files_list = [], opts = {})
|
188
|
+
verbose = opts.fetch :verbose, false
|
189
|
+
raise "[iterate_through_file_list_for_disks] Wrong input, I need an array here: #{files_list} " unless files_list.is_a?(Array)
|
190
|
+
|
191
|
+
# I decided i wont accept an emopty list, this is not how you use the gem, you lazy XXX!
|
192
|
+
# if files_list == [] # or files_list.nil? # empty -> ALL
|
193
|
+
# deb "iterate_through_file_list_for_disks(): no args provided"
|
194
|
+
# dirs = RicDisk.find_active_dirs()
|
195
|
+
# puts "DEB find_active_dirs: #{green dirs}"
|
196
|
+
# dirs.each {|dir|
|
197
|
+
# RicDisk.write_config_yaml_to_disk(dir)
|
198
|
+
# RicDisk.calculate_stats_files(dir) # dir is inutile
|
199
|
+
# } # TODO refactor in option sbrodola_afterwards=true. :)
|
200
|
+
# else
|
201
|
+
puts "iterate_through_file_list_for_disks(): I consider files_list as a list of directories to parse :)" if verbose
|
202
|
+
|
203
|
+
# dirs = RicDisk.find_active_dirs()
|
204
|
+
files_list.each do |dir|
|
205
|
+
dir = File.expand_path(dir)
|
206
|
+
if File.directory?(dir)
|
207
|
+
# if dirs.include?(dir)
|
208
|
+
puts "iterate_through_file_list_for_disks() Legit dir: #{green dir}" if verbose
|
209
|
+
rd = RicDisk.new(Storazzo::Media::AbstractRicDisk.DirFactory(dir))
|
210
|
+
pverbose true, "RicDisk from Factory (woohoo): #{rd}"
|
211
|
+
rd.write_config_yaml_to_disk(dir)
|
212
|
+
# RicDisk.write_config_yaml_to_disk(dir)
|
213
|
+
# RicDisk.calculate_stats_files (CLASS) => will become OBJECT compute_stats_files
|
214
|
+
rd.compute_stats_files() # dir is inutile # TODO
|
215
|
+
else
|
216
|
+
raise("Doesnt seem a dir to me, quitting: #{dir}")
|
222
217
|
end
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
218
|
+
end
|
219
|
+
# end
|
220
|
+
end # /iterate_through_file_list_for_disks
|
221
|
+
|
222
|
+
def config_hash
|
223
|
+
config['Config']
|
224
|
+
end
|
225
|
+
|
226
|
+
def self.safe_instance()
|
227
|
+
puts "This is a safe instance :)"
|
228
|
+
my_config = self.instance()
|
229
|
+
my_config.load
|
230
|
+
my_config
|
231
|
+
end
|
232
|
+
|
233
|
+
def self.get_config
|
234
|
+
self.instance.load unless self.instance.load_called
|
235
|
+
self.instance.get_config
|
236
|
+
end
|
237
|
+
|
238
|
+
# # gem 'after_do'
|
239
|
+
# require 'after_do'
|
240
|
+
# Storazzo::RicDiskConfig.extend AfterDo
|
241
|
+
# Storazzo::RicDiskConfig.after :instance do # |activity| #:pause, :finish, :resurrect, :do_today, :do_another_day
|
242
|
+
# puts yellow("after INSTANCE() has been called I call LOAD!!!")
|
243
|
+
# self.load
|
244
|
+
# #persistor.save activity
|
245
|
+
# end
|
246
|
+
end # class Storazzo::RicDiskConfig
|
241
247
|
end # module Storazzo
|
242
|
-
|
@@ -13,19 +13,19 @@ require_relative "./ric_disk_config"
|
|
13
13
|
=end
|
14
14
|
|
15
15
|
module Storazzo
|
16
|
-
|
17
|
-
|
16
|
+
class Storazzo::RicDiskSampleConfig < Storazzo::RicDiskConfig
|
17
|
+
# include Storazzo::Common
|
18
18
|
|
19
19
|
public
|
20
|
-
def load # _sample_version
|
21
|
-
deb("[RicDiskSampleConfig] Wheew 1! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
22
|
-
super(DefaultGemLocationForTests, :verbose => false )
|
23
|
-
end
|
24
|
-
def load_sample_version
|
25
|
-
deb white("[RicDiskSampleConfig] Wheew 2! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
26
|
-
super(DefaultGemLocationForTests, :verbose => false )
|
27
|
-
end
|
28
|
-
end
|
29
20
|
|
30
|
-
|
21
|
+
def load # _sample_version
|
22
|
+
deb("[RicDiskSampleConfig] Wheew 1! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
23
|
+
super(DefaultGemLocationForTests, :verbose => false)
|
24
|
+
end
|
31
25
|
|
26
|
+
def load_sample_version
|
27
|
+
deb white("[RicDiskSampleConfig] Wheew 2! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
28
|
+
super(DefaultGemLocationForTests, :verbose => false)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end # module Storazzo
|
@@ -1,22 +1,24 @@
|
|
1
1
|
# This class wraps the RDS file: we're going to write this RDS file
|
2
2
|
# directly in the disk: /mount/
|
3
3
|
module Storazzo
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
class Storazzo::RicDiskStatsFile
|
5
|
+
# Please keep these two in sync, until you fix them and DRY the behaviour.
|
6
|
+
DefaultName = "ricdisk_stats_v11.rds" # => RicDiskStatsFile
|
7
|
+
Version = "1.1" # @@version
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
# AttrAccessor for class - thanks StackOverflow from Android since Im in roaming :)
|
10
|
+
class << self
|
11
|
+
attr_accessor :default_name, :version
|
12
|
+
end
|
13
13
|
|
14
14
|
public
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
15
|
+
|
16
|
+
def self.default_name
|
17
|
+
DefaultName
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.version
|
21
|
+
Version
|
21
22
|
end
|
22
|
-
end
|
23
|
+
end
|
24
|
+
end
|