storazzo 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +13 -1
- data/README.md +2 -1
- data/VERSION +1 -1
- data/bin/hello-storazzo +3 -1
- data/bin/storazzo +93 -141
- data/bin/storazzo-symlink.rb +1 -1
- data/bin/storazzo-util +1 -1
- data/lib/storazzo/main.rb +2 -5
- data/lib/storazzo/media/abstract_ric_disk.rb +162 -167
- data/lib/storazzo/media/gcs_bucket.rb +64 -69
- data/lib/storazzo/media/local_folder.rb +45 -49
- data/lib/storazzo/media/mount_point.rb +11 -15
- data/lib/storazzo/ric_disk.rb +6 -8
- data/lib/storazzo/ric_disk_config.rb +210 -214
- data/lib/storazzo/ric_disk_sample_config.rb +16 -18
- data/lib/storazzo/ric_disk_statsfile.rb +3 -5
- data/lib/storazzo/search_engine.rb +64 -0
- data/lib/storazzo/version.rb +1 -1
- data/lib/storazzo.rb +1 -7
- data/storazzo.gemspec +4 -3
- data/test/media/test_local_folder.rb +2 -10
- data/test/test_ric_disk.rb +3 -1
- data/var/test/disks/ricdisk_stats_v11.rds +19 -11
- metadata +58 -18
data/lib/storazzo/ric_disk.rb
CHANGED
|
@@ -6,13 +6,12 @@
|
|
|
6
6
|
require 'digest'
|
|
7
7
|
|
|
8
8
|
module Storazzo
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
require 'socket'
|
|
9
|
+
class RicDisk
|
|
10
|
+
include Hashify
|
|
11
|
+
include ::Storazzo::Common
|
|
12
|
+
extend ::Storazzo::Common
|
|
13
|
+
extend ::Storazzo::Colors
|
|
14
|
+
require 'socket'
|
|
16
15
|
|
|
17
16
|
## Instance variables
|
|
18
17
|
|
|
@@ -428,7 +427,6 @@ module Storazzo
|
|
|
428
427
|
# puts("find_active_dirs(): found dirs " + green(active_dirs))
|
|
429
428
|
# return active_dirs
|
|
430
429
|
# end
|
|
431
|
-
end
|
|
432
430
|
end
|
|
433
431
|
end
|
|
434
432
|
|
|
@@ -22,258 +22,254 @@ end
|
|
|
22
22
|
# of the real one - yiikes. How do I fix it? Do I unsingleton it? :) Or do I create TWO singletons? :)
|
|
23
23
|
|
|
24
24
|
module Storazzo
|
|
25
|
-
|
|
26
|
-
|
|
25
|
+
class RicDiskConfig
|
|
26
|
+
include Singleton
|
|
27
|
+
include ::Storazzo::Common
|
|
28
|
+
include ::Storazzo::Colors
|
|
27
29
|
|
|
28
|
-
|
|
29
|
-
class RicDiskConfig
|
|
30
|
-
include Singleton
|
|
31
|
-
include Storazzo::Common
|
|
32
|
-
include Storazzo::Colors
|
|
30
|
+
DefaultConfigLocation = File.expand_path '~/.storazzo.yaml'
|
|
33
31
|
|
|
34
|
-
|
|
32
|
+
DEFAULT_CONFIG_LOCATIONS = [
|
|
33
|
+
File.expand_path('~/.storazzo.yaml'), # HOME
|
|
34
|
+
File.expand_path('./.storazzo.yaml') # LOCAL DIR
|
|
35
|
+
].freeze
|
|
35
36
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
File.expand_path('./.storazzo.yaml') # LOCAL DIR
|
|
39
|
-
].freeze
|
|
37
|
+
DefaultGemLocationForTests =
|
|
38
|
+
"#{File.expand_path('../..', __dir__)}/etc/storazzo_config.sample.yaml"
|
|
40
39
|
|
|
41
|
-
|
|
42
|
-
"#{File.expand_path('../..', __dir__)}/etc/storazzo_config.sample.yaml"
|
|
40
|
+
attr_accessor :config, :config_file, :load_called, :project_id
|
|
43
41
|
|
|
44
|
-
|
|
42
|
+
# Load from the first valid config.
|
|
43
|
+
def load(config_path = nil, opts = {})
|
|
44
|
+
verbose = opts.fetch :verbose, false
|
|
45
45
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
possible_locations =
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
pverbose verbose, "Searching these paths in order: #{possible_locations}"
|
|
67
|
-
# bug "This is not always an array of sTRINGS."
|
|
68
|
-
raise 'possible_locations is not an array' unless possible_locations.is_a?(Array)
|
|
46
|
+
if already_loaded? # and not self.config.nil?
|
|
47
|
+
pverbose verbose, "[#{self.class}] Config.Load: already loaded"
|
|
48
|
+
return config
|
|
49
|
+
end
|
|
50
|
+
pverbose verbose, 'Storazzo::RicDiskConfig.load(): BEGIN'
|
|
51
|
+
# trying default location
|
|
52
|
+
raise 'DefaultConfigLocation is not a string' unless DefaultConfigLocation.is_a?(String)
|
|
53
|
+
|
|
54
|
+
possible_locations = DEFAULT_CONFIG_LOCATIONS.dup # [ default_config_locations .. , "./.storazzo.yaml"]
|
|
55
|
+
|
|
56
|
+
deb "[Config.load] Possible Locations: #{possible_locations}"
|
|
57
|
+
if config_path.is_a?(String)
|
|
58
|
+
# possible_locations = [config_path] + possible_locations # .append()
|
|
59
|
+
possible_locations.unshift(config_path) # append to front
|
|
60
|
+
# OR: possible_locations.instert(0, config_path)
|
|
61
|
+
pverbose verbose, "[LOAD] possible_locations: #{possible_locations}"
|
|
62
|
+
end
|
|
63
|
+
pverbose verbose, "Searching these paths in order: #{possible_locations}"
|
|
64
|
+
# bug "This is not always an array of sTRINGS."
|
|
65
|
+
raise 'possible_locations is not an array' unless possible_locations.is_a?(Array)
|
|
69
66
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
67
|
+
possible_locations.each do |possible_path|
|
|
68
|
+
# ASSERT is a string
|
|
69
|
+
raise 'possible_path is not a string' unless possible_path.is_a?(String)
|
|
73
70
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
71
|
+
deb "before buggy expand_path paz: '#{possible_path}''"
|
|
72
|
+
paz = begin
|
|
73
|
+
File.expand_path(possible_path)
|
|
74
|
+
rescue StandardError
|
|
75
|
+
possible_path
|
|
76
|
+
end
|
|
77
|
+
raise "Not a string: #{paz}" unless paz.is_a?(String)
|
|
81
78
|
|
|
82
|
-
|
|
79
|
+
next unless File.exist?(paz)
|
|
83
80
|
|
|
84
|
-
|
|
85
|
-
|
|
81
|
+
@config_file = paz
|
|
82
|
+
@config = YAML.safe_load(File.read(paz)) # YAML.load(File.read("file_path"))
|
|
86
83
|
|
|
87
|
-
|
|
88
|
-
|
|
84
|
+
unless begin
|
|
85
|
+
@config['kind'] == 'StorazzoConfig'
|
|
86
|
+
rescue StandardError
|
|
87
|
+
false
|
|
88
|
+
end
|
|
89
|
+
puts white "RicDiskConfig.load(): Sorry this is wrong Config File. Kind=#{begin
|
|
90
|
+
@config['kind']
|
|
89
91
|
rescue StandardError
|
|
90
|
-
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
@config['kind']
|
|
94
|
-
rescue StandardError
|
|
95
|
-
$ERROR_INFO
|
|
96
|
-
end}"
|
|
97
|
-
next
|
|
98
|
-
end
|
|
99
|
-
#
|
|
100
|
-
# pp @config if verbose
|
|
101
|
-
config_ver = @config['apiVersion']
|
|
102
|
-
# puts @config[:ConfigVersion]
|
|
103
|
-
deb("OK. Storazzo::RicDiskConfig v'#{config_ver}' parsed correctly")
|
|
104
|
-
@load_called = true
|
|
105
|
-
return config
|
|
92
|
+
$ERROR_INFO
|
|
93
|
+
end}"
|
|
94
|
+
next
|
|
106
95
|
end
|
|
96
|
+
#
|
|
97
|
+
# pp @config if verbose
|
|
98
|
+
config_ver = @config['apiVersion']
|
|
99
|
+
# puts @config[:ConfigVersion]
|
|
100
|
+
deb("OK. Storazzo::RicDiskConfig v'#{config_ver}' parsed correctly")
|
|
107
101
|
@load_called = true
|
|
108
|
-
|
|
109
|
-
raise "No config found across these locations: #{possible_locations}. Consider copying and editing: #{RicDiskConfig.gem_default_config_path}"
|
|
102
|
+
return config
|
|
110
103
|
end
|
|
104
|
+
@load_called = true
|
|
105
|
+
# only get here if nothing is found
|
|
106
|
+
raise "No config found across these locations: #{possible_locations}. Consider copying and editing: #{RicDiskConfig.gem_default_config_path}"
|
|
107
|
+
end
|
|
111
108
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
109
|
+
# Obsolete, call another class instead.
|
|
110
|
+
def load_sample_version
|
|
111
|
+
raise 'DEPRECATED! USE SampleRicDiskConfig.load() instead!'
|
|
112
|
+
end
|
|
116
113
|
|
|
117
|
-
|
|
118
|
-
|
|
114
|
+
def config_ver
|
|
115
|
+
raise 'I cant compute Version since I cant compute @config. Are you sure you didnt instance this Singleton without calling load?' if @config.nil?
|
|
119
116
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
117
|
+
@config['apiVersion'] # rescue :StillUnknown
|
|
118
|
+
# config['ConfigVersion']
|
|
119
|
+
end
|
|
123
120
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
121
|
+
def config_default_folder
|
|
122
|
+
# self.
|
|
123
|
+
@config['Config']['DefaultFolder'] # rescue "Unknown config_default_folder: #{$!}"
|
|
124
|
+
end
|
|
128
125
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
126
|
+
def config_project_id
|
|
127
|
+
@config['Config']['Backends']['GoogleCloudStorage']['ProjectId']
|
|
128
|
+
end
|
|
132
129
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
130
|
+
# doesnt work :/ alias :project_id, :config_project_id
|
|
131
|
+
def project_id
|
|
132
|
+
config_project_id
|
|
133
|
+
end
|
|
137
134
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
135
|
+
# This should return true if and only if the user has configured the YAML
|
|
136
|
+
# to use GCS. For now we can just say TRUE since gsutil ls returns without project_id.
|
|
137
|
+
# However, we might have a flag enabled in storazzo config in next versions.
|
|
138
|
+
def gcs_enabled?
|
|
139
|
+
# true
|
|
140
|
+
defined?(get_config['Config']['Backends']['GoogleCloudStorage'])
|
|
141
|
+
# get_config['Config']['Backends']['GoogleCloudStorage DOESNT EXIST']
|
|
142
|
+
# defined? Backends: GoogleCloudStorage
|
|
143
|
+
end
|
|
147
144
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
145
|
+
def already_loaded?
|
|
146
|
+
# return
|
|
147
|
+
load_called == true
|
|
148
|
+
end
|
|
152
149
|
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
end
|
|
159
|
-
# puts yellow "DEB: #{@config["apiVersion"]}"
|
|
160
|
-
# "RicDiskConfig(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
|
161
|
-
"POLY_#{self.class}_(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
|
150
|
+
def to_s
|
|
151
|
+
size = begin
|
|
152
|
+
File.size(@config_file)
|
|
153
|
+
rescue StandardError
|
|
154
|
+
-1
|
|
162
155
|
end
|
|
156
|
+
# puts yellow "DEB: #{@config["apiVersion"]}"
|
|
157
|
+
# "RicDiskConfig(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
|
158
|
+
"POLY_#{self.class}_(ver=#{config_ver}, file=#{config_file}), #{white(size)} bytes" # - config_default_folder=#{self.config_default_folder}"
|
|
159
|
+
end
|
|
163
160
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
161
|
+
def to_verbose_s
|
|
162
|
+
h = {}
|
|
163
|
+
h[:description] =
|
|
164
|
+
'This is a Verbose Hash describing a RicDiskConfig or its child RicDiskSampleConfig to understand why it keeps failing..'
|
|
165
|
+
h[:to_s] = to_s
|
|
166
|
+
h[:class] = self.class
|
|
167
|
+
h[:file] = __FILE__
|
|
168
|
+
h[:id] = object_id
|
|
169
|
+
h[:get_bucket_paths] = get_bucket_paths
|
|
170
|
+
h[:get_local_folders] = get_local_folders
|
|
171
|
+
h[:config_project_id] = config_project_id
|
|
172
|
+
h
|
|
173
|
+
end
|
|
177
174
|
|
|
178
|
-
|
|
179
|
-
|
|
175
|
+
def get_config(opts = {})
|
|
176
|
+
return load(opts) if @config.nil?
|
|
180
177
|
|
|
181
|
-
|
|
182
|
-
|
|
178
|
+
@config
|
|
179
|
+
end
|
|
183
180
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
181
|
+
def self.gem_default_config_path
|
|
182
|
+
"#{Storazzo.root}/etc/storazzo_config.sample.yaml"
|
|
183
|
+
end
|
|
187
184
|
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
185
|
+
# Storazzo::RicDiskConfig
|
|
186
|
+
|
|
187
|
+
# returns all folders from file which are Directories
|
|
188
|
+
# This method is FLAKY! Sometimes gives error.
|
|
189
|
+
# LocalFolderTest#test_show_all_shouldnt_fail_and_should_return_a_non_empty_array:
|
|
190
|
+
# TypeError: no implicit conversion of Hash into String
|
|
191
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:38:in `expand_path'
|
|
192
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:38:in `block in load'
|
|
193
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:37:in `each'
|
|
194
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:37:in `load'
|
|
195
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:83:in `get_config'
|
|
196
|
+
# /Users/ricc/git/storazzo/lib/storazzo/ric_disk_config.rb:95:in `get_local_folders'
|
|
197
|
+
def get_local_folders
|
|
198
|
+
config = get_config
|
|
199
|
+
# puts config['Config']['AdditionalMountDirs']
|
|
200
|
+
config['Config']['AdditionalMountDirs'].map do |folder|
|
|
201
|
+
File.expand_path(folder)
|
|
202
|
+
rescue StandardError
|
|
203
|
+
folder
|
|
204
|
+
end.filter { |f| File.directory?(f) }
|
|
205
|
+
end
|
|
209
206
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
end
|
|
207
|
+
def get_bucket_paths
|
|
208
|
+
get_config['Config']['Backends']['GoogleCloudStorage']['BucketPaths'].map do |complex_gcs_struct|
|
|
209
|
+
complex_gcs_struct['path']
|
|
214
210
|
end
|
|
211
|
+
end
|
|
215
212
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
end
|
|
213
|
+
# UGLY CODE, copipasted from binary for ARGV, ex autosbrodola
|
|
214
|
+
def iterate_through_file_list_for_disks(files_list = [], opts = {})
|
|
215
|
+
verbose = opts.fetch :verbose, false
|
|
216
|
+
raise "[iterate_through_file_list_for_disks] Wrong input, I need an array here: #{files_list} " unless files_list.is_a?(Array)
|
|
217
|
+
|
|
218
|
+
# I decided i wont accept an emopty list, this is not how you use the gem, you lazy XXX!
|
|
219
|
+
# if files_list == [] # or files_list.nil? # empty -> ALL
|
|
220
|
+
# deb "iterate_through_file_list_for_disks(): no args provided"
|
|
221
|
+
# dirs = RicDisk.find_active_dirs()
|
|
222
|
+
# puts "DEB find_active_dirs: #{green dirs}"
|
|
223
|
+
# dirs.each {|dir|
|
|
224
|
+
# RicDisk.write_config_yaml_to_disk(dir)
|
|
225
|
+
# RicDisk.calculate_stats_files(dir) # dir is inutile
|
|
226
|
+
# } # TODO refactor in option sbrodola_afterwards=true. :)
|
|
227
|
+
# else
|
|
228
|
+
pverbose verbose,
|
|
229
|
+
'iterate_through_file_list_for_disks(): I consider files_list as a list of directories to parse :)'
|
|
230
|
+
|
|
231
|
+
# dirs = RicDisk.find_active_dirs()
|
|
232
|
+
files_list.each do |dir|
|
|
233
|
+
dir = File.expand_path(dir)
|
|
234
|
+
if File.directory?(dir)
|
|
235
|
+
# if dirs.include?(dir)
|
|
236
|
+
pverbose verbose, "iterate_through_file_list_for_disks() Legit dir: #{green dir}"
|
|
237
|
+
rd = RicDisk.new(Storazzo::Media::AbstractRicDisk.DirFactory(dir))
|
|
238
|
+
pverbose verbose, "RicDisk from Factory (woohoo): #{rd}"
|
|
239
|
+
rd.write_config_yaml_to_disk(dir)
|
|
240
|
+
# RicDisk.write_config_yaml_to_disk(dir)
|
|
241
|
+
# RicDisk.calculate_stats_files (CLASS) => will become OBJECT compute_stats_files
|
|
242
|
+
rd.compute_stats_files # dir is inutile # TODO
|
|
243
|
+
else
|
|
244
|
+
raise("Doesnt seem a dir to me, quitting: #{dir}")
|
|
249
245
|
end
|
|
250
|
-
# end
|
|
251
|
-
end
|
|
252
|
-
|
|
253
|
-
def config_hash
|
|
254
|
-
config['Config']
|
|
255
246
|
end
|
|
247
|
+
# end
|
|
248
|
+
end
|
|
256
249
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
my_config.load
|
|
261
|
-
my_config
|
|
262
|
-
end
|
|
250
|
+
def config_hash
|
|
251
|
+
config['Config']
|
|
252
|
+
end
|
|
263
253
|
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
254
|
+
def self.safe_instance
|
|
255
|
+
puts 'This is a safe instance :)'
|
|
256
|
+
my_config = instance
|
|
257
|
+
my_config.load
|
|
258
|
+
my_config
|
|
259
|
+
end
|
|
268
260
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
# Storazzo::RicDiskConfig.after :instance do # |activity| #:pause, :finish, :resurrect, :do_today, :do_another_day
|
|
273
|
-
# puts yellow("after INSTANCE() has been called I call LOAD!!!")
|
|
274
|
-
# self.load
|
|
275
|
-
# #persistor.save activity
|
|
276
|
-
# end
|
|
261
|
+
def self.get_config
|
|
262
|
+
instance.load unless instance.load_called
|
|
263
|
+
instance.get_config
|
|
277
264
|
end
|
|
265
|
+
|
|
266
|
+
# # gem 'after_do'
|
|
267
|
+
# require 'after_do'
|
|
268
|
+
# Storazzo::RicDiskConfig.extend AfterDo
|
|
269
|
+
# Storazzo::RicDiskConfig.after :instance do # |activity| #:pause, :finish, :resurrect, :do_today, :do_another_day
|
|
270
|
+
# puts yellow("after INSTANCE() has been called I call LOAD!!!")
|
|
271
|
+
# self.load
|
|
272
|
+
# #persistor.save activity
|
|
273
|
+
# end
|
|
278
274
|
end
|
|
279
275
|
end
|
|
@@ -7,27 +7,25 @@ require_relative './common'
|
|
|
7
7
|
require_relative './ric_disk_config'
|
|
8
8
|
|
|
9
9
|
module Storazzo
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
# It loads a sample config in local gem etc/. Usuful for testing or if no other config is found..
|
|
10
|
+
# This is a singleton class. You call me this way..
|
|
11
|
+
# You call me with:
|
|
12
|
+
#
|
|
13
|
+
# Storazzo::RicDiskSampleConfig.instance()
|
|
14
|
+
#
|
|
15
|
+
# It loads a sample config in local gem etc/. Usuful for testing or if no other config is found..
|
|
17
16
|
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
class RicDiskSampleConfig < RicDiskConfig
|
|
18
|
+
# include Storazzo::Common
|
|
20
19
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
20
|
+
# _sample_version
|
|
21
|
+
def load
|
|
22
|
+
deb("[RicDiskSampleConfig] Wheew 1! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
|
23
|
+
super(DefaultGemLocationForTests, verbose: false)
|
|
24
|
+
end
|
|
26
25
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
end
|
|
26
|
+
def load_sample_version
|
|
27
|
+
deb white("[RicDiskSampleConfig] Wheew 2! We're NOT destroying the world here. We're actually instancing a second Singleton which is a child of the mother, and this time doing things nicely and Rubily.")
|
|
28
|
+
super(DefaultGemLocationForTests, verbose: false)
|
|
31
29
|
end
|
|
32
30
|
end
|
|
33
31
|
end
|
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Storazzo
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class RicDiskStatsFile
|
|
4
|
+
# This class wraps the RDS file: we're going to write this RDS file
|
|
5
|
+
# directly in the disk: /mount/
|
|
6
|
+
class RicDiskStatsFile
|
|
8
7
|
# Please keep these two in sync, until you fix them and DRY the behaviour.
|
|
9
8
|
DefaultName = 'ricdisk_stats_v11.rds' # => RicDiskStatsFile
|
|
10
9
|
Version = '1.1' # @@version
|
|
@@ -21,6 +20,5 @@ module Storazzo
|
|
|
21
20
|
def self.version
|
|
22
21
|
Version
|
|
23
22
|
end
|
|
24
|
-
end
|
|
25
23
|
end
|
|
26
24
|
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'sqlite3'
|
|
4
|
+
require 'fileutils'
|
|
5
|
+
require 'google/cloud/storage'
|
|
6
|
+
|
|
7
|
+
module Storazzo
|
|
8
|
+
class SearchEngine
|
|
9
|
+
DB_PATH = File.expand_path("~/.storazzo_index.db")
|
|
10
|
+
|
|
11
|
+
def initialize
|
|
12
|
+
@db = SQLite3::Database.new(DB_PATH)
|
|
13
|
+
@db.results_as_hash = true
|
|
14
|
+
create_tables
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def create_tables
|
|
18
|
+
@db.execute <<-SQL
|
|
19
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
20
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
21
|
+
md5 VARCHAR(32),
|
|
22
|
+
size INTEGER,
|
|
23
|
+
path TEXT,
|
|
24
|
+
disk TEXT,
|
|
25
|
+
UNIQUE(disk, path)
|
|
26
|
+
);
|
|
27
|
+
SQL
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def sync_from_gcs
|
|
31
|
+
puts "Syncing metadata from GCS... (Stub)"
|
|
32
|
+
# Here we would use Google::Cloud::Storage to download `.rds` files
|
|
33
|
+
# from the designated GCS metadata bucket and then call `ingest_stats_file`.
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def query(string)
|
|
37
|
+
@db.execute("SELECT * FROM files WHERE path LIKE ? OR disk LIKE ?", ["%#{string}%", "%#{string}%"])
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def ingest_stats_file(file_path, disk_name)
|
|
41
|
+
File.readlines(file_path).each do |line|
|
|
42
|
+
next if line.start_with?('#') || line.strip.empty?
|
|
43
|
+
|
|
44
|
+
# Example format:
|
|
45
|
+
# [file_v1.2] md5 mode type datetime size [content_type] filename
|
|
46
|
+
parts = line.split(' ')
|
|
47
|
+
|
|
48
|
+
# very basic extraction trying to find the `[content_type]` bracket to locate the filename
|
|
49
|
+
content_type_idx = parts.find_index { |p| p.start_with?('[') && p.end_with?(']') && p != parts.first }
|
|
50
|
+
next unless content_type_idx
|
|
51
|
+
|
|
52
|
+
md5 = parts[1]
|
|
53
|
+
size = parts[content_type_idx - 1].to_i
|
|
54
|
+
path = parts[(content_type_idx + 1)..-1].join(' ')
|
|
55
|
+
|
|
56
|
+
begin
|
|
57
|
+
@db.execute("INSERT OR REPLACE INTO files (md5, size, path, disk) VALUES (?, ?, ?, ?)", [md5, size, path, disk_name])
|
|
58
|
+
rescue SQLite3::Exception => e
|
|
59
|
+
puts "Error inserting #{path}: #{e.message}"
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
data/lib/storazzo/version.rb
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
# I;'m copying the DHH philosophy here.
|
|
5
5
|
module Storazzo
|
|
6
6
|
# DHH_VERSION = "0.2.3._TODOFileRead.1" # TODO file.read ../../VERSION . chomp
|
|
7
|
-
RICC_VERSION = File.read('
|
|
7
|
+
RICC_VERSION = File.read(File.expand_path('../../VERSION', __dir__)).chomp
|
|
8
8
|
|
|
9
9
|
def self.version
|
|
10
10
|
RICC_VERSION
|