storazzo 0.5.7 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +7 -5
- data/Makefile +8 -6
- data/Rakefile +15 -13
- data/VERSION +1 -1
- data/bin/hello-storazzo +1 -0
- data/bin/ricdisk-magic +33 -30
- data/bin/stats-with-md5 +182 -157
- data/bin/storazzo +112 -114
- data/bin/storazzo-symlink.rb +1 -0
- data/lib/storazzo/colors.rb +77 -36
- data/lib/storazzo/common.rb +72 -68
- data/lib/storazzo/debug.rb +2 -0
- data/lib/storazzo/hashify.rb +7 -5
- data/lib/storazzo/main.rb +59 -49
- data/lib/storazzo/media/abstract_ric_disk.rb +188 -179
- data/lib/storazzo/media/gcs_bucket.rb +76 -58
- data/lib/storazzo/media/local_folder.rb +56 -42
- data/lib/storazzo/media/mount_point.rb +20 -12
- data/lib/storazzo/ric_disk.rb +386 -383
- data/lib/storazzo/ric_disk_config.rb +227 -209
- data/lib/storazzo/ric_disk_sample_config.rb +26 -24
- data/lib/storazzo/ric_disk_statsfile.rb +19 -17
- data/lib/storazzo/ric_disk_ugly.rb +1 -0
- data/lib/storazzo/version.rb +3 -3
- data/lib/storazzo.rb +6 -7
- data/storazzo.gemspec +24 -16
- data/test/benchmark/for_future_use.rb +4 -2
- data/test/benchmark/test_hashing_functions-speed.rb +17 -0
- data/test/bin/new-idea.rb +17 -0
- data/test/bin/storazzo.rb +22 -25
- data/test/media/test_abstract_ric_disk.rb +5 -3
- data/test/media/test_gcs_bucket.rb +24 -23
- data/test/media/test_local_folder.rb +24 -23
- data/test/media/test_mount_point.rb +6 -5
- data/test/test_ric_disk.rb +6 -4
- data/test/test_ric_disk_config.rb +12 -11
- data/test/test_ric_disk_stats_file.rb +5 -3
- data/test/test_storazzo.rb +6 -4
- data/var/test/disks/disk02-full/Rakefile +7 -5
- data/var/test/disks/ricdisk_stats_v11.rds +11 -0
- metadata +13 -21
data/lib/storazzo/main.rb
CHANGED
@@ -1,7 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
# main entrypoint for tests and silly stuff from Makefile..
|
2
4
|
# This Class externalizes all relevant things from other libs while I learn how to do it from there
|
3
5
|
# eg from RicDisk.
|
4
6
|
|
7
|
+
require 'English'
|
5
8
|
module Storazzo
|
6
9
|
# This is the Main Class - an entrypoint to call the meravilles hidden therein.
|
7
10
|
#
|
@@ -12,62 +15,69 @@ module Storazzo
|
|
12
15
|
# Arguments:
|
13
16
|
# message: (String) - optional
|
14
17
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
+
# Can be same name as Module: https://stackoverflow.com/questions/13261474/ruby-modules-and-classes-same-name-in-structure
|
19
|
+
module Storazzo
|
20
|
+
class Main
|
21
|
+
require 'storazzo/colors'
|
22
|
+
extend Storazzo::Colors
|
18
23
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
24
|
+
# version 1.2
|
25
|
+
def self.say_hi(message = nil)
|
26
|
+
str = get_hi(message)
|
27
|
+
puts str
|
28
|
+
str
|
29
|
+
end
|
25
30
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
31
|
+
# salutation
|
32
|
+
def self.get_hi(message = nil)
|
33
|
+
str = "Hello from Storazzo v#{begin
|
34
|
+
white Storazzo.version
|
35
|
+
rescue StandardError
|
36
|
+
"Error: #{$ERROR_INFO}"
|
37
|
+
end}!"
|
38
|
+
str += " Message: '#{yellow message.to_s}'" if message
|
39
|
+
str
|
40
|
+
end
|
32
41
|
|
33
|
-
|
34
|
-
|
42
|
+
def self.all_mounts(opts = {})
|
43
|
+
opts_verbose = opts.fetch :verbose, true
|
35
44
|
|
36
|
-
|
45
|
+
say_hi "Storazzo::Main.all_mounts(): BEGIN - RicDiskVersion is: #{Storazzo::RicDiskUgly::RICDISK_VERSION}"
|
37
46
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
47
|
+
pwhite('1. First I load the config in verbose mode..')
|
48
|
+
config = Storazzo::RicDiskConfig.instance
|
49
|
+
config.load
|
50
|
+
# puts config.object_id
|
51
|
+
# puts Storazzo::RicDiskConfig.instance.object_id
|
52
|
+
pwhite 'TODO(ricc): show a list of all RicDisk relevant mounts' if opts_verbose
|
53
|
+
# d = Storazzo::RicDisk.new
|
54
|
+
config.iterate_through_file_list_for_disks # analyze_local_system
|
55
|
+
# sbrodola_ricdisk("/Volumes/")
|
56
|
+
Storazzo::RicDisk.test
|
57
|
+
Storazzo::RicDisk.find_active_dirs
|
58
|
+
# Storazzo::RicDisk.sbrodola_ricdisk StorazzoMod::root + "./var/disks/"
|
59
|
+
# sbrodola_ricdisk(StorazzoMod::root + "./var/disks/") rescue "[Storazzo::AllMount] SomeError: #{$!}"
|
60
|
+
hi 'Storazzo::Main.all_mounts(): END'
|
61
|
+
end
|
53
62
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
63
|
+
def self.all_tests
|
64
|
+
# include vs extend: https://stackoverflow.com/questions/15097929/ruby-module-require-and-include
|
65
|
+
# => http://www.railstips.org/blog/archives/2009/05/15/include-vs-extend-in-ruby/
|
66
|
+
# include Storazzo::Colors
|
67
|
+
extend Storazzo::Colors
|
59
68
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
69
|
+
pwhite 'All tests BEGIN - todo obsolete this now that I have proper Rake testing suite...'
|
70
|
+
deb 'Maybe debug is enabled?'
|
71
|
+
say_hi
|
72
|
+
# This works with EXTEND..
|
73
|
+
puts(yellow('Just YELLOW 0'))
|
74
|
+
# This reqwuires a INCLUDE.
|
75
|
+
# puts(Storazzo::Colors.yellow "Test YELLOW 1 self")
|
76
|
+
# puts(Colors.yellow "Test YELLOW 1 self")
|
77
|
+
# puts(Colors.green "Test YELLOW 2 ohne self")
|
78
|
+
pwhite 'All tests END'
|
79
|
+
# puts "All tests END"
|
80
|
+
end
|
71
81
|
end
|
72
82
|
end
|
73
83
|
end
|
@@ -1,185 +1,194 @@
|
|
1
|
-
#
|
2
|
-
#require "storazzo/ric_disk/gcs_bucket"
|
3
|
-
#require "gcs_bucket"
|
4
|
-
# require "abstract_ric_disk"
|
5
|
-
|
6
|
-
=begin
|
7
|
-
This class is an Abstract Class which forces its 3 inquilines to implement
|
8
|
-
a number of functions, namely:
|
9
|
-
|
10
|
-
def self.list_all()
|
11
|
-
def self.list_all_with_type()
|
12
|
-
|
13
|
-
Note this needs to work without weird overrides, like:
|
14
|
-
|
15
|
-
def self.list_all_with_type(config=nil) # BAD
|
16
|
-
def self.list_all_with_type() # GOOD
|
17
|
-
|
18
|
-
|
19
|
-
=end
|
20
|
-
|
21
|
-
module Storazzo::Media
|
22
|
-
class Storazzo::Media::AbstractRicDisk
|
23
|
-
# include Storazzo::Common
|
24
|
-
extend Storazzo::Common
|
25
|
-
|
26
|
-
# DefaultStatsFilename = Storazzo::RicDiskStatsFile.default_name
|
27
|
-
|
28
|
-
# looks like there's not Abstract Class in Ruby, but also SO says he best
|
29
|
-
# way to do this is this:
|
30
|
-
|
31
|
-
# attr_accessor :name, :description, :ricdisk_file, :local_mountpoint, :wr, :path, :ricdisk_file_empty, :size, :active_dirs
|
32
|
-
attr_accessor :description
|
33
|
-
|
34
|
-
########################
|
35
|
-
# Abstract methods START
|
36
|
-
########################
|
37
|
-
def initialize(local_mount)
|
38
|
-
deb "[AbstractRicDisk.init()] Some child of AbstractRicDisk (#{self}) called me! Yummie." # disable when you dont need me anymore..
|
39
|
-
@description ||= "Not provided"
|
40
|
-
# validate
|
41
|
-
end
|
42
|
-
|
43
|
-
def self.list_all
|
44
|
-
raise "[AbstractRicDiskc::self.list_all] You should override this, says StackOverflow and Riccardo. Class=#{self.class}. Self=#{self}"
|
45
|
-
end
|
46
|
-
|
47
|
-
def self.list_all_with_type
|
48
|
-
raise "[AbstractRicDisk::self.list_all_with_type] You should override this, says StackOverflow and Riccardo. Class=#{self.class}. Self=#{self}"
|
49
|
-
end
|
50
|
-
|
51
|
-
def self.super_duper_list_all_with_type
|
52
|
-
deb "Would be cool to be able to enumerate them all.."
|
53
|
-
GcsBucket.list_all_with_type +
|
54
|
-
MountPoint.list_all_with_type +
|
55
|
-
LocalFolder.list_all_with_type
|
56
|
-
end
|
57
|
-
|
58
|
-
def parse(opts = {})
|
59
|
-
raise "[AbstractRicDiskc::parse] You should override this, says StackOverflow and Riccardo. Class=#{self.class}."
|
60
|
-
end
|
61
|
-
|
62
|
-
def writeable?
|
63
|
-
raise "[AbstractRicDiskc::writeable] You should override this in class=#{self.class}, says StackOverflow and Riccardo"
|
64
|
-
end
|
65
|
-
|
66
|
-
def get_local_mountpoint
|
67
|
-
# raise "[AbstractRicDiskc::get_local_mountpoint] You should override this, says StackOverflow and Riccardo"
|
68
|
-
raise "You forgot to set local_mountpoint in the constructor for your class, you cheeky little one!" if local_mountpoint.nil?
|
69
|
-
|
70
|
-
local_mountpoint
|
71
|
-
end
|
72
|
-
|
73
|
-
def check_implemented_correctly
|
74
|
-
# raise "[AbstractRicDiskc] You should override this, says StackOverflow and Riccardo"
|
75
|
-
raise "no @local_mountpoint" unless exists?(@local_mountpoint)
|
76
|
-
end
|
77
|
-
########################
|
78
|
-
# Abstract methods END
|
79
|
-
########################
|
80
|
-
|
81
|
-
########################
|
82
|
-
# Automated methods now...
|
83
|
-
########################
|
84
|
-
def stats_file_smart_fullpath # stats_file_default_location()
|
85
|
-
# if its writeable... if not we'll think about it later.
|
86
|
-
if writeable?
|
87
|
-
return "#{get_local_mountpoint}/#{self.default_stats_filename}"
|
88
|
-
else
|
89
|
-
# if not writeable, I will:
|
90
|
-
# 1. create a dir based on its unique format.
|
91
|
-
# 2. create a file of same look and feel (alternative - used a DASH)
|
92
|
-
return "TODO FIXME {get_local_folder}/#{unique_id}::#{self.default_stats_filename}"
|
93
|
-
# "{get_local_folder}"/#{unique_id}/#{Storazzo::RicDiskStatsFile.default_name}"
|
94
|
-
end
|
95
|
-
end
|
1
|
+
# frozen_string_literal: true
|
96
2
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
# Expand path to make it as unique as possible...
|
102
|
-
hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
103
|
-
"MD5::v1::#{hash}"
|
104
|
-
end
|
105
|
-
|
106
|
-
def to_s(verbose = false)
|
107
|
-
return to_verbose_s() if verbose
|
108
|
-
|
109
|
-
readable_class = self.class.to_s.split('::').last # Storazzo::Media::LocalFolder => LocalFolder
|
110
|
-
my_keys = self.instance_variables # wow!
|
111
|
-
"S:M:#{readable_class}(path=#{path}, r/w=#{wr}, keys=#{my_keys})"
|
112
|
-
end
|
113
|
-
|
114
|
-
def to_verbose_s
|
115
|
-
h = {}
|
116
|
-
h[:class] = self.class
|
117
|
-
h[:unique_id] = self.unique_id
|
118
|
-
h[:inspect] = self.inspect
|
119
|
-
h[:to_s] = self.to_s
|
120
|
-
h[:local_mountpoint] = local_mountpoint
|
121
|
-
h[:writeable] = self.writeable?
|
122
|
-
h[:stats_file_smart_fullpath] = stats_file_smart_fullpath
|
123
|
-
return h
|
124
|
-
end
|
125
|
-
|
126
|
-
# # Todo check on instances these 3 methods exist
|
127
|
-
def self.abstract_class_mandatory_methods
|
128
|
-
%W{
|
129
|
-
self.list_all
|
130
|
-
self.list_all_with_type
|
131
|
-
local_mountpoint
|
132
|
-
parse
|
133
|
-
writeable?
|
134
|
-
}
|
135
|
-
end
|
136
|
-
|
137
|
-
# Putting here since its same code for MountPoint or generic folder.
|
138
|
-
def parse_block_storage_folder(opts = {})
|
139
|
-
# Storazzo::RicDiskUgly.calculate_stats_files(get_local_mountpoint)
|
140
|
-
rd = Storazzo::RicDisk.new(self)
|
141
|
-
# Storazzo::RicDisk
|
142
|
-
rd.compute_stats_files(opts)
|
143
|
-
# return "42"
|
144
|
-
end
|
145
|
-
|
146
|
-
def self.default_stats_filename
|
147
|
-
raise "Unknown Storazzo::RicDiskStatsFile.default_name 1!!" if Storazzo::RicDiskStatsFile.default_name.nil?
|
148
|
-
|
149
|
-
Storazzo::RicDiskStatsFile.default_name
|
150
|
-
end
|
151
|
-
|
152
|
-
def stats_filename_default_fullpath # if not writeable we need to pick another from stats_file_smart_fullpath()
|
153
|
-
# REDUNDANT, should use stats_file_smart_fullpath instead except on the writeable? part since it would go recursive otherwise.
|
154
|
-
my_stats_filename = self.default_stats_filename
|
155
|
-
# raise "Unknown Storazzo::RicDiskStatsFile.default_name 2 !!" if Storazzo::RicDiskStatsFile.default_name.nil?
|
156
|
-
"#{local_mountpoint}/#{my_stats_filename}"
|
157
|
-
end
|
158
|
-
|
159
|
-
def validate(opts = {})
|
160
|
-
opt_verbose = opts.fetch :verbose, false
|
161
|
-
pverbose opt_verbose, "validate(): [BROKEN] We're trying to see if your object is valid, across 3 possible sub-classes."
|
162
|
-
# 1. check for
|
163
|
-
raise "Unknown local mount " unless local_mount.is_a?(String)
|
164
|
-
|
165
|
-
# 2. check thaty writeable? is true or false
|
166
|
-
my_writeable = wr
|
167
|
-
raise "Writeable is not boolean" unless (my_writeable.is_a? TrueClass or my_writeable.is_a? FalseClass)
|
168
|
-
end
|
169
|
-
|
170
|
-
# TODO use a proper Factory pattern.
|
171
|
-
def self.DirFactory(path)
|
172
|
-
raise "I need a path/directory string: #{path}" unless path.is_a?(String)
|
3
|
+
# require "storazzo/ric_disk/abstract_ric_disk"
|
4
|
+
# require "storazzo/ric_disk/gcs_bucket"
|
5
|
+
# require "gcs_bucket"
|
6
|
+
# require "abstract_ric_disk"
|
173
7
|
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
8
|
+
# This class is an Abstract Class which forces its 3 inquilines to implement
|
9
|
+
# a number of functions, namely:
|
10
|
+
#
|
11
|
+
# def self.list_all()
|
12
|
+
# def self.list_all_with_type()
|
13
|
+
#
|
14
|
+
# Note this needs to work without weird overrides, like:
|
15
|
+
#
|
16
|
+
# def self.list_all_with_type(config=nil) # BAD
|
17
|
+
# def self.list_all_with_type() # GOOD
|
18
|
+
#
|
19
|
+
#
|
20
|
+
|
21
|
+
module Storazzo
|
22
|
+
module Media
|
23
|
+
module Storazzo
|
24
|
+
module Media
|
25
|
+
class AbstractRicDisk
|
26
|
+
# include Storazzo::Common
|
27
|
+
extend Storazzo::Common
|
28
|
+
|
29
|
+
# DefaultStatsFilename = Storazzo::RicDiskStatsFile.default_name
|
30
|
+
|
31
|
+
# looks like there's not Abstract Class in Ruby, but also SO says he best
|
32
|
+
# way to do this is this:
|
33
|
+
|
34
|
+
# attr_accessor :name, :description, :ricdisk_file, :local_mountpoint, :wr, :path, :ricdisk_file_empty, :size, :active_dirs
|
35
|
+
attr_accessor :description
|
36
|
+
|
37
|
+
########################
|
38
|
+
# Abstract methods START
|
39
|
+
########################
|
40
|
+
def initialize(_local_mount)
|
41
|
+
deb "[AbstractRicDisk.init()] Some child of AbstractRicDisk (#{self}) called me! Yummie." # disable when you dont need me anymore..
|
42
|
+
@description ||= 'Not provided'
|
43
|
+
# validate
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.list_all
|
47
|
+
raise "[AbstractRicDiskc::self.list_all] You should override this, says StackOverflow and Riccardo. Class=#{self.class}. Self=#{self}"
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.list_all_with_type
|
51
|
+
raise "[AbstractRicDisk::self.list_all_with_type] You should override this, says StackOverflow and Riccardo. Class=#{self.class}. Self=#{self}"
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.super_duper_list_all_with_type
|
55
|
+
deb 'Would be cool to be able to enumerate them all..'
|
56
|
+
GcsBucket.list_all_with_type +
|
57
|
+
MountPoint.list_all_with_type +
|
58
|
+
LocalFolder.list_all_with_type
|
59
|
+
end
|
60
|
+
|
61
|
+
def parse(_opts = {})
|
62
|
+
raise "[AbstractRicDiskc::parse] You should override this, says StackOverflow and Riccardo. Class=#{self.class}."
|
63
|
+
end
|
64
|
+
|
65
|
+
def writeable?
|
66
|
+
raise "[AbstractRicDiskc::writeable] You should override this in class=#{self.class}, says StackOverflow and Riccardo"
|
67
|
+
end
|
68
|
+
|
69
|
+
def get_local_mountpoint
|
70
|
+
# raise "[AbstractRicDiskc::get_local_mountpoint] You should override this, says StackOverflow and Riccardo"
|
71
|
+
raise 'You forgot to set local_mountpoint in the constructor for your class, you cheeky little one!' if local_mountpoint.nil?
|
72
|
+
|
73
|
+
local_mountpoint
|
74
|
+
end
|
75
|
+
|
76
|
+
def check_implemented_correctly
|
77
|
+
# raise "[AbstractRicDiskc] You should override this, says StackOverflow and Riccardo"
|
78
|
+
raise 'no @local_mountpoint' unless exists?(@local_mountpoint)
|
79
|
+
end
|
80
|
+
########################
|
81
|
+
# Abstract methods END
|
82
|
+
########################
|
83
|
+
|
84
|
+
########################
|
85
|
+
# Automated methods now...
|
86
|
+
########################
|
87
|
+
# stats_file_default_location()
|
88
|
+
def stats_file_smart_fullpath
|
89
|
+
# if its writeable... if not we'll think about it later.
|
90
|
+
if writeable?
|
91
|
+
"#{get_local_mountpoint}/#{default_stats_filename}"
|
92
|
+
else
|
93
|
+
# if not writeable, I will:
|
94
|
+
# 1. create a dir based on its unique format.
|
95
|
+
# 2. create a file of same look and feel (alternative - used a DASH)
|
96
|
+
"TODO FIXME {get_local_folder}/#{unique_id}::#{default_stats_filename}"
|
97
|
+
# "{get_local_folder}"/#{unique_id}/#{Storazzo::RicDiskStatsFile.default_name}"
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# Needs to be some unique id based on its thingy, but might change in the future:
|
102
|
+
# Solutions:
|
103
|
+
# {PHILOSOPHY}::{ID}
|
104
|
+
def unique_id
|
105
|
+
# Expand path to make it as unique as possible...
|
106
|
+
hash = Digest::MD5.hexdigest(File.expand_path(get_local_mountpoint))
|
107
|
+
"MD5::v1::#{hash}"
|
108
|
+
end
|
109
|
+
|
110
|
+
def to_s(verbose = false)
|
111
|
+
return to_verbose_s if verbose
|
112
|
+
|
113
|
+
readable_class = self.class.to_s.split('::').last # Storazzo::Media::LocalFolder => LocalFolder
|
114
|
+
my_keys = instance_variables # wow!
|
115
|
+
"S:M:#{readable_class}(path=#{path}, r/w=#{wr}, keys=#{my_keys})"
|
116
|
+
end
|
117
|
+
|
118
|
+
def to_verbose_s
|
119
|
+
h = {}
|
120
|
+
h[:class] = self.class
|
121
|
+
h[:unique_id] = unique_id
|
122
|
+
h[:inspect] = inspect
|
123
|
+
h[:to_s] = to_s
|
124
|
+
h[:local_mountpoint] = local_mountpoint
|
125
|
+
h[:writeable] = writeable?
|
126
|
+
h[:stats_file_smart_fullpath] = stats_file_smart_fullpath
|
127
|
+
h
|
128
|
+
end
|
129
|
+
|
130
|
+
# # Todo check on instances these 3 methods exist
|
131
|
+
def self.abstract_class_mandatory_methods
|
132
|
+
%w[
|
133
|
+
self.list_all
|
134
|
+
self.list_all_with_type
|
135
|
+
local_mountpoint
|
136
|
+
parse
|
137
|
+
writeable?
|
138
|
+
]
|
139
|
+
end
|
140
|
+
|
141
|
+
# Putting here since its same code for MountPoint or generic folder.
|
142
|
+
def parse_block_storage_folder(opts = {})
|
143
|
+
# Storazzo::RicDiskUgly.calculate_stats_files(get_local_mountpoint)
|
144
|
+
rd = Storazzo::RicDisk.new(self)
|
145
|
+
# Storazzo::RicDisk
|
146
|
+
rd.compute_stats_files(opts)
|
147
|
+
# return "42"
|
148
|
+
end
|
149
|
+
|
150
|
+
def self.default_stats_filename
|
151
|
+
raise 'Unknown Storazzo::RicDiskStatsFile.default_name 1!!' if Storazzo::RicDiskStatsFile.default_name.nil?
|
152
|
+
|
153
|
+
Storazzo::RicDiskStatsFile.default_name
|
154
|
+
end
|
155
|
+
|
156
|
+
# if not writeable we need to pick another from stats_file_smart_fullpath()
|
157
|
+
def stats_filename_default_fullpath
|
158
|
+
# REDUNDANT, should use stats_file_smart_fullpath instead except on the writeable? part since it would go recursive otherwise.
|
159
|
+
my_stats_filename = default_stats_filename
|
160
|
+
# raise "Unknown Storazzo::RicDiskStatsFile.default_name 2 !!" if Storazzo::RicDiskStatsFile.default_name.nil?
|
161
|
+
"#{local_mountpoint}/#{my_stats_filename}"
|
162
|
+
end
|
163
|
+
|
164
|
+
def validate(opts = {})
|
165
|
+
opt_verbose = opts.fetch :verbose, false
|
166
|
+
pverbose opt_verbose,
|
167
|
+
"validate(): [BROKEN] We're trying to see if your object is valid, across 3 possible sub-classes."
|
168
|
+
# 1. check for
|
169
|
+
raise 'Unknown local mount ' unless local_mount.is_a?(String)
|
170
|
+
|
171
|
+
# 2. check thaty writeable? is true or false
|
172
|
+
my_writeable = wr
|
173
|
+
raise 'Writeable is not boolean' unless my_writeable.is_a?(TrueClass) || my_writeable.is_a?(FalseClass)
|
174
|
+
end
|
175
|
+
|
176
|
+
# TODO: use a proper Factory pattern.
|
177
|
+
def self.DirFactory(path)
|
178
|
+
raise "I need a path/directory string: #{path}" unless path.is_a?(String)
|
179
|
+
|
180
|
+
deb 'TODO: if coincides with MountPoint, instance THAT'
|
181
|
+
# if path in Storazzo::Media::MountPoint.list_all_mount_points
|
182
|
+
# then return ...
|
183
|
+
if path =~ %r{^gs://}
|
184
|
+
deb 'Smells like GCS'
|
185
|
+
return GcsBucket.new(path)
|
186
|
+
end
|
187
|
+
deb 'Smells like LocalFolder :)'
|
188
|
+
LocalFolder.new(path)
|
189
|
+
end
|
190
|
+
end
|
180
191
|
end
|
181
|
-
deb "Smells like LocalFolder :)"
|
182
|
-
return LocalFolder.new(path)
|
183
192
|
end
|
184
193
|
end
|
185
194
|
end
|
@@ -1,71 +1,89 @@
|
|
1
|
-
|
2
|
-
# class Storazzo::Media::AbstractRicDisk
|
3
|
-
class Storazzo::Media::GcsBucket < Storazzo::Media::AbstractRicDisk
|
4
|
-
extend Storazzo::Common
|
1
|
+
# frozen_string_literal: true
|
5
2
|
|
6
|
-
|
3
|
+
require 'English'
|
7
4
|
|
8
|
-
|
9
|
-
|
5
|
+
module Storazzo
|
6
|
+
module Media
|
7
|
+
# class Storazzo::Media::AbstractRicDisk
|
8
|
+
module Storazzo
|
9
|
+
module Media
|
10
|
+
class GcsBucket < Storazzo::Media::AbstractRicDisk
|
11
|
+
extend Storazzo::Common
|
10
12
|
|
11
|
-
|
12
|
-
@description = "MountPoint in '#{local_mount}' pointing at TODO with mount options = TODO"
|
13
|
-
project_id ||= _autodetect_project_id()
|
14
|
-
@project_id = project_id
|
15
|
-
raise "Sorry local mount doesnt exist!" unless File.exist?(@local_mountpoint)
|
13
|
+
attr_accessor :project_id
|
16
14
|
|
17
|
-
|
18
|
-
|
19
|
-
end
|
15
|
+
def initialize(local_mount, project_id = nil)
|
16
|
+
deb '[Storazzo::Media::GcsBucket] initialize'
|
20
17
|
|
21
|
-
|
22
|
-
|
23
|
-
|
18
|
+
@local_mountpoint = File.expand_path(local_mount)
|
19
|
+
@description = "MountPoint in '#{local_mount}' pointing at TODO with mount options = TODO"
|
20
|
+
project_id ||= _autodetect_project_id
|
21
|
+
@project_id = project_id
|
22
|
+
raise 'Sorry local mount doesnt exist!' unless File.exist?(@local_mountpoint)
|
24
23
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
@wr = writeable? # File.writable?(stats_filename_default_fullpath) # .writeable? stats_file_smart_fullpath
|
25
|
+
begin
|
26
|
+
super(local_mount)
|
27
|
+
rescue StandardError
|
28
|
+
"SUPER_ERROR(#{local_mount}): #{$ERROR_INFO}"
|
29
|
+
end
|
30
|
+
end
|
29
31
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
# [42, 43]
|
34
|
-
# deb config.get_local_folders
|
35
|
-
# TODO
|
36
|
-
pverbose true,
|
37
|
-
"TODO(ricc): also add gsutil ls. For that please use the new 'list_all_with_type' (Id refactor it but Im afraid of leaving bad code dangling so before a proper refactor lets implement both side by side"
|
38
|
-
config.get_bucket_paths
|
39
|
-
end
|
32
|
+
def _autodetect_project_id
|
33
|
+
'todo-my-project-id-123' # TODO: fix
|
34
|
+
end
|
40
35
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
# getFromConfig
|
47
|
-
deb "I'm now returning a 'complex' array to tell the caller what kind of element they're getting, eg: GCS from Config Yaml, vs GCS from gsutil ls "
|
48
|
-
list_from_config_with_type = config.get_bucket_paths.map { |path| [:config_gcs_bucket, path] }
|
49
|
-
ret_list = list_from_config_with_type
|
50
|
-
if (config.gcs_enabled?) # should be if GCS enabled :) project_id
|
51
|
-
# so I concatenate Apples with Bananas with names
|
52
|
-
#ret_list += list_available_buckets(config.project_id).map { |path|
|
53
|
-
ret_list += list_available_buckets.map { |path|
|
54
|
-
[:gsutil_ls_gcs_bucket, path]
|
55
|
-
}
|
56
|
-
else
|
57
|
-
deb "list_all_with_type(): empty project id. Skipping"
|
58
|
-
end
|
36
|
+
# (config = nil)
|
37
|
+
def self.list_all
|
38
|
+
# get lisrts from Config singletone
|
39
|
+
# puts " self.list_all: loading config "
|
40
|
+
config ||= Storazzo::RicDiskConfig.instance # # ).get_config
|
59
41
|
|
60
|
-
|
61
|
-
|
42
|
+
config.load # in case I need to load it for the first time
|
43
|
+
# puts config['Config']['AdditionalMountDirs']
|
44
|
+
# puts "TODO see config: #{config}"
|
45
|
+
# [42, 43]
|
46
|
+
# deb config.get_local_folders
|
47
|
+
# TODO
|
48
|
+
pverbose true,
|
49
|
+
"TODO(ricc): also add gsutil ls. For that please use the new 'list_all_with_type' (Id refactor it but Im afraid of leaving bad code dangling so before a proper refactor lets implement both side by side"
|
50
|
+
config.get_bucket_paths
|
51
|
+
end
|
52
|
+
|
53
|
+
# (config = nil)
|
54
|
+
def self.list_all_with_type
|
55
|
+
deb 'GCS::list_all_with_type() -- pull config'
|
56
|
+
config ||= Storazzo::RicDiskConfig.instance
|
57
|
+
config.load # in case I need to load it for the first time
|
58
|
+
# puts "GCS::list_all_with_type() -- TODO(ricc): also add gsutil ls"
|
59
|
+
# getFromConfig
|
60
|
+
deb "I'm now returning a 'complex' array to tell the caller what kind of element they're getting, eg: GCS from Config Yaml, vs GCS from gsutil ls "
|
61
|
+
list_from_config_with_type = config.get_bucket_paths.map do |path|
|
62
|
+
[:config_gcs_bucket, path]
|
63
|
+
end
|
64
|
+
ret_list = list_from_config_with_type
|
65
|
+
if config.gcs_enabled? # should be if GCS enabled :) project_id
|
66
|
+
# so I concatenate Apples with Bananas with names
|
67
|
+
# ret_list += list_available_buckets(config.project_id).map { |path|
|
68
|
+
ret_list += list_available_buckets.map do |path|
|
69
|
+
[:gsutil_ls_gcs_bucket, path]
|
70
|
+
end
|
71
|
+
else
|
72
|
+
deb 'list_all_with_type(): empty project id. Skipping'
|
73
|
+
end
|
62
74
|
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
75
|
+
ret_list
|
76
|
+
end
|
77
|
+
|
78
|
+
def self.list_available_buckets(_opts = {})
|
79
|
+
# project_id = opts.fetch :project_id, nil
|
80
|
+
# list_of_buckets = `gsutil ls --project '#{project_id}'`.chomp.split("\n")
|
81
|
+
list_of_buckets = `gsutil ls`.chomp.split("\n")
|
82
|
+
deb "list_of_buckets: #{list_of_buckets}"
|
83
|
+
list_of_buckets
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
69
87
|
end
|
70
88
|
end
|
71
89
|
end
|