storazzo 0.5.2 → 0.5.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/LICENSE +20 -1
- data/Makefile +2 -0
- data/Rakefile +11 -0
- data/VERSION +1 -1
- data/bin/stats-with-md5 +0 -36
- data/bin/storazzo +132 -4
- data/lib/storazzo/common.rb +5 -2
- data/lib/storazzo/media/abstract_ric_disk.rb +26 -6
- data/lib/storazzo/media/gcs_bucket.rb +23 -10
- data/lib/storazzo/media/local_folder.rb +2 -2
- data/lib/storazzo/media/mount_point.rb +8 -0
- data/lib/storazzo/ric_disk_config.rb +21 -7
- data/test/benchmark/for_future_use.rb +13 -0
- data/test/bin/storazzo.rb +28 -0
- data/test/media/test_abstract_ric_disk.rb +1 -1
- data/test/media/test_gcs_bucket.rb +46 -13
- data/test/media/test_local_folder.rb +9 -11
- data/test/media/test_mount_point.rb +2 -1
- data/test/test_ric_disk.rb +2 -2
- data/test/test_ric_disk_config.rb +1 -1
- data/var/test/disks/ricdisk_stats_v11.rds +0 -22
- metadata +6 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: eb18e024b54651bcd5bfc08aa3d67691b9dec1a1d725ffd66a4369ab16fbc634
|
4
|
+
data.tar.gz: '0870aa94ba6e3e9393aefea8e7ddcfc5df6103e3ff0942b1c4bc2ee9e0c648b5'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e4c821df6d4345a2b3ead5853583175a443f8f4ae96ff2ca4f04fc1272c6aa7e25dbb2a4116be68bea51f797251967e607c2b3128c3401dd321d8efcdc232fcb
|
7
|
+
data.tar.gz: b6452fc5e1cb20d5808cf82f763b2cfd65529195afb006fd30bcee4d08254b45e3a0fb3cc53c7e8e60ff3be36273636c8142c12036b0d88eefba94393d7da035
|
data/LICENSE
CHANGED
@@ -1 +1,20 @@
|
|
1
|
-
|
1
|
+
Copyright 2022 Riccardo Carlesso (MIT)
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/Makefile
CHANGED
@@ -63,6 +63,8 @@ test-media-subfolder:
|
|
63
63
|
rake test TEST="test/media/*.rb"
|
64
64
|
test-verbose:
|
65
65
|
rake test:verbose --verbose
|
66
|
+
test-silent:
|
67
|
+
RUBYOPT="-W0" rake test:silent --verbose
|
66
68
|
test-single-file-continuously:
|
67
69
|
\watch -n 5 --color rake test TEST="test/media/test_local_folder.rb"
|
68
70
|
|
data/Rakefile
CHANGED
@@ -29,6 +29,17 @@ namespace :test do
|
|
29
29
|
t.pattern = 'test/**/test_*.rb'
|
30
30
|
$DEBUG = true
|
31
31
|
end
|
32
|
+
|
33
|
+
desc "Test Silently and removes VERBOSITY"
|
34
|
+
Rake::TestTask.new(:silent) do |t|
|
35
|
+
t.libs << "test"
|
36
|
+
t.libs << "test/media"
|
37
|
+
t.verbose = false
|
38
|
+
t.warning = false
|
39
|
+
t.pattern = 'test/**/test_*.rb'
|
40
|
+
puts "Note: Best to call me with RUBYOPT='-W0'. Now RUBYOPT=#{ ENV['RUBYOPT'] }" if ENV['RUBYOPT'].nil?
|
41
|
+
$DEBUG = false
|
42
|
+
end
|
32
43
|
end
|
33
44
|
# namespace :verbose_test do
|
34
45
|
# desc "Test tests/media/* code"
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.5.
|
1
|
+
0.5.7
|
data/bin/stats-with-md5
CHANGED
@@ -109,41 +109,6 @@ def compute_stats_and_md5(file)
|
|
109
109
|
ret = {}
|
110
110
|
ret[:name] = file
|
111
111
|
|
112
|
-
<<<<<<< HEAD
|
113
|
-
begin
|
114
|
-
stats = File.stat(file)
|
115
|
-
ret[:stats_object] = stats # TODO deprecate
|
116
|
-
deb("Stats methods: #{stats.methods.sort.join(', ')}")
|
117
|
-
deb(stats.ctime)
|
118
|
-
# puts(stats.birthtime rescue (stats.ctime))
|
119
|
-
# On Mac/Linux: see test/dumps/***.yaml
|
120
|
-
ret[:size] = stats.size
|
121
|
-
ret[:mode] = stats.mode
|
122
|
-
# datetimes
|
123
|
-
ret[:stat_safebirthtime] = getSafeCreationTime(file) # in Mac uses birthtime,but on Linux wont work
|
124
|
-
# defined?(stats.birthtime) ? # rescue stats.mtime # eg, 2022-03-05 21:47:51 +0100 on Mac (not implemented on my Linuix)#
|
125
|
-
# stats.birthtime : # works on Mac
|
126
|
-
# stats.ctime
|
127
|
-
ret[:stat_mtime] = stats.mtime # eg, 2022-03-05 21:47:51 +0100 Returns the modification time of stat.
|
128
|
-
ret[:stat_ctime] = stats.ctime # eg, 2022-03-05 21:47:51 +0100 Returns the change time for stat (that is, the time directory information about the file was changed, not the file itself). Note that on Windows (NTFS), returns creation time (birth time).
|
129
|
-
ret[:stat_atime] = stats.atime # eg, 2022-03-05 21:47:51 +0100 Last Access
|
130
|
-
ret[:stat_gid] = stats.gid # Group Id
|
131
|
-
ret[:stat_uid] = stats.uid # UUID
|
132
|
-
ret[:stat_ftype] = stats.ftype #
|
133
|
-
|
134
|
-
ret[:md5] = '______________NONE______________'
|
135
|
-
if stats.ftype != "directory"
|
136
|
-
file_content = File.read(file)
|
137
|
-
ret[:md5] = Digest::MD5.hexdigest(file_content) # rescue 'none ' #=> string with hexadecimal digits
|
138
|
-
end
|
139
|
-
rescue Errno::EISDIR => e
|
140
|
-
# puts "It's a dir, nothing I can do here except skipping the stuff"
|
141
|
-
ret[:md5] = "_________I'm a dir sorry________"
|
142
|
-
rescue Exception => e
|
143
|
-
ret[:error] = e
|
144
|
-
end
|
145
|
-
ret
|
146
|
-
=======
|
147
112
|
begin
|
148
113
|
stats = File.stat(file)
|
149
114
|
ret[:stats_object] = stats # TODO deprecate
|
@@ -177,7 +142,6 @@ def compute_stats_and_md5(file)
|
|
177
142
|
ret[:error] = e
|
178
143
|
end
|
179
144
|
ret
|
180
|
-
>>>>>>> 97a64f1 (perfected CLI)
|
181
145
|
end
|
182
146
|
|
183
147
|
$print_stats_and_md5_version = "1.1b_220805_F" # files
|
data/bin/storazzo
CHANGED
@@ -1,7 +1,12 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
3
|
require 'storazzo'
|
4
|
+
#gem 'storazzo', :path => File.expand_path("../lib", __FILE__)
|
4
5
|
|
6
|
+
require 'optparse'
|
7
|
+
|
8
|
+
extend Storazzo::Colors
|
9
|
+
include Storazzo::Common # instead
|
5
10
|
|
6
11
|
if RUBY_VERSION.split('.')[0] == 1
|
7
12
|
puts "Refusing to launch a script form Ruby 1. Sorry Ric, its 2020 damn it!"
|
@@ -11,7 +16,20 @@ if RUBY_VERSION.split('.')[0] == 1
|
|
11
16
|
$PROG_VER = '0.1'
|
12
17
|
$DEBUG = ENV['DEBUG'] == 'true' # (true/false)
|
13
18
|
|
14
|
-
|
19
|
+
$actions = %w{
|
20
|
+
show auto help
|
21
|
+
}.sort
|
22
|
+
|
23
|
+
|
24
|
+
# you can't but love Ruby monkey-patching ability..
|
25
|
+
class Array
|
26
|
+
# puts "array drop"
|
27
|
+
def tail
|
28
|
+
drop(1) # https://stackoverflow.com/questions/3615700/ruby-what-is-the-easiest-way-to-remove-the-first-element-from-an-array
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
HISTORY = <<-BIG_LONG_MULTILINE
|
15
33
|
2022-0806 v0.1 First stesure
|
16
34
|
BIG_LONG_MULTILINE
|
17
35
|
|
@@ -25,8 +43,118 @@ $description = <<DESCRIPTION_HEREFILE.gsub(/\s+/, " ").strip
|
|
25
43
|
DESCRIPTION_HEREFILE
|
26
44
|
# >> "SELECT * FROM users ORDER BY users.id DESC"
|
27
45
|
|
28
|
-
|
29
|
-
|
46
|
+
$myconf = {
|
47
|
+
:app_name => "Storazzo CLI: #{$0}",
|
48
|
+
:description => $description,
|
49
|
+
# TODO move to some class default
|
50
|
+
#:media_dirs => %w{/media/riccardo/ /Volumes/ /mnt/ ~/git/storazzo/var/test/
|
51
|
+
# /sobenme/giusto/per/imparare/ad/ammutolire/gli/errori/},
|
52
|
+
#:mount_types => %w{vfat ntfs},
|
53
|
+
}
|
54
|
+
|
55
|
+
def usage(comment = nil)
|
56
|
+
puts white($optparse.banner)
|
57
|
+
puts($optparse.summarize)
|
58
|
+
#puts("Description: " + gray($myconf[:description]))
|
59
|
+
puts red("ERROR: " + comment) if comment
|
60
|
+
# puts "Description: #{ $myconf[:description] }"
|
61
|
+
exit 13
|
62
|
+
end
|
63
|
+
|
64
|
+
# include it in main if you want a custome one
|
65
|
+
def init() # see lib_autoinit in lib/util.rb
|
66
|
+
$opts = {}
|
67
|
+
# setting defaults
|
68
|
+
$opts[:verbose] = false
|
69
|
+
$opts[:dryrun] = false
|
70
|
+
$opts[:debug] = false
|
71
|
+
$opts[:force] = false
|
72
|
+
|
73
|
+
$optparse = OptionParser.new do |opts|
|
74
|
+
opts.banner = "#{$0} v.#{$PROG_VER}\n Usage: #{File.basename $0} [options] ACTION [file1] [file2] ..."
|
75
|
+
opts.separator "Actions: #{$actions.join ', '}"
|
76
|
+
opts.separator " $0 show # show options and local folders "
|
77
|
+
opts.separator " $0 auto # takes automated actions based on config and local disk. For lazy users"
|
78
|
+
opts.separator "Options: "
|
79
|
+
opts.on('-d', '--debug', 'enables debug (DFLT=false)') { $opts[:debug] = true; $DEBUG = true }
|
80
|
+
opts.on('-f', '--force', 'force stuff (DFLT=false)') { $opts[:force] = true }
|
81
|
+
opts.on('-h', '--help', 'Display this screen') { usage }
|
82
|
+
# opts.on( '-j', '--jabba', 'Activates my Jabber powerful CLI' ) { $opts[:jabba] = true }
|
83
|
+
opts.on('-n', '--dryrun', "Don't really execute code") { $opts[:dryrun] = true }
|
84
|
+
opts.on('-l', '--logfile FILE', 'Write log to FILE') { |file| $opts[:logfile] = file }
|
85
|
+
opts.on('-v', '--verbose', 'Output more information') { $opts[:verbose] = true }
|
86
|
+
opts.separator("Description: " + gray($myconf[:description]))
|
87
|
+
#opts.separator " -- "
|
88
|
+
end
|
89
|
+
$optparse.parse!
|
90
|
+
end
|
91
|
+
|
92
|
+
def show_action(argv)
|
93
|
+
deb "ARGV remaning is: #{argv}"
|
94
|
+
deb("TODO show_action")
|
95
|
+
#puts(white("1. Config"))
|
96
|
+
#Pry::ColorPrinter.pp $config.to_verbose_s
|
97
|
+
puts(white("2. Mounts/Dirs"))
|
98
|
+
Pry::ColorPrinter.pp Storazzo::Media::GcsBucket.list_all_with_type
|
99
|
+
pp Storazzo::Media::AbstractRicDisk.super_duper_list_all_with_type
|
100
|
+
|
101
|
+
end
|
102
|
+
|
103
|
+
def real_program
|
104
|
+
deb("Hello world from a templated '#{yellow $0}'")
|
105
|
+
deb "+ Options are: #{gray $opts}"
|
106
|
+
deb "+ Depured args: #{azure ARGV}"
|
107
|
+
deb "+ Script-specifig super-cool conf: #{green $prog_conf_d}"
|
108
|
+
deb "+ Your configuration: #{purple $myconf.inspect}"
|
109
|
+
|
110
|
+
# Your code goes here...
|
111
|
+
puts white("Hello world from #{$myconf[:app_name]}!")
|
112
|
+
puts "Description: '''#{white $myconf[:description]}'''"
|
113
|
+
Storazzo::Main.say_hi("Note: if the version of this storazzo is behind local version youre probably using a old library.
|
30
114
|
I still need to learn how to call the binary with local/current lib/: bundle exec stiKazzi?")
|
31
|
-
Storazzo::Main.say_hi($description)
|
32
115
|
|
116
|
+
$config = Storazzo::RicDiskConfig.instance()
|
117
|
+
$config.load
|
118
|
+
deb "StorazzoConfig: #{$config}"
|
119
|
+
# config.load # auto_sbrodola(ARGV)
|
120
|
+
puts yellow("ARGV: #{ARGV}")
|
121
|
+
#config.iterate_through_file_list_for_disks(ARGV)
|
122
|
+
|
123
|
+
if_deb? { Storazzo::Main.say_hi("ARGV: #{ARGV}") }
|
124
|
+
|
125
|
+
if (ARGV == [])
|
126
|
+
usage("I need at least one argument (storazzo ACTION)")
|
127
|
+
end
|
128
|
+
argv_action = ARGV[0]
|
129
|
+
case argv_action # first argment - action
|
130
|
+
when "show", "list"
|
131
|
+
show_action(ARGV.tail)
|
132
|
+
when "auto"
|
133
|
+
auto_action(ARGV.tail)
|
134
|
+
when "help"
|
135
|
+
usage("There you go, here's your HELP :)")
|
136
|
+
when String
|
137
|
+
usage("Unknown action1: #{argv_action}. Available actions: #{$actions}")
|
138
|
+
else
|
139
|
+
usage("Unknown action2, really strange!: '#{argv_action}'. Actions: #{$actions}")
|
140
|
+
end
|
141
|
+
|
142
|
+
end
|
143
|
+
|
144
|
+
def main(filename)
|
145
|
+
deb "I'm called by #{white filename}"
|
146
|
+
deb "HISTORY: #{gray HISTORY}"
|
147
|
+
init # Enable this to have command line parsing capabilities!
|
148
|
+
puts white("$DEBUG is #{$DEBUG}. Tu turn on, call me with -d") unless $DEBUG
|
149
|
+
# warn "[warn] template v#{$TEMPLATE_VER }: proviamo il warn che magari depreca il DEB"
|
150
|
+
real_program
|
151
|
+
|
152
|
+
if_deb? do
|
153
|
+
puts "First I need to figure out how to bring in all the libraries in here.."
|
154
|
+
Storazzo::Main.say_hi("ARGV is: #{ARGV.join ', '}")
|
155
|
+
puts Storazzo.version
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
|
160
|
+
main(__FILE__)
|
data/lib/storazzo/common.rb
CHANGED
@@ -34,7 +34,8 @@ module Storazzo::Common
|
|
34
34
|
include Storazzo::Colors
|
35
35
|
|
36
36
|
def deb(s)
|
37
|
-
|
37
|
+
#useless: put logic in _debug_true() # return if ENV['RUBYOPT'] == "-W0"
|
38
|
+
puts "[DEB👀] #{yellow(s)}" if _debug_true() # $DEBUG
|
38
39
|
end
|
39
40
|
|
40
41
|
# this has a yield
|
@@ -60,6 +61,7 @@ module Storazzo::Common
|
|
60
61
|
|
61
62
|
def pverbose(is_verbose, str)
|
62
63
|
# puts "[V📚RB💀S📚] #{gray str}"
|
64
|
+
return if ENV['RUBYOPT'] == "-W0"
|
63
65
|
puts "[🦋🐛🐝🐞🐜🕷🕸🦂🦗🦟] #{gray str}" # insects: http://xahlee.info/comp/unicode_insects.html
|
64
66
|
end
|
65
67
|
|
@@ -83,7 +85,8 @@ module Storazzo::Common
|
|
83
85
|
private
|
84
86
|
|
85
87
|
def _debug_true
|
86
|
-
|
88
|
+
return false if ENV["RUBYOPT"] == "-W0"
|
89
|
+
$DEBUG or ENV["DEBUG"] == 'true' # but FALSE isor _debug_true
|
87
90
|
end
|
88
91
|
|
89
92
|
# puts "[DEBUG ENABLED!]" if _debug_true
|
@@ -1,3 +1,23 @@
|
|
1
|
+
#require "storazzo/ric_disk/abstract_ric_disk"
|
2
|
+
#require "storazzo/ric_disk/gcs_bucket"
|
3
|
+
#require "gcs_bucket"
|
4
|
+
# require "abstract_ric_disk"
|
5
|
+
|
6
|
+
=begin
|
7
|
+
This class is an Abstract Class which forces its 3 inquilines to implement
|
8
|
+
a number of functions, namely:
|
9
|
+
|
10
|
+
def self.list_all()
|
11
|
+
def self.list_all_with_type()
|
12
|
+
|
13
|
+
Note this needs to work without weird overrides, like:
|
14
|
+
|
15
|
+
def self.list_all_with_type(config=nil) # BAD
|
16
|
+
def self.list_all_with_type() # GOOD
|
17
|
+
|
18
|
+
|
19
|
+
=end
|
20
|
+
|
1
21
|
module Storazzo::Media
|
2
22
|
class Storazzo::Media::AbstractRicDisk
|
3
23
|
# include Storazzo::Common
|
@@ -21,11 +41,11 @@ module Storazzo::Media
|
|
21
41
|
end
|
22
42
|
|
23
43
|
def self.list_all
|
24
|
-
raise "[AbstractRicDiskc::self.list_all] You should override this, says StackOverflow and Riccardo"
|
44
|
+
raise "[AbstractRicDiskc::self.list_all] You should override this, says StackOverflow and Riccardo. Class=#{self.class}. Self=#{self}"
|
25
45
|
end
|
26
46
|
|
27
47
|
def self.list_all_with_type
|
28
|
-
raise "[
|
48
|
+
raise "[AbstractRicDisk::self.list_all_with_type] You should override this, says StackOverflow and Riccardo. Class=#{self.class}. Self=#{self}"
|
29
49
|
end
|
30
50
|
|
31
51
|
def self.super_duper_list_all_with_type
|
@@ -36,11 +56,11 @@ module Storazzo::Media
|
|
36
56
|
end
|
37
57
|
|
38
58
|
def parse(opts = {})
|
39
|
-
raise "[AbstractRicDiskc::parse] You should override this, says StackOverflow and Riccardo"
|
59
|
+
raise "[AbstractRicDiskc::parse] You should override this, says StackOverflow and Riccardo. Class=#{self.class}."
|
40
60
|
end
|
41
61
|
|
42
62
|
def writeable?
|
43
|
-
raise "[AbstractRicDiskc::writeable] You should override this in
|
63
|
+
raise "[AbstractRicDiskc::writeable] You should override this in class=#{self.class}, says StackOverflow and Riccardo"
|
44
64
|
end
|
45
65
|
|
46
66
|
def get_local_mountpoint
|
@@ -137,8 +157,8 @@ module Storazzo::Media
|
|
137
157
|
end
|
138
158
|
|
139
159
|
def validate(opts = {})
|
140
|
-
|
141
|
-
|
160
|
+
opt_verbose = opts.fetch :verbose, false
|
161
|
+
pverbose opt_verbose, "validate(): [BROKEN] We're trying to see if your object is valid, across 3 possible sub-classes."
|
142
162
|
# 1. check for
|
143
163
|
raise "Unknown local mount " unless local_mount.is_a?(String)
|
144
164
|
|
@@ -1,14 +1,16 @@
|
|
1
1
|
module Storazzo::Media
|
2
|
-
class Storazzo::
|
2
|
+
# class Storazzo::Media::AbstractRicDisk
|
3
|
+
class Storazzo::Media::GcsBucket < Storazzo::Media::AbstractRicDisk
|
3
4
|
extend Storazzo::Common
|
4
5
|
|
5
6
|
attr_accessor :project_id
|
6
7
|
|
7
|
-
def initialize(local_mount, project_id
|
8
|
+
def initialize(local_mount, project_id=nil)
|
8
9
|
deb "[Storazzo::Media::GcsBucket] initialize"
|
9
10
|
|
10
11
|
@local_mountpoint = File.expand_path(local_mount)
|
11
12
|
@description = "MountPoint in '#{local_mount}' pointing at TODO with mount options = TODO"
|
13
|
+
project_id ||= _autodetect_project_id()
|
12
14
|
@project_id = project_id
|
13
15
|
raise "Sorry local mount doesnt exist!" unless File.exist?(@local_mountpoint)
|
14
16
|
|
@@ -16,7 +18,11 @@ module Storazzo::Media
|
|
16
18
|
super(local_mount) rescue "SUPER_ERROR(#{local_mount}): #{$!}"
|
17
19
|
end
|
18
20
|
|
19
|
-
def
|
21
|
+
def _autodetect_project_id
|
22
|
+
"todo-my-project-id-123" # TODO fix
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.list_all() # (config = nil)
|
20
26
|
# get lisrts from Config singletone
|
21
27
|
# puts " self.list_all: loading config "
|
22
28
|
config ||= Storazzo::RicDiskConfig.instance # # ).get_config
|
@@ -32,25 +38,32 @@ module Storazzo::Media
|
|
32
38
|
config.get_bucket_paths
|
33
39
|
end
|
34
40
|
|
35
|
-
def self.list_all_with_type(config = nil)
|
41
|
+
def self.list_all_with_type() # (config = nil)
|
42
|
+
deb "GCS::list_all_with_type() -- pull config"
|
36
43
|
config ||= Storazzo::RicDiskConfig.instance
|
37
44
|
config.load # in case I need to load it for the first time
|
38
|
-
|
45
|
+
#puts "GCS::list_all_with_type() -- TODO(ricc): also add gsutil ls"
|
39
46
|
# getFromConfig
|
40
47
|
deb "I'm now returning a 'complex' array to tell the caller what kind of element they're getting, eg: GCS from Config Yaml, vs GCS from gsutil ls "
|
41
48
|
list_from_config_with_type = config.get_bucket_paths.map { |path| [:config_gcs_bucket, path] }
|
42
|
-
|
49
|
+
ret_list = list_from_config_with_type
|
50
|
+
if (config.gcs_enabled?) # should be if GCS enabled :) project_id
|
43
51
|
# so I concatenate Apples with Bananas with names
|
44
|
-
|
52
|
+
#ret_list += list_available_buckets(config.project_id).map { |path|
|
53
|
+
ret_list += list_available_buckets.map { |path|
|
45
54
|
[:gsutil_ls_gcs_bucket, path]
|
46
55
|
}
|
56
|
+
else
|
57
|
+
deb "list_all_with_type(): empty project id. Skipping"
|
47
58
|
end
|
48
59
|
|
49
|
-
return
|
60
|
+
return ret_list
|
50
61
|
end
|
51
62
|
|
52
|
-
def self.list_available_buckets(
|
53
|
-
|
63
|
+
def self.list_available_buckets(opts = {})
|
64
|
+
#project_id = opts.fetch :project_id, nil
|
65
|
+
#list_of_buckets = `gsutil ls --project '#{project_id}'`.chomp.split("\n")
|
66
|
+
list_of_buckets = `gsutil ls`.chomp.split("\n")
|
54
67
|
deb "list_of_buckets: #{list_of_buckets}"
|
55
68
|
list_of_buckets
|
56
69
|
end
|
@@ -20,9 +20,9 @@ module Storazzo::Media
|
|
20
20
|
super(local_mount) rescue "SUPER_ERROR(#{local_mount}): #{$!}"
|
21
21
|
end
|
22
22
|
|
23
|
-
def self.list_all(config = nil
|
23
|
+
def self.list_all() # WRONG: config = nil
|
24
24
|
# get lists from Config singleton
|
25
|
-
config
|
25
|
+
config = Storazzo::RicDiskConfig.instance # # ).get_config
|
26
26
|
config.load
|
27
27
|
config.get_local_folders
|
28
28
|
end
|
@@ -7,6 +7,14 @@ module Storazzo::Media
|
|
7
7
|
deb "Maybe its abuot time you refactor that method here :)"
|
8
8
|
RicDisk.interesting_mount_points()
|
9
9
|
end
|
10
|
+
|
11
|
+
def self.list_all
|
12
|
+
RicDisk.interesting_mount_points
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.list_all_with_type()
|
16
|
+
list_all.map{|x| [:mount_point_todo_less_generic, x] }
|
17
|
+
end
|
10
18
|
end
|
11
19
|
end
|
12
20
|
|
@@ -41,7 +41,7 @@ module Storazzo
|
|
41
41
|
File.expand_path('../../../', __FILE__) +
|
42
42
|
"/etc/storazzo_config.sample.yaml"
|
43
43
|
|
44
|
-
attr_accessor :config, :config_file, :load_called
|
44
|
+
attr_accessor :config, :config_file, :load_called, :project_id
|
45
45
|
|
46
46
|
public
|
47
47
|
|
@@ -50,7 +50,7 @@ module Storazzo
|
|
50
50
|
verbose = opts.fetch :verbose, false
|
51
51
|
|
52
52
|
if already_loaded? # and not self.config.nil?
|
53
|
-
pverbose verbose, "[#{self.class}]
|
53
|
+
pverbose verbose, "[#{self.class}] Config.Load: already loaded"
|
54
54
|
return self.config
|
55
55
|
end
|
56
56
|
pverbose verbose, "Storazzo::RicDiskConfig.load(): BEGIN"
|
@@ -65,7 +65,7 @@ module Storazzo
|
|
65
65
|
# OR: possible_locations.instert(0, config_path)
|
66
66
|
pverbose verbose, "[LOAD] possible_locations: #{possible_locations}"
|
67
67
|
end
|
68
|
-
|
68
|
+
pverbose verbose, "Searching these paths in order: #{possible_locations}"
|
69
69
|
# bug "This is not always an array of sTRINGS."
|
70
70
|
raise "possible_locations is not an array" unless possible_locations.is_a?(Array)
|
71
71
|
|
@@ -90,7 +90,6 @@ module Storazzo
|
|
90
90
|
config_ver = @config["apiVersion"]
|
91
91
|
# puts @config[:ConfigVersion]
|
92
92
|
deb("OK. Storazzo::RicDiskConfig v'#{config_ver}' parsed correctly")
|
93
|
-
# puts "[VERBOSE] RicDiskConfig.to_s: #{self}" if verbose
|
94
93
|
@load_called = true
|
95
94
|
return self.config
|
96
95
|
end
|
@@ -121,6 +120,21 @@ module Storazzo
|
|
121
120
|
@config['Config']['Backends']['GoogleCloudStorage']['ProjectId']
|
122
121
|
end
|
123
122
|
|
123
|
+
#doesnt work :/ alias :project_id, :config_project_id
|
124
|
+
def project_id
|
125
|
+
config_project_id
|
126
|
+
end
|
127
|
+
|
128
|
+
# This should return true if and only if the user has configured the YAML
|
129
|
+
# to use GCS. For now we can just say TRUE since gsutil ls returns without project_id.
|
130
|
+
# However, we might have a flag enabled in storazzo config in next versions.
|
131
|
+
def gcs_enabled?
|
132
|
+
#true
|
133
|
+
defined?(get_config['Config']['Backends']['GoogleCloudStorage'])
|
134
|
+
#get_config['Config']['Backends']['GoogleCloudStorage DOESNT EXIST']
|
135
|
+
# defined? Backends: GoogleCloudStorage
|
136
|
+
end
|
137
|
+
|
124
138
|
def already_loaded?
|
125
139
|
# return
|
126
140
|
load_called == true
|
@@ -198,16 +212,16 @@ module Storazzo
|
|
198
212
|
# RicDisk.calculate_stats_files(dir) # dir is inutile
|
199
213
|
# } # TODO refactor in option sbrodola_afterwards=true. :)
|
200
214
|
# else
|
201
|
-
|
215
|
+
pverbose verbose, "iterate_through_file_list_for_disks(): I consider files_list as a list of directories to parse :)"
|
202
216
|
|
203
217
|
# dirs = RicDisk.find_active_dirs()
|
204
218
|
files_list.each do |dir|
|
205
219
|
dir = File.expand_path(dir)
|
206
220
|
if File.directory?(dir)
|
207
221
|
# if dirs.include?(dir)
|
208
|
-
|
222
|
+
pverbose verbose, "iterate_through_file_list_for_disks() Legit dir: #{green dir}"
|
209
223
|
rd = RicDisk.new(Storazzo::Media::AbstractRicDisk.DirFactory(dir))
|
210
|
-
pverbose
|
224
|
+
pverbose verbose, "RicDisk from Factory (woohoo): #{rd}"
|
211
225
|
rd.write_config_yaml_to_disk(dir)
|
212
226
|
# RicDisk.write_config_yaml_to_disk(dir)
|
213
227
|
# RicDisk.calculate_stats_files (CLASS) => will become OBJECT compute_stats_files
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# copied from https://github.com/minitest/minitest
|
2
|
+
|
3
|
+
# optionally run benchmarks, good for CI-only work!
|
4
|
+
require "minitest/benchmark" if ENV["BENCHMARK"]
|
5
|
+
|
6
|
+
class TestMeme < Minitest::Benchmark
|
7
|
+
# Override self.bench_range or default range is [1, 10, 100, 1_000, 10_000]
|
8
|
+
def bench_my_algorithm
|
9
|
+
assert_performance_linear 0.9999 do |n| # n is a range value
|
10
|
+
@obj.my_algorithm(n)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
|
2
|
+
|
3
|
+
|
4
|
+
|
5
|
+
require "minitest/autorun"
|
6
|
+
require "storazzo"
|
7
|
+
|
8
|
+
class BinaryStorazzoTest < Minitest::Test
|
9
|
+
|
10
|
+
def test_binary_returns_error_with_random_argv
|
11
|
+
ret = `bin/storazzo aaa`
|
12
|
+
#puts ret
|
13
|
+
#puts "return code is '#{$?.exitstatus}'"
|
14
|
+
should_match_string = 'Unknown action1: aaa. Available actions:' # 'ERROR: Unknown action1: aaa. Available actions: ["auto", "help", "show"]'
|
15
|
+
assert_equal(
|
16
|
+
$?.exitstatus,
|
17
|
+
13,
|
18
|
+
"Wrong argument should return 13"
|
19
|
+
)
|
20
|
+
matches_expected_error = ret.match(should_match_string)
|
21
|
+
#puts "matches_expected_error: ''#{matches_expected_error}''"
|
22
|
+
assert(
|
23
|
+
not( matches_expected_error.nil?),
|
24
|
+
"Binary should return a string which should contain this string: '#{should_match_string}'"
|
25
|
+
)
|
26
|
+
end
|
27
|
+
|
28
|
+
end
|
@@ -7,7 +7,7 @@ require "storazzo"
|
|
7
7
|
class AbstractRicDiskTest < Minitest::Test
|
8
8
|
include Storazzo::Common
|
9
9
|
|
10
|
-
def
|
10
|
+
def test_super_duper_list_all_with_type_returns_something
|
11
11
|
deb "This would be already... something :) it means they're all implemented"
|
12
12
|
ret = Storazzo::Media::AbstractRicDisk.super_duper_list_all_with_type
|
13
13
|
assert(
|
@@ -3,30 +3,33 @@ require "storazzo"
|
|
3
3
|
require "storazzo/ric_disk"
|
4
4
|
require "storazzo/ric_disk_config"
|
5
5
|
require 'storazzo/colors'
|
6
|
-
require "storazzo/media/local_folder"
|
6
|
+
#require "storazzo/media/local_folder"
|
7
|
+
require "storazzo/media/gcs_bucket"
|
7
8
|
|
8
9
|
require 'pry' # must install the gem... but you ALWAYS want pry installed anyways
|
9
10
|
|
11
|
+
# TEST: watch rake test TEST="test/media/test_gcs_bucket.rb"
|
12
|
+
|
10
13
|
class GcsBucketTest < Minitest::Test
|
11
14
|
include Storazzo::Common
|
12
15
|
|
13
16
|
def setup # tear_up
|
14
17
|
deb "[GcsBucketTest] TEAR_UP with sample Config"
|
15
18
|
# removeme = Storazzo::RicDiskConfig.instance()
|
16
|
-
|
17
|
-
|
18
|
-
deb "[GcsBucketTest] TEAR_UP config_obj: '''#{
|
19
|
+
@sample_config_obj = Storazzo::RicDiskSampleConfig.instance()
|
20
|
+
@sample_config_hash = @sample_config_obj.load()
|
21
|
+
deb "[GcsBucketTest] TEAR_UP config_obj: '''#{@sample_config_obj}'''"
|
19
22
|
end
|
20
23
|
|
21
|
-
def
|
24
|
+
def test_that_test_buckets_are_the_two_I_know
|
22
25
|
# copied from etc/sample.yaml :)
|
23
|
-
|
26
|
+
expected_test_buckets_list = %w{
|
24
27
|
gs://my-local-backup/storazzo/backups/
|
25
28
|
gs://my-other-bucket/
|
26
29
|
}
|
27
|
-
actual_list = Storazzo::
|
30
|
+
actual_list = Storazzo::Media::GcsBucket.list_all(@sample_config_obj)
|
28
31
|
assert_equal(
|
29
|
-
|
32
|
+
expected_test_buckets_list.sort,
|
30
33
|
actual_list.sort,
|
31
34
|
"These are the two lists from Sample Storazzo Config"
|
32
35
|
)
|
@@ -41,18 +44,48 @@ class GcsBucketTest < Minitest::Test
|
|
41
44
|
# puts Storazzo::RicDiskSampleConfig
|
42
45
|
# config_obj = Storazzo::RicDiskSampleConfig.instance()
|
43
46
|
if_deb? do
|
44
|
-
Pry::ColorPrinter.pp(
|
47
|
+
Pry::ColorPrinter.pp(@sample_config_obj.to_verbose_s())
|
45
48
|
end
|
46
49
|
# pp green(config_obj.to_verbose_s())
|
47
50
|
|
48
|
-
l =
|
51
|
+
l = @sample_config_obj.load
|
49
52
|
Pry::ColorPrinter.pp(l) if $DEBUG
|
50
53
|
|
51
|
-
puts "
|
54
|
+
puts "@sample_config_obj: #{@sample_config_obj}"
|
52
55
|
# config_obj.load # _sample_version
|
53
56
|
end
|
54
57
|
|
55
|
-
|
56
|
-
|
58
|
+
def test_super_duper_list_works
|
59
|
+
# we had a problem on GCS side
|
60
|
+
#Storazzo::Media::AbstractRicDisk.super_duper_list_all_with_type
|
61
|
+
expected_ret = [
|
62
|
+
[:config_gcs_bucket, "gs://my-local-backup/storazzo/backups/"],
|
63
|
+
[:config_gcs_bucket, "gs://my-other-bucket/"]
|
64
|
+
]
|
65
|
+
ret = Storazzo::Media::GcsBucket.list_all_with_type()
|
66
|
+
Pry::ColorPrinter.pp(ret)
|
67
|
+
assert_equal(
|
68
|
+
ret.class,
|
69
|
+
Array,
|
70
|
+
"test_super_duper_list_all_with_type_returns_something should return an array.."
|
71
|
+
)
|
72
|
+
assert_equal(ret,expected_ret,"These are the two buckets I expect from test.."
|
73
|
+
)
|
74
|
+
end
|
75
|
+
|
76
|
+
def test_gsutil_returns_something
|
77
|
+
ret = Storazzo::Media::GcsBucket.list_available_buckets()
|
78
|
+
Pry::ColorPrinter.pp(ret)
|
79
|
+
end
|
80
|
+
|
81
|
+
# def test_super_duper_list_all_with_type_returns_something
|
82
|
+
# deb "This would be already... something :) it means they're all implemented"
|
83
|
+
# ret = Storazzo::Media::AbstractRicDisk.super_duper_list_all_with_type
|
84
|
+
# assert(
|
85
|
+
# ret.class,
|
86
|
+
# Array,
|
87
|
+
# "test_super_duper_list_all_with_type_returns_something should return an array.."
|
88
|
+
# )
|
57
89
|
# end
|
90
|
+
|
58
91
|
end
|
@@ -16,14 +16,14 @@ class LocalFolderTest < Minitest::Test
|
|
16
16
|
# #"Hello from Storazzo", Storazzo::Main.hi("ruby this should fail")
|
17
17
|
# #assert_match "ruby this should fail", Storazzo::Main.hi("ruby this should fail")
|
18
18
|
# end
|
19
|
-
def tear_up
|
20
|
-
|
19
|
+
#def tear_up
|
20
|
+
def setup
|
21
|
+
include Storazzo::Colors
|
21
22
|
puts yellow("LocalFolderTest: tear up")
|
22
|
-
#
|
23
|
-
|
24
|
-
|
25
|
-
puts
|
26
|
-
|
23
|
+
# @config_useless = Storazzo::RicDiskConfig.instance()
|
24
|
+
@config = Storazzo::RicDiskSampleConfig.safe_instance()
|
25
|
+
@config_load = @config.load()
|
26
|
+
puts @config.to_verbose_s
|
27
27
|
# my_class = Storazzo::RicDisk::LocalFolder
|
28
28
|
# my_obj = Storazzo::RicDisk::LocalFolder
|
29
29
|
end
|
@@ -46,9 +46,7 @@ class LocalFolderTest < Minitest::Test
|
|
46
46
|
# To only test this:
|
47
47
|
# $ ruby -I test test/test_local_folder.rb -n test_first_directory_parsing_actually_works
|
48
48
|
def test_1_first_directory_parsing_actually_works()
|
49
|
-
# include module
|
50
49
|
|
51
|
-
# p $vediamo_se_funge
|
52
50
|
puts("(#{__FILE__}) WEIRD THING: This test is flaky. SKipping for now until I complete the LocalFolder.parse() code")
|
53
51
|
folders = Storazzo::Media::LocalFolder.list_all
|
54
52
|
puts "Folders: #{folders}"
|
@@ -72,8 +70,8 @@ class LocalFolderTest < Minitest::Test
|
|
72
70
|
end
|
73
71
|
|
74
72
|
def test_vars_transporeted_across_teraup_and_tests
|
75
|
-
puts
|
76
|
-
puts
|
73
|
+
puts @config_load
|
74
|
+
puts @config
|
77
75
|
end
|
78
76
|
|
79
77
|
def test_2_iterate_through_file_list_for_disks
|
@@ -20,6 +20,7 @@ class MediaMountPointTest < Minitest::Test
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def test_what_skip_means
|
23
|
-
skip 'Check for
|
23
|
+
skip 'Check for blah blah blah. Found example online. Please Riccardo fix. This is just an example of how to SKIP a test :)'
|
24
24
|
end
|
25
|
+
|
25
26
|
end
|
data/test/test_ric_disk.rb
CHANGED
@@ -2,8 +2,8 @@ require "minitest/autorun"
|
|
2
2
|
require "storazzo"
|
3
3
|
|
4
4
|
class RicDiskTest < Minitest::Test
|
5
|
-
def
|
6
|
-
# actual_list = Storazzo::
|
5
|
+
def test_factory_works_for_gcs
|
6
|
+
# actual_list = Storazzo::Media::GcsBucket.list_all($sample_config_obj)
|
7
7
|
rd1 = Storazzo::RicDisk.new('Doesnt accept a string, should be a Gcs Something... Plus why is this test here and not under TestGcsBucket?!?')
|
8
8
|
hash = rd1.to_verbose_s()
|
9
9
|
pp hash
|
@@ -17,7 +17,7 @@ class RicDiskConfigTest < Minitest::Test
|
|
17
17
|
config_obj = Storazzo::RicDiskSampleConfig.instance()
|
18
18
|
puts "config_obj.class: #{config_obj.class}"
|
19
19
|
config = config_obj.load # _sample_version
|
20
|
-
pverbose
|
20
|
+
pverbose $DEBUG, "[test_load_sample_version] Config: #{pp config}"
|
21
21
|
# puts '[RicDiskConfigTest] TODO lets make sure the gem being imported is actually in /etc/storazz-config.smaple blah blah'
|
22
22
|
# puts "[RicDiskConfigTest] config_file: ", config_obj.config_file
|
23
23
|
assert_equal(
|
@@ -1,22 +0,0 @@
|
|
1
|
-
# [print_stats_and_md5_for_directory] DIR to explore (make sure you glob also): .
|
2
|
-
# DEB Options: {:max_files=>nil, :color=>false, :verbose=>false, :autowrite=>false}
|
3
|
-
[file_v1.2] d41d8cd98f00b204e9800998ecf8427e 100644 f 2022-07-30T11:54:42+02:00 0 [text/plain] ./ricdisk_stats_v11.rds
|
4
|
-
[file_v1.2] ______________NONE______________ 040755 d 2022-07-28T12:11:48+02:00 200 [inode/directory] ./disk01-empty
|
5
|
-
[file_v1.2] ______________NONE______________ 040755 d 2022-07-29T19:00:54+02:00 300 [inode/directory] ./disk02-full
|
6
|
-
[file_v1.2] d41d8cd98f00b204e9800998ecf8427e 100644 f 2022-07-24T06:38:25+02:00 0 [inode/x-empty] ./disk02-full/fake file.touch
|
7
|
-
[file_v1.2] f344106fac7926960912486256a56af9 100644 f 2022-07-24T06:38:25+02:00 454 [text/plain] ./disk02-full/ls.txt
|
8
|
-
[file_v1.2] 437b184a6b045975f0fb6c8c978f28ad 100644 f 2022-07-29T19:01:26+02:00 507 [text/plain] ./disk02-full/Rakefile
|
9
|
-
# [print_stats_and_md5_for_directory] DIR to explore (make sure you glob also): ./disk01-empty
|
10
|
-
# DEB Options: {:max_files=>nil, :color=>false, :verbose=>false, :autowrite=>false}
|
11
|
-
[file_v1.2] f8382af52d00023dc4ee9e9029b367b9 100644 f 2022-07-28T14:30:46+02:00 1022 [text/plain] ./disk01-empty/.ricdisk
|
12
|
-
[file_v1.2] d41d8cd98f00b204e9800998ecf8427e 100644 f 2022-07-24T06:38:25+02:00 0 [inode/x-empty] ./disk01-empty/.keep
|
13
|
-
# [print_stats_and_md5_for_directory] DIR to explore (make sure you glob also): ./disk02-full
|
14
|
-
# DEB Options: {:max_files=>nil, :color=>false, :verbose=>false, :autowrite=>false}
|
15
|
-
[file_v1.2] d41d8cd98f00b204e9800998ecf8427e 100644 f 2022-07-24T06:38:25+02:00 0 [inode/x-empty] ./disk02-full/fake file.touch
|
16
|
-
[file_v1.2] f344106fac7926960912486256a56af9 100644 f 2022-07-24T06:38:25+02:00 454 [text/plain] ./disk02-full/ls.txt
|
17
|
-
[file_v1.2] 437b184a6b045975f0fb6c8c978f28ad 100644 f 2022-07-29T19:01:26+02:00 507 [text/plain] ./disk02-full/Rakefile
|
18
|
-
[file_v1.2] d41d8cd98f00b204e9800998ecf8427e 100644 f 2022-07-24T06:38:25+02:00 0 [inode/x-empty] ./disk02-full/.ricdisk
|
19
|
-
[file_v1.2] d41d8cd98f00b204e9800998ecf8427e 100644 f 2022-07-24T06:38:25+02:00 0 [inode/x-empty] ./disk02-full/fake file.touch
|
20
|
-
[file_v1.2] f344106fac7926960912486256a56af9 100644 f 2022-07-24T06:38:25+02:00 454 [text/plain] ./disk02-full/ls.txt
|
21
|
-
[file_v1.2] 437b184a6b045975f0fb6c8c978f28ad 100644 f 2022-07-29T19:01:26+02:00 507 [text/plain] ./disk02-full/Rakefile
|
22
|
-
# [stats-with-md5] Time taken for processing 15 files: 0.35655
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: storazzo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Riccardo Carlesso
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-08-
|
11
|
+
date: 2022-08-10 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pry
|
@@ -66,6 +66,8 @@ files:
|
|
66
66
|
- lib/storazzo/ric_disk_ugly.rb
|
67
67
|
- lib/storazzo/version.rb
|
68
68
|
- storazzo.gemspec
|
69
|
+
- test/benchmark/for_future_use.rb
|
70
|
+
- test/bin/storazzo.rb
|
69
71
|
- test/media/test_abstract_ric_disk.rb
|
70
72
|
- test/media/test_gcs_bucket.rb
|
71
73
|
- test/media/test_local_folder.rb
|
@@ -106,8 +108,10 @@ specification_version: 4
|
|
106
108
|
summary: storazzo is an amazing gem. Code is in https://github.com/palladius/storazzo
|
107
109
|
test_files:
|
108
110
|
- test/test_storazzo.rb
|
111
|
+
- test/benchmark/for_future_use.rb
|
109
112
|
- test/test_ric_disk_stats_file.rb
|
110
113
|
- test/test_ric_disk.rb
|
114
|
+
- test/bin/storazzo.rb
|
111
115
|
- test/test_ric_disk_config.rb
|
112
116
|
- test/media/test_gcs_bucket.rb
|
113
117
|
- test/media/test_mount_point.rb
|